gcc: Drop 7.3 since 8.2 is working fine for us

We've had gcc 8.2 around for long enough that 7.3 can be removed now.

(From OE-Core rev: 20aea61385e1a53ac245353899277ba20104ed2f)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

.gitignore

@@ -1,6 +1,7 @@
 *.pyc
 *.pyo
 /*.patch
+/.repo/
 /build*/
 pyshtables.py
 pstage/

bitbake/bin/bitbake-worker

@@ -136,7 +136,7 @@ def sigterm_handler(signum, frame):
     os.killpg(0, signal.SIGTERM)
     sys.exit()
 
-def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, appends, taskdepdata, extraconfigdata, quieterrors=False, dry_run_exec=False):
+def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, taskhash, appends, taskdepdata, extraconfigdata, quieterrors=False, dry_run_exec=False):
     # We need to setup the environment BEFORE the fork, since
     # a fork() or exec*() activates PSEUDO...
 
@@ -234,7 +234,7 @@ def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, append
                 ret = 0
 
                 the_data = bb_cache.loadDataFull(fn, appends)
-                the_data.setVar('BB_TASKHASH', workerdata["runq_hash"][task])
+                the_data.setVar('BB_TASKHASH', taskhash)
 
                 bb.utils.set_process_name("%s:%s" % (the_data.getVar("PN"), taskname.replace("do_", "")))
 
@@ -425,10 +425,10 @@ class BitbakeWorker(object):
         sys.exit(0)
 
     def handle_runtask(self, data):
-        fn, task, taskname, quieterrors, appends, taskdepdata, dry_run_exec = pickle.loads(data)
+        fn, task, taskname, taskhash, quieterrors, appends, taskdepdata, dry_run_exec = pickle.loads(data)
         workerlog_write("Handling runtask %s %s %s\n" % (task, fn, taskname))
 
-        pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.databuilder, self.workerdata, fn, task, taskname, appends, taskdepdata, self.extraconfigdata, quieterrors, dry_run_exec)
+        pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.databuilder, self.workerdata, fn, task, taskname, taskhash, appends, taskdepdata, self.extraconfigdata, quieterrors, dry_run_exec)
 
         self.build_pids[pid] = task
         self.build_pipes[pid] = runQueueWorkerPipe(pipein, pipeout)

bitbake/bin/toaster

@@ -161,7 +161,9 @@ fi
 
 export BBBASEDIR=`dirname $TOASTER`/..
 MANAGE="python3 $BBBASEDIR/lib/toaster/manage.py"
-OE_ROOT=`dirname $TOASTER`/../..
+if [ -z "$OE_ROOT" ]; then
+    OE_ROOT=`dirname $TOASTER`/../..
+fi
 
 # this is the configuraton file we are using for toaster
 # we are using the same logic that oe-setup-builddir uses

bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml

@@ -588,6 +588,14 @@
                         The name of the path in which to place the checkout.
                         By default, the path is <filename>git/</filename>.
                         </para></listitem>
+                    <listitem><para><emphasis>"usehead":</emphasis>
+                        Enables local <filename>git://</filename> URLs to use the
+                        current branch HEAD as the revision for use with
+                        <filename>AUTOREV</filename>.
+                        The "usehead" parameter implies no branch and only works
+                        when the transfer protocol is
+                        <filename>file://</filename>.
+                        </para></listitem>
                 </itemizedlist>
                 Here are some example URLs:
                 <literallayout class='monospaced'>

bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.xml

@@ -342,13 +342,14 @@
 
             <para>
                 When you name an append file, you can use the
-                wildcard character (%) to allow for matching recipe names.
+                "<filename>%</filename>" wildcard character to allow for matching
+                recipe names.
                 For example, suppose you have an append file named
                 as follows:
                 <literallayout class='monospaced'>
      busybox_1.21.%.bbappend
                 </literallayout>
-                That append file would match any <filename>busybox_1.21.x.bb</filename>
+                That append file would match any <filename>busybox_1.21.</filename><replaceable>x</replaceable><filename>.bb</filename>
                 version of the recipe.
                 So, the append file would match the following recipe names:
                 <literallayout class='monospaced'>
@@ -356,6 +357,14 @@
      busybox_1.21.2.bb
      busybox_1.21.3.bb
                 </literallayout>
+                <note><title>Important</title>
+                    The use of the "<filename>%</filename>" character
+                    is limited in that it only works directly in front of the
+                    <filename>.bbappend</filename> portion of the append file's
+                    name.
+                    You cannot use the wildcard character in any other
+                    location of the name.
+                </note>
                 If the <filename>busybox</filename> recipe was updated to
                 <filename>busybox_1.3.0.bb</filename>, the append name would not
                 match.

bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml

@@ -352,8 +352,9 @@
      FOO2_remove = "abc def"
                 </literallayout>
                 The variable <filename>FOO</filename> becomes
-                "  789 123456    " and <filename>FOO2</filename> becomes
-                "  ghi abcdef    ".
+                "&nbsp;&nbsp;789 123456&nbsp;&nbsp;&nbsp;&nbsp;"
+                and <filename>FOO2</filename> becomes
+                "&nbsp;&nbsp;ghi abcdef&nbsp;&nbsp;&nbsp;&nbsp;".
             </para>
 
             <para>
@@ -2751,4 +2752,30 @@
             </itemizedlist>
         </para>
     </section>
+
+    <section id='wildcard-support-in-variables'>
+        <title>Wildcard Support in Variables</title>
+
+        <para>
+            Support for wildcard use in variables varies depending on the
+            context in which it is used.
+            For example, some variables and file names allow limited use of
+            wildcards through the "<filename>%</filename>" and
+            "<filename>*</filename>" characters.
+            Other variables or names support Python's
+            <ulink url='https://docs.python.org/3/library/glob.html'><filename>glob</filename></ulink>
+            syntax,
+            <ulink url='https://docs.python.org/3/library/fnmatch.html#module-fnmatch'><filename>fnmatch</filename></ulink>
+            syntax, or
+            <ulink url='https://docs.python.org/3/library/re.html#re'><filename>Regular Expression (re)</filename></ulink>
+            syntax.
+        </para>
+
+        <para>
+            For variables that have wildcard suport, the
+            documentation describes which form of wildcard, its
+            use, and its limitations.
+        </para>
+    </section>
+
 </chapter>

bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.xml

@@ -115,7 +115,8 @@
                             is either not set or set to "0".
                             </para></listitem>
                         <listitem><para>
-                            Limited support for wildcard matching against the
+                            Limited support for the "<filename>*</filename>"
+                            wildcard character for matching against the
                             beginning of host names exists.
                             For example, the following setting matches
                             <filename>git.gnu.org</filename>,
@@ -124,6 +125,20 @@
                             <literallayout class='monospaced'>
      BB_ALLOWED_NETWORKS = "*.gnu.org"
                             </literallayout>
+                            <note><title>Important</title>
+                                <para>The use of the "<filename>*</filename>"
+                                character only works at the beginning of
+                                a host name and it must be isolated from
+                                the remainder of the host name.
+                                You cannot use the wildcard character in any
+                                other location of the name or combined with
+                                the front part of the name.</para>
+
+                                <para>For example,
+                                <filename>*.foo.bar</filename> is supported,
+                                while <filename>*aa.foo.bar</filename> is not.
+                                </para>
+                            </note>
                             </para></listitem>
                         <listitem><para>
                             Mirrors not in the host list are skipped and
@@ -1082,7 +1097,19 @@
 
         <glossentry id='var-BBFILES'><glossterm>BBFILES</glossterm>
             <glossdef>
-                <para>List of recipe files BitBake uses to build software.</para>
+                <para>
+                    A space-separated list of recipe files BitBake uses to
+                    build software.
+                </para>
+
+                <para>
+                    When specifying recipe files, you can pattern match using
+                    Python's
+                    <ulink url='https://docs.python.org/3/library/glob.html'><filename>glob</filename></ulink>
+                    syntax.
+                    For details on the syntax, see the documentation by
+                    following the previous link.
+                </para>
             </glossdef>
         </glossentry>
 
@@ -1166,15 +1193,19 @@
                     match any of the expressions.
                     It is as if BitBake does not see them at all.
                     Consequently, matching files are not parsed or otherwise
-                    used by BitBake.</para>
+                    used by BitBake.
+                </para>
+
                 <para>
                     The values you provide are passed to Python's regular
                     expression compiler.
+                    Consequently, the syntax follows Python's Regular
+                    Expression (re) syntax.
                     The expressions are compared against the full paths to
                     the files.
                     For complete syntax information, see Python's
                     documentation at
-                    <ulink url='http://docs.python.org/release/2.3/lib/re-syntax.html'></ulink>.
+                    <ulink url='http://docs.python.org/3/library/re.html#re'></ulink>.
                 </para>
 
                 <para>
@@ -1933,15 +1964,27 @@
                     you want to select, and you should set
                     <link linkend='var-PV'><filename>PV</filename></link>
                     accordingly for precedence.
-                    You can use the "<filename>%</filename>" character as a
-                    wildcard to match any number of characters, which can be
-                    useful when specifying versions that contain long revision
-                    numbers that could potentially change.
+                </para>
+
+                <para>
+                    The <filename>PREFERRED_VERSION</filename> variable
+                    supports limited wildcard use through the
+                    "<filename>%</filename>" character.
+                    You can use the character to match any number of
+                    characters, which can be useful when specifying versions
+                    that contain long revision numbers that potentially change.
                     Here are two examples:
                     <literallayout class='monospaced'>
      PREFERRED_VERSION_python = "2.7.3"
      PREFERRED_VERSION_linux-yocto = "4.12%"
                     </literallayout>
+                    <note><title>Important</title>
+                        The use of the "<filename>%</filename>" character
+                        is limited in that it only works at the end of the
+                        string.
+                        You cannot use the wildcard character in any other
+                        location of the string.
+                    </note>
                 </para>
             </glossdef>
         </glossentry>

bitbake/lib/bb/cache.py

@@ -97,7 +97,7 @@ class CoreRecipeInfo(RecipeInfoCommon):
 
         self.skipreason = self.getvar('__SKIPPED', metadata)
         if self.skipreason:
-            self.pn = self.getvar('PN', metadata) or bb.parse.BBHandler.vars_from_file(filename,metadata)[0]
+            self.pn = self.getvar('PN', metadata) or bb.parse.vars_from_file(filename,metadata)[0]
             self.skipped = True
             self.provides  = self.depvar('PROVIDES', metadata)
             self.rprovides = self.depvar('RPROVIDES', metadata)

bitbake/lib/bb/codeparser.py

@@ -368,8 +368,9 @@ class ShellParser():
     def _parse_shell(self, value):
         try:
             tokens, _ = pyshyacc.parse(value, eof=True, debug=False)
-        except pyshlex.NeedMore:
-            raise sherrors.ShellSyntaxError("Unexpected EOF")
+        except Exception:
+            bb.error('Error during parse shell code, the last 5 lines are:\n%s' % '\n'.join(value.split('\n')[-5:]))
+            raise
 
         self.process_tokens(tokens)
 

bitbake/lib/bb/cooker.py

@@ -175,18 +175,31 @@ class BBCooker:
 
         self.configuration = configuration
 
+        bb.debug(1, "BBCooker starting %s" % time.time())
+        sys.stdout.flush()
+
         self.configwatcher = pyinotify.WatchManager()
+        bb.debug(1, "BBCooker pyinotify1 %s" % time.time())
+        sys.stdout.flush()
+
         self.configwatcher.bbseen = []
         self.configwatcher.bbwatchedfiles = []
         self.confignotifier = pyinotify.Notifier(self.configwatcher, self.config_notifications)
+        bb.debug(1, "BBCooker pyinotify2 %s" % time.time())
+        sys.stdout.flush()
         self.watchmask = pyinotify.IN_CLOSE_WRITE | pyinotify.IN_CREATE | pyinotify.IN_DELETE | \
                          pyinotify.IN_DELETE_SELF | pyinotify.IN_MODIFY | pyinotify.IN_MOVE_SELF | \
                          pyinotify.IN_MOVED_FROM | pyinotify.IN_MOVED_TO
         self.watcher = pyinotify.WatchManager()
+        bb.debug(1, "BBCooker pyinotify3 %s" % time.time())
+        sys.stdout.flush()
         self.watcher.bbseen = []
         self.watcher.bbwatchedfiles = []
         self.notifier = pyinotify.Notifier(self.watcher, self.notifications)
 
+        bb.debug(1, "BBCooker pyinotify complete %s" % time.time())
+        sys.stdout.flush()
+
         # If being called by something like tinfoil, we need to clean cached data
         # which may now be invalid
         bb.parse.clear_cache()
@@ -196,6 +209,9 @@ class BBCooker:
 
         self.initConfigurationData()
 
+        bb.debug(1, "BBCooker parsed base configuration %s" % time.time())
+        sys.stdout.flush()
+
         # we log all events to a file if so directed
         if self.configuration.writeeventlog:
             # register the log file writer as UI Handler
@@ -233,6 +249,9 @@ class BBCooker:
         # Let SIGHUP exit as SIGTERM
         signal.signal(signal.SIGHUP, self.sigterm_exception)
 
+        bb.debug(1, "BBCooker startup complete %s" % time.time())
+        sys.stdout.flush()
+
     def process_inotify_updates(self):
         for n in [self.confignotifier, self.notifier]:
             if n.check_events(timeout=0):
@@ -609,14 +628,7 @@ class BBCooker:
                 k2 = k.split(":do_")
                 k = k2[0]
                 ktask = k2[1]
-            if mc:
-                # Provider might be from another mc
-                for mcavailable in self.multiconfigs:
-                    # The first element is empty
-                    if mcavailable:
-                        taskdata[mcavailable].add_provider(localdata[mcavailable], self.recipecaches[mcavailable], k)
-            else:
-                taskdata[mc].add_provider(localdata[mc], self.recipecaches[mc], k)
+            taskdata[mc].add_provider(localdata[mc], self.recipecaches[mc], k)
             current += 1
             if not ktask.startswith("do_"):
                 ktask = "do_%s" % ktask

bitbake/lib/bb/data.py

@@ -322,8 +322,6 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
             if varflags.get("python"):
                 value = d.getVarFlag(key, "_content", False)
                 parser = bb.codeparser.PythonParser(key, logger)
-                if value and "\t" in value:
-                    logger.warning("Variable %s contains tabs, please remove these (%s)" % (key, d.getVar("FILE")))
                 parser.parse_python(value, filename=varflags.get("filename"), lineno=varflags.get("lineno"))
                 deps = deps | parser.references
                 deps = deps | (keys & parser.execs)

bitbake/lib/bb/data_smart.py

@@ -39,10 +39,11 @@ from bb.COW  import COWDictBase
 logger = logging.getLogger("BitBake.Data")
 
 __setvar_keyword__ = ["_append", "_prepend", "_remove"]
-__setvar_regexp__ = re.compile('(?P<base>.*?)(?P<keyword>_append|_prepend|_remove)(_(?P<add>[^A-Z]*))?$')
+__setvar_regexp__ = re.compile(r'(?P<base>.*?)(?P<keyword>_append|_prepend|_remove)(_(?P<add>[^A-Z]*))?$')
 __expand_var_regexp__ = re.compile(r"\${[^{}@\n\t :]+}")
 __expand_python_regexp__ = re.compile(r"\${@.+?}")
-__whitespace_split__ = re.compile('(\s)')
+__whitespace_split__ = re.compile(r'(\s)')
+__override_regexp__ = re.compile(r'[a-z0-9]+')
 
 def infer_caller_details(loginfo, parent = False, varval = True):
     """Save the caller the trouble of specifying everything."""
@@ -122,7 +123,11 @@ class VariableParse:
                 connector = self.d["_remote_data"]
                 return connector.expandPythonRef(self.varname, code, self.d)
 
-            codeobj = compile(code.strip(), self.varname or "<expansion>", "eval")
+            if self.varname:
+                varname = 'Var <%s>' % self.varname
+            else:
+                varname = '<expansion>'
+            codeobj = compile(code.strip(), varname, "eval")
 
             parser = bb.codeparser.PythonParser(self.varname, logger)
             parser.parse_python(code)
@@ -427,7 +432,8 @@ class DataSmart(MutableMapping):
             except bb.parse.SkipRecipe:
                 raise
             except Exception as exc:
-                raise ExpansionError(varname, s, exc) from exc
+                tb = sys.exc_info()[2]
+                raise ExpansionError(varname, s, exc).with_traceback(tb) from exc
 
         varparse.value = s
 
@@ -592,7 +598,7 @@ class DataSmart(MutableMapping):
         # aka pay the cookie monster
         override = var[var.rfind('_')+1:]
         shortvar = var[:var.rfind('_')]
-        while override and override.islower():
+        while override and __override_regexp__.match(override):
             if shortvar not in self.overridedata:
                 self.overridedata[shortvar] = []
             if [var, override] not in self.overridedata[shortvar]:

bitbake/lib/bb/fetch2/__init__.py

@@ -256,7 +256,7 @@ class URI(object):
 
         # Identify if the URI is relative or not
         if urlp.scheme in self._relative_schemes and \
-           re.compile("^\w+:(?!//)").match(uri):
+           re.compile(r"^\w+:(?!//)").match(uri):
             self.relative = True
 
         if not self.relative:
@@ -827,6 +827,7 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None):
                   'NO_PROXY', 'no_proxy',
                   'ALL_PROXY', 'all_proxy',
                   'GIT_PROXY_COMMAND',
+                  'GIT_SSH',
                   'GIT_SSL_CAINFO',
                   'GIT_SMART_HTTP',
                   'SSH_AUTH_SOCK', 'SSH_AGENT_PID',
@@ -1026,7 +1027,7 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
         raise
 
     except IOError as e:
-        if e.errno in [os.errno.ESTALE]:
+        if e.errno in [errno.ESTALE]:
             logger.warning("Stale Error Observed %s." % ud.url)
             return False
         raise
@@ -1468,7 +1469,7 @@ class FetchMethod(object):
                 else:
                     cmd = 'rpm2cpio.sh %s | cpio -id' % (file)
             elif file.endswith('.deb') or file.endswith('.ipk'):
-                output = subprocess.check_output('ar -t %s' % file, preexec_fn=subprocess_setup, shell=True)
+                output = subprocess.check_output(['ar', '-t', file], preexec_fn=subprocess_setup)
                 datafile = None
                 if output:
                     for line in output.decode().splitlines():
@@ -1715,7 +1716,7 @@ class Fetch(object):
                 update_stamp(ud, self.d)
 
             except IOError as e:
-                if e.errno in [os.errno.ESTALE]:
+                if e.errno in [errno.ESTALE]:
                     logger.error("Stale Error Observed %s." % u)
                     raise ChecksumError("Stale Error Detected")
 

bitbake/lib/bb/fetch2/git.py

@@ -488,12 +488,15 @@ class Git(FetchMethod):
                 source_error.append("clone directory not available or not up to date: " + ud.clonedir)
 
         if not source_found:
-            if ud.shallow and os.path.exists(ud.fullshallow):
-                bb.utils.mkdirhier(destdir)
-                runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=destdir)
-                source_found = True
+            if ud.shallow:
+                if os.path.exists(ud.fullshallow):
+                    bb.utils.mkdirhier(destdir)
+                    runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=destdir)
+                    source_found = True
+                else:
+                    source_error.append("shallow clone not available: " + ud.fullshallow)
             else:
-                source_error.append("shallow clone not enabled or not available: " + ud.fullshallow)
+                source_error.append("shallow clone not enabled")
 
         if not source_found:
             raise bb.fetch2.UnpackError("No up to date source found: " + "; ".join(source_error), ud.url)

bitbake/lib/bb/fetch2/gitsm.py

@@ -92,7 +92,7 @@ class GitSM(Git):
             url = uris[module].replace('%s:' % proto, 'gitsm:', 1)
             url += ';protocol=%s' % proto
             url += ";name=%s" % module
-            url += ";bareclone=1;nocheckout=1"
+            url += ";bareclone=1;nocheckout=1;nobranch=1"
 
             ld = d.createCopy()
             # Not necessary to set SRC_URI, since we're passing the URI to
@@ -152,9 +152,9 @@ class GitSM(Git):
         if submodules and not os.path.exists(os.path.join(repo_conf, 'modules')):
             os.mkdir(os.path.join(repo_conf, 'modules'))
 
-        for module in submodules:
-            srcpath = os.path.join(ud.clonedir, 'modules', module)
-            modpath = os.path.join(repo_conf, 'modules', module)
+        for module, md in submodules.items():
+            srcpath = os.path.join(ud.clonedir, 'modules', md['path'])
+            modpath = os.path.join(repo_conf, 'modules', md['path'])
 
             if os.path.exists(srcpath):
                 if os.path.exists(os.path.join(srcpath, '.git')):
@@ -187,9 +187,8 @@ class GitSM(Git):
                 # No submodules to update
                 continue
 
-            submodules = list(self.parse_gitmodules(gitmodules).keys())
-
-        self.copy_submodules(submodules, ud, dest, d)
+            submodules = self.parse_gitmodules(gitmodules)
+            self.copy_submodules(submodules, ud, dest, d)
 
     def unpack(self, ud, destdir, d):
         Git.unpack(self, ud, destdir, d)
@@ -200,7 +199,7 @@ class GitSM(Git):
         else:
             repo_conf = os.path.join(ud.destdir, '.git')
 
-        submodules = []
+        update_submodules = False
         paths = {}
         uris = {}
         local_paths = {}
@@ -211,41 +210,41 @@ class GitSM(Git):
                 # No submodules to update
                 continue
 
-            for m, md in self.parse_gitmodules(gitmodules).items():
-                submodules.append(m)
-                paths[m] = md['path']
-                uris[m] = md['url']
+            submodules = self.parse_gitmodules(gitmodules)
+            self.copy_submodules(submodules, ud, ud.destdir, d)
 
-        self.copy_submodules(submodules, ud, ud.destdir, d)
+            submodules_queue = [(module, os.path.join(repo_conf, 'modules', md['path'])) for module, md in submodules.items()]
+            while len(submodules_queue) != 0:
+                module, modpath = submodules_queue.pop()
 
-        submodules_queue = [(module, os.path.join(repo_conf, 'modules', module)) for module in submodules]
-        while len(submodules_queue) != 0:
-            module, modpath = submodules_queue.pop()
+                # add submodule children recursively
+                try:
+                    gitmodules = runfetchcmd("%s show HEAD:.gitmodules" % (ud.basecmd), d, quiet=True, workdir=modpath)
+                    for m, md in self.parse_gitmodules(gitmodules).items():
+                        submodules_queue.append([m, os.path.join(modpath, 'modules', md['path'])])
+                except:
+                    # no children
+                    pass
 
-            # add submodule children recursively
-            try:
-                gitmodules = runfetchcmd("%s show HEAD:.gitmodules" % (ud.basecmd), d, quiet=True, workdir=modpath)
-                for m, md in self.parse_gitmodules(gitmodules).items():
-                    submodules_queue.append([m, os.path.join(modpath, 'modules', m)])
-            except:
-                # no children
-                pass
 
-            # Determine (from the submodule) the correct url to reference
-            try:
-                output = runfetchcmd("%(basecmd)s config remote.origin.url" % {'basecmd': ud.basecmd}, d, workdir=modpath)
-            except bb.fetch2.FetchError as e:
-                # No remote url defined in this submodule
-                continue
+                # There are submodules to update
+                update_submodules = True
 
-            local_paths[module] = output
+                # Determine (from the submodule) the correct url to reference
+                try:
+                    output = runfetchcmd("%(basecmd)s config remote.origin.url" % {'basecmd': ud.basecmd}, d, workdir=modpath)
+                except bb.fetch2.FetchError as e:
+                    # No remote url defined in this submodule
+                    continue
 
-            # Setup the local URL properly (like git submodule init or sync would do...)
-            runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_paths[module]}, d, workdir=ud.destdir)
+                local_paths[module] = output
 
-            # Ensure the submodule repository is NOT set to bare, since we're checking it out...
-            runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=modpath)
+                # Setup the local URL properly (like git submodule init or sync would do...)
+                runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_paths[module]}, d, workdir=ud.destdir)
 
-        if submodules:
+                # Ensure the submodule repository is NOT set to bare, since we're checking it out...
+                runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=modpath)
+
+        if update_submodules:
             # Run submodule update, this sets up the directories -- without touching the config
             runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)

bitbake/lib/bb/fetch2/npm.py

@@ -226,7 +226,7 @@ class Npm(FetchMethod):
                         self._getshrinkeddependencies(obj, data['dependencies'][obj], data['dependencies'][obj]['version'], d, ud, lockdown, manifest, False)
                         return
         outputurl = "invalid"
-        if ('resolved' not in data) or (not data['resolved'].startswith('http')):
+        if ('resolved' not in data) or (not data['resolved'].startswith('http://') and not data['resolved'].startswith('https://')):
             # will be the case for ${PN}
             fetchcmd = "npm view %s@%s dist.tarball --registry %s" % (pkg, version, ud.registry)
             logger.debug(2, "Found this matching URL: %s" % str(fetchcmd))

bitbake/lib/bb/fetch2/svn.py

@@ -63,6 +63,9 @@ class Svn(FetchMethod):
         relpath = self._strip_leading_slashes(ud.path)
         ud.pkgdir = os.path.join(svndir, ud.host, relpath)
         ud.moddir = os.path.join(ud.pkgdir, ud.module)
+        # Protects the repository from concurrent updates, e.g. from two
+        # recipes fetching different revisions at the same time
+        ud.svnlock = os.path.join(ud.pkgdir, "svn.lock")
 
         ud.setup_revisions(d)
 
@@ -123,35 +126,40 @@ class Svn(FetchMethod):
 
         logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'")
 
-        if os.access(os.path.join(ud.moddir, '.svn'), os.R_OK):
-            svnupdatecmd = self._buildsvncommand(ud, d, "update")
-            logger.info("Update " + ud.url)
-            # We need to attempt to run svn upgrade first in case its an older working format
-            try:
-                runfetchcmd(ud.basecmd + " upgrade", d, workdir=ud.moddir)
-            except FetchError:
-                pass
-            logger.debug(1, "Running %s", svnupdatecmd)
-            bb.fetch2.check_network_access(d, svnupdatecmd, ud.url)
-            runfetchcmd(svnupdatecmd, d, workdir=ud.moddir)
-        else:
-            svnfetchcmd = self._buildsvncommand(ud, d, "fetch")
-            logger.info("Fetch " + ud.url)
-            # check out sources there
-            bb.utils.mkdirhier(ud.pkgdir)
-            logger.debug(1, "Running %s", svnfetchcmd)
-            bb.fetch2.check_network_access(d, svnfetchcmd, ud.url)
-            runfetchcmd(svnfetchcmd, d, workdir=ud.pkgdir)
+        lf = bb.utils.lockfile(ud.svnlock)
 
-        scmdata = ud.parm.get("scmdata", "")
-        if scmdata == "keep":
-            tar_flags = ""
-        else:
-            tar_flags = "--exclude='.svn'"
+        try:
+            if os.access(os.path.join(ud.moddir, '.svn'), os.R_OK):
+                svnupdatecmd = self._buildsvncommand(ud, d, "update")
+                logger.info("Update " + ud.url)
+                # We need to attempt to run svn upgrade first in case its an older working format
+                try:
+                    runfetchcmd(ud.basecmd + " upgrade", d, workdir=ud.moddir)
+                except FetchError:
+                    pass
+                logger.debug(1, "Running %s", svnupdatecmd)
+                bb.fetch2.check_network_access(d, svnupdatecmd, ud.url)
+                runfetchcmd(svnupdatecmd, d, workdir=ud.moddir)
+            else:
+                svnfetchcmd = self._buildsvncommand(ud, d, "fetch")
+                logger.info("Fetch " + ud.url)
+                # check out sources there
+                bb.utils.mkdirhier(ud.pkgdir)
+                logger.debug(1, "Running %s", svnfetchcmd)
+                bb.fetch2.check_network_access(d, svnfetchcmd, ud.url)
+                runfetchcmd(svnfetchcmd, d, workdir=ud.pkgdir)
 
-        # tar them up to a defined filename
-        runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, ud.path_spec), d,
-                    cleanup=[ud.localpath], workdir=ud.pkgdir)
+            scmdata = ud.parm.get("scmdata", "")
+            if scmdata == "keep":
+                tar_flags = ""
+            else:
+                tar_flags = "--exclude='.svn'"
+
+            # tar them up to a defined filename
+            runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, ud.path_spec), d,
+                        cleanup=[ud.localpath], workdir=ud.pkgdir)
+        finally:
+            bb.utils.unlockfile(lf)
 
     def clean(self, ud, d):
         """ Clean SVN specific files and dirs """

bitbake/lib/bb/fetch2/wget.py

@@ -168,7 +168,7 @@ class Wget(FetchMethod):
                 """
                 host = req.host
                 if not host:
-                    raise urlllib2.URLError('no host given')
+                    raise urllib.error.URLError('no host given')
 
                 h = http_class(host, timeout=req.timeout) # will parse host:port
                 h.set_debuglevel(self._debuglevel)

bitbake/lib/bb/main.py

@@ -448,7 +448,7 @@ def setup_bitbake(configParams, configuration, extrafeatures=None):
                 else:
                     logger.info("Reconnecting to bitbake server...")
                     if not os.path.exists(sockname):
-                        print("Previous bitbake instance shutting down?, waiting to retry...")
+                        logger.info("Previous bitbake instance shutting down?, waiting to retry...")
                         i = 0
                         lock = None
                         # Wait for 5s or until we can get the lock

bitbake/lib/bb/parse/ast.py

@@ -178,7 +178,7 @@ class MethodNode(AstNode):
             funcname = ("__anon_%s_%s" % (self.lineno, self.filename.translate(MethodNode.tr_tbl)))
             self.python = True
             text = "def %s(d):\n" % (funcname) + text
-            bb.methodpool.insert_method(funcname, text, self.filename, self.lineno - len(self.body))
+            bb.methodpool.insert_method(funcname, text, self.filename, self.lineno - len(self.body) - 1)
             anonfuncs = data.getVar('__BBANONFUNCS', False) or []
             anonfuncs.append(funcname)
             data.setVar('__BBANONFUNCS', anonfuncs)

bitbake/lib/bb/parse/parse_py/BBHandler.py

@@ -45,7 +45,8 @@ __addtask_regexp__       = re.compile("addtask\s+(?P<func>\w+)\s*((before\s*(?P<
 __deltask_regexp__       = re.compile("deltask\s+(?P<func>\w+)")
 __addhandler_regexp__    = re.compile( r"addhandler\s+(.+)" )
 __def_regexp__           = re.compile( r"def\s+(\w+).*:" )
-__python_func_regexp__   = re.compile( r"(\s+.*)|(^$)" )
+__python_func_regexp__   = re.compile( r"(\s+.*)|(^$)|(^#)" )
+__python_tab_regexp__    = re.compile(" *\t")
 
 __infunc__ = []
 __inpython__ = False
@@ -160,6 +161,16 @@ def handle(fn, d, include):
 
 def feeder(lineno, s, fn, root, statements, eof=False):
     global __func_start_regexp__, __inherit_regexp__, __export_func_regexp__, __addtask_regexp__, __addhandler_regexp__, __def_regexp__, __python_func_regexp__, __inpython__, __infunc__, __body__, bb, __residue__, __classname__
+
+    # Check tabs in python functions:
+    # - def py_funcname(): covered by __inpython__
+    # - python(): covered by '__anonymous' == __infunc__[0]
+    # - python funcname(): covered by __infunc__[3]
+    if __inpython__ or (__infunc__ and ('__anonymous' == __infunc__[0] or __infunc__[3])):
+        tab = __python_tab_regexp__.match(s)
+        if tab:
+            bb.warn('python should use 4 spaces indentation, but found tabs in %s, line %s' % (root, lineno))
+
     if __infunc__:
         if s == '}':
             __body__.append('')

bitbake/lib/bb/persist_data.py

@@ -29,6 +29,7 @@ import warnings
 from bb.compat import total_ordering
 from collections import Mapping
 import sqlite3
+import contextlib
 
 sqlversion = sqlite3.sqlite_version_info
 if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
@@ -36,84 +37,181 @@ if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
 
 
 logger = logging.getLogger("BitBake.PersistData")
-if hasattr(sqlite3, 'enable_shared_cache'):
-    try:
-        sqlite3.enable_shared_cache(True)
-    except sqlite3.OperationalError:
-        pass
-
 
 @total_ordering
 class SQLTable(collections.MutableMapping):
+    class _Decorators(object):
+        @staticmethod
+        def retry(*, reconnect=True):
+            """
+            Decorator that restarts a function if a database locked sqlite
+            exception occurs. If reconnect is True, the database connection
+            will be closed and reopened each time a failure occurs
+            """
+            def retry_wrapper(f):
+                def wrap_func(self, *args, **kwargs):
+                    # Reconnect if necessary
+                    if self.connection is None and reconnect:
+                        self.reconnect()
+
+                    count = 0
+                    while True:
+                        try:
+                            return f(self, *args, **kwargs)
+                        except sqlite3.OperationalError as exc:
+                            if 'is locked' in str(exc) and count < 500:
+                                count = count + 1
+                                if reconnect:
+                                    self.reconnect()
+                                continue
+                            raise
+                return wrap_func
+            return retry_wrapper
+
+        @staticmethod
+        def transaction(f):
+            """
+            Decorator that starts a database transaction and creates a database
+            cursor for performing queries. If no exception is thrown, the
+            database results are commited. If an exception occurs, the database
+            is rolled back. In all cases, the cursor is closed after the
+            function ends.
+
+            Note that the cursor is passed as an extra argument to the function
+            after `self` and before any of the normal arguments
+            """
+            def wrap_func(self, *args, **kwargs):
+                # Context manager will COMMIT the database on success,
+                # or ROLLBACK on an exception
+                with self.connection:
+                    # Automatically close the cursor when done
+                    with contextlib.closing(self.connection.cursor()) as cursor:
+                        return f(self, cursor, *args, **kwargs)
+            return wrap_func
+
     """Object representing a table/domain in the database"""
     def __init__(self, cachefile, table):
         self.cachefile = cachefile
         self.table = table
-        self.cursor = connect(self.cachefile)
 
-        self._execute("CREATE TABLE IF NOT EXISTS %s(key TEXT, value TEXT);"
-                      % table)
+        self.connection = None
+        self._execute_single("CREATE TABLE IF NOT EXISTS %s(key TEXT PRIMARY KEY NOT NULL, value TEXT);" % table)
 
-    def _execute(self, *query):
-        """Execute a query, waiting to acquire a lock if necessary"""
-        count = 0
-        while True:
-            try:
-                return self.cursor.execute(*query)
-            except sqlite3.OperationalError as exc:
-                if 'database is locked' in str(exc) and count < 500:
-                    count = count + 1
+    @_Decorators.retry(reconnect=False)
+    @_Decorators.transaction
+    def _setup_database(self, cursor):
+        cursor.execute("pragma synchronous = off;")
+        # Enable WAL and keep the autocheckpoint length small (the default is
+        # usually 1000). Persistent caches are usually read-mostly, so keeping
+        # this short will keep readers running quickly
+        cursor.execute("pragma journal_mode = WAL;")
+        cursor.execute("pragma wal_autocheckpoint = 100;")
+
+    def reconnect(self):
+        if self.connection is not None:
+            self.connection.close()
+        self.connection = sqlite3.connect(self.cachefile, timeout=5)
+        self.connection.text_factory = str
+        self._setup_database()
+
+    @_Decorators.retry()
+    @_Decorators.transaction
+    def _execute_single(self, cursor, *query):
+        """
+        Executes a single query and discards the results. This correctly closes
+        the database cursor when finished
+        """
+        cursor.execute(*query)
+
+    @_Decorators.retry()
+    def _row_iter(self, f, *query):
+        """
+        Helper function that returns a row iterator. Each time __next__ is
+        called on the iterator, the provided function is evaluated to determine
+        the return value
+        """
+        class CursorIter(object):
+            def __init__(self, cursor):
+                self.cursor = cursor
+
+            def __iter__(self):
+                return self
+
+            def __next__(self):
+                row = self.cursor.fetchone()
+                if row is None:
                     self.cursor.close()
-                    self.cursor = connect(self.cachefile)
-                    continue
-                raise
+                    raise StopIteration
+                return f(row)
+
+            def __enter__(self):
+                return self
+
+            def __exit__(self, typ, value, traceback):
+                self.cursor.close()
+                return False
+
+        cursor = self.connection.cursor()
+        try:
+            cursor.execute(*query)
+            return CursorIter(cursor)
+        except:
+            cursor.close()
 
     def __enter__(self):
-        self.cursor.__enter__()
+        self.connection.__enter__()
         return self
 
     def __exit__(self, *excinfo):
-        self.cursor.__exit__(*excinfo)
+        self.connection.__exit__(*excinfo)
 
-    def __getitem__(self, key):
-        data = self._execute("SELECT * from %s where key=?;" %
-                             self.table, [key])
-        for row in data:
+    @_Decorators.retry()
+    @_Decorators.transaction
+    def __getitem__(self, cursor, key):
+        cursor.execute("SELECT * from %s where key=?;" % self.table, [key])
+        row = cursor.fetchone()
+        if row is not None:
             return row[1]
         raise KeyError(key)
 
-    def __delitem__(self, key):
+    @_Decorators.retry()
+    @_Decorators.transaction
+    def __delitem__(self, cursor, key):
         if key not in self:
             raise KeyError(key)
-        self._execute("DELETE from %s where key=?;" % self.table, [key])
+        cursor.execute("DELETE from %s where key=?;" % self.table, [key])
 
-    def __setitem__(self, key, value):
+    @_Decorators.retry()
+    @_Decorators.transaction
+    def __setitem__(self, cursor, key, value):
         if not isinstance(key, str):
             raise TypeError('Only string keys are supported')
         elif not isinstance(value, str):
             raise TypeError('Only string values are supported')
 
-        data = self._execute("SELECT * from %s where key=?;" %
-                                   self.table, [key])
-        exists = len(list(data))
-        if exists:
-            self._execute("UPDATE %s SET value=? WHERE key=?;" % self.table,
-                          [value, key])
+        cursor.execute("SELECT * from %s where key=?;" % self.table, [key])
+        row = cursor.fetchone()
+        if row is not None:
+            cursor.execute("UPDATE %s SET value=? WHERE key=?;" % self.table, [value, key])
         else:
-            self._execute("INSERT into %s(key, value) values (?, ?);" %
-                          self.table, [key, value])
+            cursor.execute("INSERT into %s(key, value) values (?, ?);" % self.table, [key, value])
 
-    def __contains__(self, key):
-        return key in set(self)
+    @_Decorators.retry()
+    @_Decorators.transaction
+    def __contains__(self, cursor, key):
+        cursor.execute('SELECT * from %s where key=?;' % self.table, [key])
+        return cursor.fetchone() is not None
 
-    def __len__(self):
-        data = self._execute("SELECT COUNT(key) FROM %s;" % self.table)
-        for row in data:
+    @_Decorators.retry()
+    @_Decorators.transaction
+    def __len__(self, cursor):
+        cursor.execute("SELECT COUNT(key) FROM %s;" % self.table)
+        row = cursor.fetchone()
+        if row is not None:
             return row[0]
 
     def __iter__(self):
-        data = self._execute("SELECT key FROM %s;" % self.table)
-        return (row[0] for row in data)
+        return self._row_iter(lambda row: row[0], "SELECT key from %s;" % self.table)
 
     def __lt__(self, other):
         if not isinstance(other, Mapping):
@@ -122,25 +220,27 @@ class SQLTable(collections.MutableMapping):
         return len(self) < len(other)
 
     def get_by_pattern(self, pattern):
-        data = self._execute("SELECT * FROM %s WHERE key LIKE ?;" %
-                             self.table, [pattern])
-        return [row[1] for row in data]
+        return self._row_iter(lambda row: row[1], "SELECT * FROM %s WHERE key LIKE ?;" %
+                              self.table, [pattern])
 
     def values(self):
         return list(self.itervalues())
 
     def itervalues(self):
-        data = self._execute("SELECT value FROM %s;" % self.table)
-        return (row[0] for row in data)
+        return self._row_iter(lambda row: row[0], "SELECT value FROM %s;" %
+                              self.table)
 
     def items(self):
         return list(self.iteritems())
 
     def iteritems(self):
-        return self._execute("SELECT * FROM %s;" % self.table)
+        return self._row_iter(lambda row: (row[0], row[1]), "SELECT * FROM %s;" %
+                              self.table)
 
-    def clear(self):
-        self._execute("DELETE FROM %s;" % self.table)
+    @_Decorators.retry()
+    @_Decorators.transaction
+    def clear(self, cursor):
+        cursor.execute("DELETE FROM %s;" % self.table)
 
     def has_key(self, key):
         return key in self
@@ -194,12 +294,6 @@ class PersistData(object):
         """
         del self.data[domain][key]
 
-def connect(database):
-    connection = sqlite3.connect(database, timeout=5, isolation_level=None)
-    connection.execute("pragma synchronous = off;")
-    connection.text_factory = str
-    return connection
-
 def persist(domain, d):
     """Convenience factory for SQLTable objects based upon metadata"""
     import bb.utils

bitbake/lib/bb/pysh/builtin.py

@@ -1,710 +0,0 @@
-# builtin.py - builtins and utilities definitions for pysh.
-#
-# Copyright 2007 Patrick Mezard
-#
-# This software may be used and distributed according to the terms
-# of the GNU General Public License, incorporated herein by reference.
-
-"""Builtin and internal utilities implementations.
-
-- Beware not to use python interpreter environment as if it were the shell
-environment. For instance, commands working directory must be explicitely handled
-through env['PWD'] instead of relying on python working directory.
-"""
-import errno
-import optparse
-import os
-import re
-import subprocess
-import sys
-import time
-
-def has_subprocess_bug():
-    return getattr(subprocess, 'list2cmdline') and \
-       (    subprocess.list2cmdline(['']) == '' or \
-            subprocess.list2cmdline(['foo|bar']) == 'foo|bar')
-            
-# Detect python bug 1634343: "subprocess swallows empty arguments under win32"
-# <http://sourceforge.net/tracker/index.php?func=detail&aid=1634343&group_id=5470&atid=105470>
-# Also detect: "[ 1710802 ] subprocess must escape redirection characters under win32"
-# <http://sourceforge.net/tracker/index.php?func=detail&aid=1710802&group_id=5470&atid=105470>
-if has_subprocess_bug():
-    import subprocess_fix
-    subprocess.list2cmdline = subprocess_fix.list2cmdline
-
-from sherrors import *
-
-class NonExitingParser(optparse.OptionParser):
-    """OptionParser default behaviour upon error is to print the error message and
-    exit. Raise a utility error instead.
-    """
-    def error(self, msg):
-        raise UtilityError(msg)
-
-#-------------------------------------------------------------------------------  
-# set special builtin
-#-------------------------------------------------------------------------------  
-OPT_SET = NonExitingParser(usage="set - set or unset options and positional parameters")
-OPT_SET.add_option( '-f', action='store_true', dest='has_f', default=False,
-    help='The shell shall disable pathname expansion.')
-OPT_SET.add_option('-e', action='store_true', dest='has_e', default=False,
-    help="""When this option is on, if a simple command fails for any of the \
-    reasons listed in Consequences of Shell Errors or returns an exit status \
-    value >0, and is not part of the compound list following a while, until, \
-    or if keyword, and is not a part of an AND or OR list, and is not a \
-    pipeline preceded by the ! reserved word, then the shell shall immediately \
-    exit.""")
-OPT_SET.add_option('-x', action='store_true', dest='has_x', default=False,
-    help="""The shell shall write to standard error a trace for each command \
-    after it expands the command and before it executes it. It is unspecified \
-    whether the command that turns tracing off is traced.""")
-
-def builtin_set(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-
-    option, args = OPT_SET.parse_args(args)
-    env = interp.get_env()
-    
-    if option.has_f:
-        env.set_opt('-f')
-    if option.has_e:
-        env.set_opt('-e')
-    if option.has_x:
-        env.set_opt('-x')
-    return 0
-    
-#-------------------------------------------------------------------------------  
-# shift special builtin
-#-------------------------------------------------------------------------------  
-def builtin_shift(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-        
-    params = interp.get_env().get_positional_args()
-    if args:
-        try:
-            n = int(args[0])
-            if n > len(params):
-                raise ValueError()
-        except ValueError:
-            return 1
-    else:
-        n = 1
-        
-    params[:n] = []
-    interp.get_env().set_positional_args(params)
-    return 0
-    
-#-------------------------------------------------------------------------------  
-# export special builtin
-#-------------------------------------------------------------------------------  
-OPT_EXPORT = NonExitingParser(usage="set - set or unset options and positional parameters")
-OPT_EXPORT.add_option('-p', action='store_true', dest='has_p', default=False)
-
-def builtin_export(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-        
-    option, args = OPT_EXPORT.parse_args(args)
-    if option.has_p:
-        raise NotImplementedError()
-    
-    for arg in args:
-        try:
-            name, value = arg.split('=', 1)
-        except ValueError:
-            name, value = arg, None
-        env = interp.get_env().export(name, value)
-    
-    return 0
-    
-#-------------------------------------------------------------------------------  
-# return special builtin
-#-------------------------------------------------------------------------------  
-def builtin_return(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-    res = 0
-    if args:
-        try:
-            res = int(args[0])
-        except ValueError:
-            res = 0
-        if not 0<=res<=255:
-            res = 0
-            
-    # BUG: should be last executed command exit code        
-    raise ReturnSignal(res)
-
-#-------------------------------------------------------------------------------  
-# trap special builtin
-#-------------------------------------------------------------------------------  
-def builtin_trap(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-    if len(args) < 2:
-        stderr.write('trap: usage: trap [[arg] signal_spec ...]\n')
-        return 2
-
-    action = args[0]
-    for sig in args[1:]:
-        try:
-            env.traps[sig] = action
-        except Exception as e:
-            stderr.write('trap: %s\n' % str(e))
-    return 0
-
-#-------------------------------------------------------------------------------  
-# unset special builtin
-#-------------------------------------------------------------------------------
-OPT_UNSET = NonExitingParser("unset - unset values and attributes of variables and functions")
-OPT_UNSET.add_option( '-f', action='store_true', dest='has_f', default=False)
-OPT_UNSET.add_option( '-v', action='store_true', dest='has_v', default=False)
-
-def builtin_unset(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-        
-    option, args = OPT_UNSET.parse_args(args)
-    
-    status = 0
-    env = interp.get_env()
-    for arg in args:    
-        try:
-            if option.has_f:
-                env.remove_function(arg)
-            else:
-                del env[arg]
-        except KeyError:
-            pass
-        except VarAssignmentError:
-            status = 1
-            
-    return status
-
-#-------------------------------------------------------------------------------  
-# wait special builtin
-#-------------------------------------------------------------------------------  
-def builtin_wait(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-
-    return interp.wait([int(arg) for arg in args])
-
-#-------------------------------------------------------------------------------  
-# cat utility
-#-------------------------------------------------------------------------------
-def utility_cat(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-
-    if not args:
-        args = ['-']
-
-    status = 0
-    for arg in args:
-        if arg == '-':
-            data = stdin.read()
-        else:
-            path = os.path.join(env['PWD'], arg)
-            try:
-                f = file(path, 'rb')
-                try:
-                    data = f.read()
-                finally:
-                    f.close()
-            except IOError as e:
-                if e.errno != errno.ENOENT:
-                    raise
-                status = 1
-                continue
-        stdout.write(data)
-        stdout.flush()
-    return status
-    
-#-------------------------------------------------------------------------------  
-# cd utility
-#-------------------------------------------------------------------------------  
-OPT_CD = NonExitingParser("cd - change the working directory")
-
-def utility_cd(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-
-    option, args = OPT_CD.parse_args(args)
-    env = interp.get_env()
-    
-    directory = None
-    printdir = False
-    if not args:
-        home = env.get('HOME')
-        if home:
-            # Unspecified, do nothing
-            return 0
-        else:
-            directory = home
-    elif len(args)==1:
-        directory = args[0]
-        if directory=='-':
-            if 'OLDPWD' not in env:
-                raise UtilityError("OLDPWD not set")
-            printdir = True
-            directory = env['OLDPWD']
-    else:
-        raise UtilityError("too many arguments")
-            
-    curpath = None
-    # Absolute directories will be handled correctly by the os.path.join call.
-    if not directory.startswith('.') and not directory.startswith('..'):
-        cdpaths = env.get('CDPATH', '.').split(';')
-        for cdpath in cdpaths:
-            p = os.path.join(cdpath, directory)
-            if os.path.isdir(p):
-                curpath = p
-                break
-    
-    if curpath is None:
-        curpath = directory
-    curpath = os.path.join(env['PWD'], directory)
-
-    env['OLDPWD'] = env['PWD']
-    env['PWD'] = curpath
-    if printdir:
-        stdout.write('%s\n' % curpath)
-    return 0
-
-#-------------------------------------------------------------------------------  
-# colon utility
-#-------------------------------------------------------------------------------  
-def utility_colon(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-    return 0
-    
-#-------------------------------------------------------------------------------  
-# echo utility
-#-------------------------------------------------------------------------------  
-def utility_echo(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-        
-    # Echo only takes arguments, no options. Use printf if you need fancy stuff.
-    output = ' '.join(args) + '\n'
-    stdout.write(output)
-    stdout.flush()
-    return 0
-    
-#-------------------------------------------------------------------------------  
-# egrep utility
-#-------------------------------------------------------------------------------
-# egrep is usually a shell script.
-# Unfortunately, pysh does not support shell scripts *with arguments* right now,
-# so the redirection is implemented here, assuming grep is available.
-def utility_egrep(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-        
-    return run_command('grep', ['-E'] + args, interp, env, stdin, stdout, 
-        stderr, debugflags)
-    
-#-------------------------------------------------------------------------------  
-# env utility
-#-------------------------------------------------------------------------------  
-def utility_env(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-    
-    if args and args[0]=='-i':
-        raise NotImplementedError('env: -i option is not implemented')
-    
-    i = 0
-    for arg in args:
-        if '=' not in arg:
-            break
-        # Update the current environment
-        name, value = arg.split('=', 1)
-        env[name] = value
-        i += 1
-        
-    if args[i:]:
-        # Find then execute the specified interpreter
-        utility = env.find_in_path(args[i])
-        if not utility:
-            return 127
-        args[i:i+1] = utility
-        name = args[i]
-        args = args[i+1:]
-        try:
-            return run_command(name, args, interp, env, stdin, stdout, stderr, 
-                debugflags)
-        except UtilityError:
-            stderr.write('env: failed to execute %s' % ' '.join([name]+args))
-            return 126            
-    else:
-        for pair in env.get_variables().iteritems():
-            stdout.write('%s=%s\n' % pair)
-    return 0
-    
-#-------------------------------------------------------------------------------  
-# exit utility
-#-------------------------------------------------------------------------------
-def utility_exit(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-        
-    res = None
-    if args:
-        try:
-            res = int(args[0])
-        except ValueError:
-            res = None
-        if not 0<=res<=255:
-            res = None
-            
-    if res is None:
-        # BUG: should be last executed command exit code
-        res = 0
-        
-    raise ExitSignal(res)
-
-#-------------------------------------------------------------------------------  
-# fgrep utility
-#-------------------------------------------------------------------------------
-# see egrep
-def utility_fgrep(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-        
-    return run_command('grep', ['-F'] + args, interp, env, stdin, stdout, 
-        stderr, debugflags)
-
-#-------------------------------------------------------------------------------  
-# gunzip utility
-#-------------------------------------------------------------------------------
-# see egrep
-def utility_gunzip(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-        
-    return run_command('gzip', ['-d'] + args, interp, env, stdin, stdout, 
-        stderr, debugflags)
-    
-#-------------------------------------------------------------------------------  
-# kill utility
-#-------------------------------------------------------------------------------
-def utility_kill(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-        
-    for arg in args:
-        pid = int(arg)
-        status = subprocess.call(['pskill', '/T', str(pid)],
-                                shell=True,
-                                stdout=subprocess.PIPE,
-                                stderr=subprocess.PIPE)
-        # pskill is asynchronous, hence the stupid polling loop
-        while 1:
-            p = subprocess.Popen(['pslist', str(pid)],
-                                shell=True,
-                                stdout=subprocess.PIPE,
-                                stderr=subprocess.STDOUT)
-            output = p.communicate()[0]
-            if ('process %d was not' % pid) in output:
-                break
-            time.sleep(1)
-    return status
-    
-#-------------------------------------------------------------------------------  
-# mkdir utility
-#-------------------------------------------------------------------------------
-OPT_MKDIR = NonExitingParser("mkdir - make directories.")
-OPT_MKDIR.add_option('-p', action='store_true', dest='has_p', default=False)
-
-def utility_mkdir(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-        
-    # TODO: implement umask
-    # TODO: implement proper utility error report
-    option, args = OPT_MKDIR.parse_args(args)
-    for arg in args:
-        path = os.path.join(env['PWD'], arg)
-        if option.has_p:
-            try:
-                os.makedirs(path)
-            except IOError as e:
-                if e.errno != errno.EEXIST:
-                    raise
-        else:               
-            os.mkdir(path)
-    return 0
-
-#-------------------------------------------------------------------------------  
-# netstat utility
-#-------------------------------------------------------------------------------
-def utility_netstat(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    # Do you really expect me to implement netstat ?
-    # This empty form is enough for Mercurial tests since it's
-    # supposed to generate nothing upon success. Faking this test
-    # is not a big deal either.
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-    return 0
-    
-#-------------------------------------------------------------------------------  
-# pwd utility
-#-------------------------------------------------------------------------------  
-OPT_PWD = NonExitingParser("pwd - return working directory name")
-OPT_PWD.add_option('-L', action='store_true', dest='has_L', default=True,
-    help="""If the PWD environment variable contains an absolute pathname of \
-    the current directory that does not contain the filenames dot or dot-dot, \
-    pwd shall write this pathname to standard output. Otherwise, the -L option \
-    shall behave as the -P option.""")
-OPT_PWD.add_option('-P', action='store_true', dest='has_L', default=False,
-    help="""The absolute pathname written shall not contain filenames that, in \
-    the context of the pathname, refer to files of type symbolic link.""")
-
-def utility_pwd(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-
-    option, args = OPT_PWD.parse_args(args)        
-    stdout.write('%s\n' % env['PWD'])
-    return 0
-    
-#-------------------------------------------------------------------------------  
-# printf utility
-#-------------------------------------------------------------------------------
-RE_UNESCAPE = re.compile(r'(\\x[a-zA-Z0-9]{2}|\\[0-7]{1,3}|\\.)')
-
-def utility_printf(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-        
-    def replace(m):
-        assert m.group()
-        g = m.group()[1:]
-        if g.startswith('x'):
-            return chr(int(g[1:], 16))
-        if len(g) <= 3 and len([c for c in g if c in '01234567']) == len(g):
-            # Yay, an octal number
-            return chr(int(g, 8))
-        return {
-            'a': '\a',
-            'b': '\b',
-            'f': '\f',
-            'n': '\n',
-            'r': '\r',
-            't': '\t',
-            'v': '\v',
-            '\\': '\\',
-        }.get(g)
-        
-    # Convert escape sequences
-    format = re.sub(RE_UNESCAPE, replace, args[0])
-    stdout.write(format % tuple(args[1:]))
-    return 0
-    
-#-------------------------------------------------------------------------------  
-# true utility
-#-------------------------------------------------------------------------------
-def utility_true(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-    return 0
-
-#-------------------------------------------------------------------------------  
-# sed utility
-#-------------------------------------------------------------------------------
-RE_SED = re.compile(r'^s(.).*\1[a-zA-Z]*$')
-
-# cygwin sed fails with some expressions when they do not end with a single space.
-# see unit tests for details. Interestingly, the same expressions works perfectly
-# in cygwin shell.
-def utility_sed(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-        
-    # Scan pattern arguments and append a space if necessary
-    for i in range(len(args)):
-        if not RE_SED.search(args[i]):
-            continue
-        args[i] = args[i] + ' '
-
-    return run_command(name, args, interp, env, stdin, stdout, 
-        stderr, debugflags)
-
-#-------------------------------------------------------------------------------  
-# sleep utility
-#-------------------------------------------------------------------------------
-def utility_sleep(name, args, interp, env, stdin, stdout, stderr, debugflags):
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-    time.sleep(int(args[0]))
-    return 0
-    
-#-------------------------------------------------------------------------------  
-# sort utility
-#-------------------------------------------------------------------------------
-OPT_SORT = NonExitingParser("sort - sort, merge, or sequence check text files")
-
-def utility_sort(name, args, interp, env, stdin, stdout, stderr, debugflags):
-
-    def sort(path):
-        if path == '-':
-            lines = stdin.readlines()
-        else:
-            try:
-                f = file(path)
-                try:
-                    lines = f.readlines()
-                finally:
-                    f.close()
-            except IOError as e:
-                stderr.write(str(e) + '\n')
-                return 1
-        
-        if lines and lines[-1][-1]!='\n':
-            lines[-1] = lines[-1] + '\n'
-        return lines
-    
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-        
-    option, args = OPT_SORT.parse_args(args)
-    alllines = []
-    
-    if len(args)<=0:
-        args += ['-']
-        
-    # Load all files lines
-    curdir = os.getcwd()
-    try:
-        os.chdir(env['PWD'])
-        for path in args:
-            alllines += sort(path)
-    finally:
-        os.chdir(curdir)
-            
-    alllines.sort()
-    for line in alllines:
-        stdout.write(line)
-    return 0
-    
-#-------------------------------------------------------------------------------
-# hg utility
-#-------------------------------------------------------------------------------
-
-hgcommands = [
-    'add',
-    'addremove',
-    'commit', 'ci',
-    'debugrename',
-    'debugwalk',
-    'falabala', # Dummy command used in a mercurial test
-    'incoming',
-    'locate',
-    'pull',
-    'push',
-    'qinit',
-    'remove', 'rm',
-    'rename', 'mv',
-    'revert',    
-    'showconfig',
-    'status', 'st',
-    'strip',
-    ]
-
-def rewriteslashes(name, args):
-    # Several hg commands output file paths, rewrite the separators
-    if len(args) > 1 and name.lower().endswith('python') \
-       and args[0].endswith('hg'):
-        for cmd in hgcommands:
-            if cmd in args[1:]:
-                return True
-            
-    # svn output contains many paths with OS specific separators.
-    # Normalize these to unix paths.
-    base = os.path.basename(name)
-    if base.startswith('svn'):
-        return True
-    
-    return False
-
-def rewritehg(output):
-    if not output:
-        return output
-    # Rewrite os specific messages
-    output = output.replace(': The system cannot find the file specified',
-                            ': No such file or directory')
-    output = re.sub(': Access is denied.*$', ': Permission denied', output)
-    output = output.replace(': No connection could be made because the target machine actively refused it',
-                            ': Connection refused')
-    return output
-                            
-
-def run_command(name, args, interp, env, stdin, stdout,
-                stderr, debugflags):
-    # Execute the command
-    if 'debug-utility' in debugflags:
-        print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
-
-    hgbin = interp.options().hgbinary
-    ishg = hgbin and ('hg' in name or args and 'hg' in args[0])
-    unixoutput = 'cygwin' in name or ishg
-    
-    exec_env = env.get_variables()        
-    try:
-        # BUG: comparing file descriptor is clearly not a reliable way to tell
-        # whether they point on the same underlying object. But in pysh limited
-        # scope this is usually right, we do not expect complicated redirections
-        # besides usual 2>&1.
-        # Still there is one case we have but cannot deal with is when stdout
-        # and stderr are redirected *by pysh caller*. This the reason for the
-        # --redirect pysh() option.
-        # Now, we want to know they are the same because we sometimes need to 
-        # transform the command output, mostly remove CR-LF to ensure that
-        # command output is unix-like. Cygwin utilies are a special case because
-        # they explicitely set their output streams to binary mode, so we have
-        # nothing to do. For all others commands, we have to guess whether they
-        # are sending text data, in which case the transformation must be done.
-        # Again, the NUL character test is unreliable but should be enough for
-        # hg tests.
-        redirected = stdout.fileno()==stderr.fileno()
-        if not redirected:
-            p = subprocess.Popen([name] + args, cwd=env['PWD'], env=exec_env, 
-                    stdin=stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        else:
-            p = subprocess.Popen([name] + args, cwd=env['PWD'], env=exec_env, 
-                    stdin=stdin, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-        out, err = p.communicate()
-    except WindowsError as e:
-        raise UtilityError(str(e))
-
-    if not unixoutput:
-        def encode(s):
-            if '\0' in s:
-                return s
-            return s.replace('\r\n', '\n')
-    else:
-        encode = lambda s: s
-
-    if rewriteslashes(name, args):
-        encode1_ = encode
-        def encode(s):
-            s = encode1_(s)
-            s = s.replace('\\\\', '\\')
-            s = s.replace('\\', '/')
-            return s
-
-    if ishg:
-        encode2_ = encode
-        def encode(s):
-            return rewritehg(encode2_(s))
-    
-    stdout.write(encode(out))
-    if not redirected:
-        stderr.write(encode(err))
-    return p.returncode
-            

bitbake/lib/bb/pysh/lsprof.py

@@ -1,116 +0,0 @@
-#! /usr/bin/env python
-
-import sys
-from _lsprof import Profiler, profiler_entry
-
-__all__ = ['profile', 'Stats']
-
-def profile(f, *args, **kwds):
-    """XXX docstring"""
-    p = Profiler()
-    p.enable(subcalls=True, builtins=True)
-    try:
-        f(*args, **kwds)
-    finally:
-        p.disable()
-    return Stats(p.getstats())
-
-
-class Stats(object):
-    """XXX docstring"""
-
-    def __init__(self, data):
-        self.data = data
-
-    def sort(self, crit="inlinetime"):
-        """XXX docstring"""
-        if crit not in profiler_entry.__dict__:
-            raise ValueError("Can't sort by %s" % crit)
-        self.data.sort(lambda b, a: cmp(getattr(a, crit),
-                                        getattr(b, crit)))
-        for e in self.data:
-            if e.calls:
-                e.calls.sort(lambda b, a: cmp(getattr(a, crit),
-                                              getattr(b, crit)))
-
-    def pprint(self, top=None, file=None, limit=None, climit=None):
-        """XXX docstring"""
-        if file is None:
-            file = sys.stdout
-        d = self.data
-        if top is not None:
-            d = d[:top]
-        cols = "% 12s %12s %11.4f %11.4f   %s\n"
-        hcols = "% 12s %12s %12s %12s %s\n"
-        cols2 = "+%12s %12s %11.4f %11.4f +  %s\n"
-        file.write(hcols % ("CallCount", "Recursive", "Total(ms)",
-                            "Inline(ms)", "module:lineno(function)"))
-        count = 0
-        for e in d:
-            file.write(cols % (e.callcount, e.reccallcount, e.totaltime,
-                               e.inlinetime, label(e.code)))
-            count += 1
-            if limit is not None and count == limit:
-                return
-            ccount = 0
-            if e.calls:
-                for se in e.calls:
-                    file.write(cols % ("+%s" % se.callcount, se.reccallcount,
-                                       se.totaltime, se.inlinetime,
-                                       "+%s" % label(se.code)))
-                    count += 1
-                    ccount += 1
-                    if limit is not None and count == limit:
-                        return
-                    if climit is not None and ccount == climit:
-                        break
-
-    def freeze(self):
-        """Replace all references to code objects with string
-        descriptions; this makes it possible to pickle the instance."""
-
-        # this code is probably rather ickier than it needs to be!
-        for i in range(len(self.data)):
-            e = self.data[i]
-            if not isinstance(e.code, str):
-                self.data[i] = type(e)((label(e.code),) + e[1:])
-            if e.calls:
-                for j in range(len(e.calls)):
-                    se = e.calls[j]
-                    if not isinstance(se.code, str):
-                        e.calls[j] = type(se)((label(se.code),) + se[1:])
-
-_fn2mod = {}
-
-def label(code):
-    if isinstance(code, str):
-        return code
-    try:
-        mname = _fn2mod[code.co_filename]
-    except KeyError:
-        for k, v in sys.modules.items():
-            if v is None:
-                continue
-            if not hasattr(v, '__file__'):
-                continue
-            if not isinstance(v.__file__, str):
-                continue
-            if v.__file__.startswith(code.co_filename):
-                mname = _fn2mod[code.co_filename] = k
-                break
-        else:
-            mname = _fn2mod[code.co_filename] = '<%s>'%code.co_filename
-
-    return '%s:%d(%s)' % (mname, code.co_firstlineno, code.co_name)
-
-
-if __name__ == '__main__':
-    import os
-    sys.argv = sys.argv[1:]
-    if not sys.argv:
-        print >> sys.stderr, "usage: lsprof.py <script> <arguments...>"
-        sys.exit(2)
-    sys.path.insert(0, os.path.abspath(os.path.dirname(sys.argv[0])))
-    stats = profile(execfile, sys.argv[0], globals(), locals())
-    stats.sort()
-    stats.pprint()

bitbake/lib/bb/pysh/pysh.py

@@ -1,167 +0,0 @@
-# pysh.py - command processing for pysh.
-#
-# Copyright 2007 Patrick Mezard
-#
-# This software may be used and distributed according to the terms
-# of the GNU General Public License, incorporated herein by reference.
-
-import optparse
-import os
-import sys
-
-import interp
-
-SH_OPT = optparse.OptionParser(prog='pysh', usage="%prog [OPTIONS]", version='0.1')
-SH_OPT.add_option('-c', action='store_true', dest='command_string', default=None, 
-    help='A string that shall be interpreted by the shell as one or more commands')
-SH_OPT.add_option('--redirect-to', dest='redirect_to', default=None, 
-    help='Redirect script commands stdout and stderr to the specified file')
-# See utility_command in builtin.py about the reason for this flag.
-SH_OPT.add_option('--redirected', dest='redirected', action='store_true', default=False, 
-    help='Tell the interpreter that stdout and stderr are actually the same objects, which is really stdout')
-SH_OPT.add_option('--debug-parsing', action='store_true', dest='debug_parsing', default=False, 
-    help='Trace PLY execution')
-SH_OPT.add_option('--debug-tree', action='store_true', dest='debug_tree', default=False, 
-    help='Display the generated syntax tree.')
-SH_OPT.add_option('--debug-cmd', action='store_true', dest='debug_cmd', default=False, 
-    help='Trace command execution before parameters expansion and exit status.')
-SH_OPT.add_option('--debug-utility', action='store_true', dest='debug_utility', default=False, 
-    help='Trace utility calls, after parameters expansions')
-SH_OPT.add_option('--ast', action='store_true', dest='ast', default=False,
-    help='Encoded commands to execute in a subprocess')
-SH_OPT.add_option('--profile', action='store_true', default=False,
-    help='Profile pysh run')
-    
-    
-def split_args(args):
-    # Separate shell arguments from command ones
-    # Just stop at the first argument not starting with a dash. I know, this is completely broken,
-    # it ignores files starting with a dash or may take option values for command file. This is not
-    # supposed to happen for now
-    command_index = len(args)
-    for i,arg in enumerate(args):
-        if not arg.startswith('-'):
-            command_index = i
-            break
-            
-    return args[:command_index], args[command_index:]
-
-
-def fixenv(env):
-    path = env.get('PATH')
-    if path is not None:
-        parts = path.split(os.pathsep)
-        # Remove Windows utilities from PATH, they are useless at best and
-        # some of them (find) may be confused with other utilities.
-        parts = [p for p in parts if 'system32' not in p.lower()]
-        env['PATH'] = os.pathsep.join(parts)
-    if env.get('HOME') is None:
-        # Several utilities, including cvsps, cannot work without
-        # a defined HOME directory.
-        env['HOME'] = os.path.expanduser('~')
-    return env
-
-def _sh(cwd, shargs, cmdargs, options, debugflags=None, env=None):
-    if os.environ.get('PYSH_TEXT') != '1':
-        import msvcrt
-        for fp in (sys.stdin, sys.stdout, sys.stderr):
-            msvcrt.setmode(fp.fileno(), os.O_BINARY)
-
-    hgbin = os.environ.get('PYSH_HGTEXT') != '1'
-    
-    if debugflags is None:
-        debugflags = []
-        if options.debug_parsing:    debugflags.append('debug-parsing')
-        if options.debug_utility:    debugflags.append('debug-utility')
-        if options.debug_cmd:        debugflags.append('debug-cmd')
-        if options.debug_tree:       debugflags.append('debug-tree')
-    
-    if env is None:
-        env = fixenv(dict(os.environ))
-    if cwd is None:
-        cwd = os.getcwd()
-
-    if not cmdargs:
-        # Nothing to do
-        return 0
-
-    ast = None
-    command_file = None
-    if options.command_string:
-        input = cmdargs[0]
-        if not options.ast:
-            input += '\n'
-        else:
-            args, input = interp.decodeargs(input), None
-            env, ast = args
-            cwd = env.get('PWD', cwd)
-    else:
-        command_file = cmdargs[0]
-        arguments = cmdargs[1:]
-
-        prefix = interp.resolve_shebang(command_file, ignoreshell=True)
-        if prefix:
-            input = ' '.join(prefix + [command_file] + arguments)
-        else:
-            # Read commands from file
-            f = file(command_file)
-            try:
-                # Trailing newline to help the parser
-                input = f.read() + '\n'
-            finally:
-                f.close()
-    
-    redirect = None
-    try:
-        if options.redirected:
-            stdout = sys.stdout
-            stderr = stdout
-        elif options.redirect_to:
-            redirect = open(options.redirect_to, 'wb')
-            stdout = redirect
-            stderr = redirect
-        else:
-            stdout = sys.stdout
-            stderr = sys.stderr
-            
-        # TODO: set arguments to environment variables
-        opts = interp.Options()
-        opts.hgbinary = hgbin
-        ip = interp.Interpreter(cwd, debugflags, stdout=stdout, stderr=stderr,
-                                opts=opts)
-        try:
-            # Export given environment in shell object
-            for k,v in env.iteritems():
-                ip.get_env().export(k,v)
-            return ip.execute_script(input, ast, scriptpath=command_file)
-        finally:
-            ip.close()
-    finally:
-        if redirect is not None:
-            redirect.close()
-
-def sh(cwd=None, args=None, debugflags=None, env=None):
-    if args is None:
-        args = sys.argv[1:]
-    shargs, cmdargs = split_args(args)
-    options, shargs = SH_OPT.parse_args(shargs)
-
-    if options.profile:
-        import lsprof
-        p = lsprof.Profiler()
-        p.enable(subcalls=True)
-        try:
-            return _sh(cwd, shargs, cmdargs, options, debugflags, env)
-        finally:
-            p.disable()
-            stats = lsprof.Stats(p.getstats())
-            stats.sort()
-            stats.pprint(top=10, file=sys.stderr, climit=5)
-    else:
-        return _sh(cwd, shargs, cmdargs, options, debugflags, env)
-            
-def main():
-    sys.exit(sh())
-
-if __name__=='__main__':
-    main()

bitbake/lib/bb/pysh/pyshlex.py

@@ -13,11 +13,6 @@
 # PLY in pull mode. It was designed to work incrementally and it would not be
 # that hard to enable pull mode.
 import re
-try:
-    s = set()
-    del s
-except NameError:
-    from Set import Set as set
 
 from ply import lex
 from bb.pysh.sherrors import *

bitbake/lib/bb/pysh/pyshyacc.py

@@ -636,13 +636,16 @@ def p_empty(p):
 def p_error(p):
     msg = []
     w = msg.append
-    w('%r\n' % p)
-    w('followed by:\n')
-    for i in range(5):
-        n = yacc.token()
-        if not n:
-            break
-        w('  %r\n' % n)
+    if p:
+        w('%r\n' % p)
+        w('followed by:\n')
+        for i in range(5):
+            n = yacc.token()
+            if not n:
+                break
+            w('  %r\n' % n)
+    else:
+        w('Unexpected EOF')
     raise sherrors.ShellSyntaxError(''.join(msg))
 
 # Build the parser

bitbake/lib/bb/pysh/sherrors.py

@@ -13,29 +13,3 @@ class ShellError(Exception):
 
 class ShellSyntaxError(ShellError):
     pass
-    
-class UtilityError(ShellError):
-    """Raised upon utility syntax error (option or operand error)."""
-    pass
-   
-class ExpansionError(ShellError):
-    pass
-     
-class CommandNotFound(ShellError):
-    """Specified command was not found."""
-    pass
-    
-class RedirectionError(ShellError):
-    pass
-    
-class VarAssignmentError(ShellError):
-    """Variable assignment error."""
-    pass
-    
-class ExitSignal(ShellError):
-    """Exit signal."""
-    pass
-    
-class ReturnSignal(ShellError):
-    """Exit signal."""
-    pass

bitbake/lib/bb/pysh/subprocess_fix.py

@@ -1,77 +0,0 @@
-# subprocess - Subprocesses with accessible I/O streams
-#
-# For more information about this module, see PEP 324.
-#
-# This module should remain compatible with Python 2.2, see PEP 291.
-#
-# Copyright (c) 2003-2005 by Peter Astrand <astrand@lysator.liu.se>
-#
-# Licensed to PSF under a Contributor Agreement.
-# See http://www.python.org/2.4/license for licensing details.
-
-def list2cmdline(seq):
-    """
-    Translate a sequence of arguments into a command line
-    string, using the same rules as the MS C runtime:
-
-    1) Arguments are delimited by white space, which is either a
-       space or a tab.
-
-    2) A string surrounded by double quotation marks is
-       interpreted as a single argument, regardless of white space
-       contained within.  A quoted string can be embedded in an
-       argument.
-
-    3) A double quotation mark preceded by a backslash is
-       interpreted as a literal double quotation mark.
-
-    4) Backslashes are interpreted literally, unless they
-       immediately precede a double quotation mark.
-
-    5) If backslashes immediately precede a double quotation mark,
-       every pair of backslashes is interpreted as a literal
-       backslash.  If the number of backslashes is odd, the last
-       backslash escapes the next double quotation mark as
-       described in rule 3.
-    """
-
-    # See
-    # http://msdn.microsoft.com/library/en-us/vccelng/htm/progs_12.asp
-    result = []
-    needquote = False
-    for arg in seq:
-        bs_buf = []
-
-        # Add a space to separate this argument from the others
-        if result:
-            result.append(' ')
-
-        needquote = (" " in arg) or ("\t" in arg) or ("|" in arg) or arg == ""
-        if needquote:
-            result.append('"')
-
-        for c in arg:
-            if c == '\\':
-                # Don't know if we need to double yet.
-                bs_buf.append(c)
-            elif c == '"':
-                # Double backspaces.
-                result.append('\\' * len(bs_buf)*2)
-                bs_buf = []
-                result.append('\\"')
-            else:
-                # Normal char
-                if bs_buf:
-                    result.extend(bs_buf)
-                    bs_buf = []
-                result.append(c)
-
-        # Add remaining backspaces, if any.
-        if bs_buf:
-            result.extend(bs_buf)
-
-        if needquote:
-            result.extend(bs_buf)
-            result.append('"')
-            
-    return ''.join(result)

bitbake/lib/bb/runqueue.py

@@ -1224,17 +1224,12 @@ class RunQueue:
         bb.utils.nonblockingfd(worker.stdout)
         workerpipe = runQueuePipe(worker.stdout, None, self.cfgData, self, rqexec)
 
-        runqhash = {}
-        for tid in self.rqdata.runtaskentries:
-            runqhash[tid] = self.rqdata.runtaskentries[tid].hash
-
         workerdata = {
             "taskdeps" : self.rqdata.dataCaches[mc].task_deps,
             "fakerootenv" : self.rqdata.dataCaches[mc].fakerootenv,
             "fakerootdirs" : self.rqdata.dataCaches[mc].fakerootdirs,
             "fakerootnoenv" : self.rqdata.dataCaches[mc].fakerootnoenv,
             "sigdata" : bb.parse.siggen.get_taskdata(),
-            "runq_hash" : runqhash,
             "logdefaultdebug" : bb.msg.loggerDefaultDebugLevel,
             "logdefaultverbose" : bb.msg.loggerDefaultVerbose,
             "logdefaultverboselogs" : bb.msg.loggerVerboseLogs,
@@ -1409,6 +1404,12 @@ class RunQueue:
                 bb.event.fire(bb.event.DepTreeGenerated(depgraph), self.cooker.data)
 
         if self.state is runQueueSceneInit:
+            if not self.dm_event_handler_registered:
+                 res = bb.event.register(self.dm_event_handler_name,
+                                         lambda x: self.dm.check(self) if self.state in [runQueueSceneRun, runQueueRunning, runQueueCleanUp] else False,
+                                         ('bb.event.HeartbeatEvent',))
+                 self.dm_event_handler_registered = True
+
             dump = self.cooker.configuration.dump_signatures
             if dump:
                 self.rqdata.init_progress_reporter.finish()
@@ -1425,11 +1426,6 @@ class RunQueue:
                 self.rqexe = RunQueueExecuteScenequeue(self)
 
         if self.state is runQueueSceneRun:
-            if not self.dm_event_handler_registered:
-                 res = bb.event.register(self.dm_event_handler_name,
-                                         lambda x: self.dm.check(self) if self.state in [runQueueSceneRun, runQueueRunning, runQueueCleanUp] else False,
-                                         ('bb.event.HeartbeatEvent',))
-                 self.dm_event_handler_registered = True
             retval = self.rqexe.execute()
 
         if self.state is runQueueRunInit:
@@ -2030,6 +2026,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
             taskdepdata = self.build_taskdepdata(task)
 
             taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn]
+            taskhash = self.rqdata.get_task_hash(task)
             if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not (self.cooker.configuration.dry_run or self.rqdata.setscene_enforce):
                 if not mc in self.rq.fakeworker:
                     try:
@@ -2039,10 +2036,10 @@ class RunQueueExecuteTasks(RunQueueExecute):
                         self.rq.state = runQueueFailed
                         self.stats.taskFailed()
                         return True
-                self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>")
+                self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>")
                 self.rq.fakeworker[mc].process.stdin.flush()
             else:
-                self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>")
+                self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>")
                 self.rq.worker[mc].process.stdin.flush()
 
             self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)
@@ -2456,13 +2453,14 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
             taskdepdata = self.build_taskdepdata(task)
 
             taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn]
+            taskhash = self.rqdata.get_task_hash(task)
             if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run:
                 if not mc in self.rq.fakeworker:
                     self.rq.start_fakeworker(self, mc)
-                self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>")
+                self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>")
                 self.rq.fakeworker[mc].process.stdin.flush()
             else:
-                self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>")
+                self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>")
                 self.rq.worker[mc].process.stdin.flush()
 
             self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)

bitbake/lib/bb/server/process.py

@@ -239,6 +239,12 @@ class ProcessServer(multiprocessing.Process):
         while not lock:
             with bb.utils.timeout(3):
                 lock = bb.utils.lockfile(lockfile, shared=False, retry=False, block=True)
+                if lock:
+                    # We hold the lock so we can remove the file (hide stale pid data)
+                    bb.utils.remove(lockfile)
+                    bb.utils.unlockfile(lock)
+                    return
+
                 if not lock:
                     # Some systems may not have lsof available
                     procs = None
@@ -259,10 +265,6 @@ class ProcessServer(multiprocessing.Process):
                     if procs:
                         msg += ":\n%s" % str(procs)
                     print(msg)
-                    return
-        # We hold the lock so we can remove the file (hide stale pid data)
-        bb.utils.remove(lockfile)
-        bb.utils.unlockfile(lock)
 
     def idle_commands(self, delay, fds=None):
         nextsleep = delay
@@ -398,52 +400,69 @@ class BitBakeServer(object):
         os.close(self.readypipein)
 
         ready = ConnectionReader(self.readypipe)
-        r = ready.poll(30)
+        r = ready.poll(5)
+        if not r:
+            bb.note("Bitbake server didn't start within 5 seconds, waiting for 90")
+            r = ready.poll(90)
         if r:
             try:
                 r = ready.get()
             except EOFError:
                 # Trap the child exitting/closing the pipe and error out
                 r = None
-        if not r or r != "ready":
+        if not r or r[0] != "r":
             ready.close()
-            bb.error("Unable to start bitbake server")
+            bb.error("Unable to start bitbake server (%s)" % str(r))
             if os.path.exists(logfile):
                 logstart_re = re.compile(self.start_log_format % ('([0-9]+)', '([0-9-]+ [0-9:.]+)'))
                 started = False
                 lines = []
+                lastlines = []
                 with open(logfile, "r") as f:
                     for line in f:
                         if started:
                             lines.append(line)
                         else:
+                            lastlines.append(line)
                             res = logstart_re.match(line.rstrip())
                             if res:
                                 ldatetime = datetime.datetime.strptime(res.group(2), self.start_log_datetime_format)
                                 if ldatetime >= startdatetime:
                                     started = True
                                     lines.append(line)
+                        if len(lastlines) > 60:
+                            lastlines = lastlines[-60:]
                 if lines:
-                    if len(lines) > 10:
-                        bb.error("Last 10 lines of server log for this session (%s):\n%s" % (logfile, "".join(lines[-10:])))
+                    if len(lines) > 60:
+                        bb.error("Last 60 lines of server log for this session (%s):\n%s" % (logfile, "".join(lines[-60:])))
                     else:
                         bb.error("Server log for this session (%s):\n%s" % (logfile, "".join(lines)))
+                elif lastlines:
+                        bb.error("Server didn't start, last 60 loglines (%s):\n%s" % (logfile, "".join(lastlines)))
+            else:
+                bb.error("%s doesn't exist" % logfile)
+
             raise SystemExit(1)
+
         ready.close()
 
     def _startServer(self):
         print(self.start_log_format % (os.getpid(), datetime.datetime.now().strftime(self.start_log_datetime_format)))
+        sys.stdout.flush()
+
         server = ProcessServer(self.bitbake_lock, self.sock, self.sockname)
         self.configuration.setServerRegIdleCallback(server.register_idle_function)
         os.close(self.readypipe)
         writer = ConnectionWriter(self.readypipein)
         self.cooker = bb.cooker.BBCooker(self.configuration, self.featureset)
-        writer.send("ready")
+        writer.send("r")
         writer.close()
         server.cooker = self.cooker
         server.server_timeout = self.configuration.server_timeout
         server.xmlrpcinterface = self.configuration.xmlrpcinterface
         print("Started bitbake server pid %d" % os.getpid())
+        sys.stdout.flush()
+
         server.start()
 
 def connectProcessServer(sockname, featureset):
@@ -452,16 +471,17 @@ def connectProcessServer(sockname, featureset):
     # AF_UNIX has path length issues so chdir here to workaround
     cwd = os.getcwd()
 
-    try:
-        os.chdir(os.path.dirname(sockname))
-        sock.connect(os.path.basename(sockname))
-    finally:
-        os.chdir(cwd)
-
     readfd = writefd = readfd1 = writefd1 = readfd2 = writefd2 = None
     eq = command_chan_recv = command_chan = None
 
+    sock.settimeout(2)
+
     try:
+        try:
+            os.chdir(os.path.dirname(sockname))
+            sock.connect(os.path.basename(sockname))
+        finally:
+            os.chdir(cwd)
 
         # Send an fd for the remote to write events to
         readfd, writefd = os.pipe()
@@ -490,7 +510,8 @@ def connectProcessServer(sockname, featureset):
             command_chan.close()
         for i in [writefd, readfd1, writefd2]:
             try:
-                os.close(i)
+                if i:
+                    os.close(i)
             except OSError:
                 pass
         sock.close()

bitbake/lib/bb/siggen.py

@@ -185,7 +185,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
             if not self.rundep_check(fn, recipename, task, dep, depname, dataCache):
                 continue
             if dep not in self.taskhash:
-                bb.fatal("%s is not in taskhash, caller isn't calling in dependency order?", dep)
+                bb.fatal("%s is not in taskhash, caller isn't calling in dependency order?" % dep)
             data = data + self.taskhash[dep]
             self.runtaskdeps[k].append(dep)
 
@@ -311,6 +311,13 @@ class SignatureGeneratorBasic(SignatureGenerator):
 class SignatureGeneratorBasicHash(SignatureGeneratorBasic):
     name = "basichash"
 
+    def get_stampfile_hash(self, task):
+        if task in self.taskhash:
+            return self.taskhash[task]
+
+        # If task is not in basehash, then error
+        return self.basehash[task]
+
     def stampfile(self, stampbase, fn, taskname, extrainfo, clean=False):
         if taskname != "do_setscene" and taskname.endswith("_setscene"):
             k = fn + "." + taskname[:-9]
@@ -318,11 +325,9 @@ class SignatureGeneratorBasicHash(SignatureGeneratorBasic):
             k = fn + "." + taskname
         if clean:
             h = "*"
-        elif k in self.taskhash:
-            h = self.taskhash[k]
         else:
-            # If k is not in basehash, then error
-            h = self.basehash[k]
+            h = self.get_stampfile_hash(k)
+
         return ("%s.%s.%s.%s" % (stampbase, taskname, h, extrainfo)).rstrip('.')
 
     def stampcleanmask(self, stampbase, fn, taskname, extrainfo):
@@ -342,10 +347,10 @@ def dump_this_task(outfile, d):
 def init_colors(enable_color):
     """Initialise colour dict for passing to compare_sigfiles()"""
     # First set up the colours
-    colors = {'color_title':   '\033[1;37;40m',
-              'color_default': '\033[0;37;40m',
-              'color_add':     '\033[1;32;40m',
-              'color_remove':  '\033[1;31;40m',
+    colors = {'color_title':   '\033[1m',
+              'color_default': '\033[0m',
+              'color_add':     '\033[0;32m',
+              'color_remove':  '\033[0;31m',
              }
     # Leave all keys present but clear the values
     if not enable_color:

bitbake/lib/bb/tests/fetch.py

@@ -1708,9 +1708,8 @@ class GitShallowTest(FetcherTest):
         with self.assertRaises(bb.fetch2.UnpackError) as context:
             fetcher.unpack(self.d.getVar('WORKDIR'))
 
-        self.assertTrue("No up to date source found" in context.exception.msg)
-        self.assertTrue("clone directory not available or not up to date" in context.exception.msg)
-        self.assertTrue("shallow clone not enabled or not available" in context.exception.msg)
+        self.assertIn("No up to date source found", context.exception.msg)
+        self.assertIn("clone directory not available or not up to date", context.exception.msg)
 
     @skipIfNoNetwork()
     def test_that_unpack_does_work_when_using_git_shallow_tarball_but_tarball_is_not_available(self):

bitbake/lib/bb/utils.py

@@ -27,7 +27,8 @@ import bb
 import bb.msg
 import multiprocessing
 import fcntl
-import imp
+import importlib
+from importlib import machinery
 import itertools
 import subprocess
 import glob
@@ -43,7 +44,7 @@ from contextlib import contextmanager
 from ctypes import cdll
 
 logger = logging.getLogger("BitBake.Util")
-python_extensions = [e for e, _, _ in imp.get_suffixes()]
+python_extensions = importlib.machinery.all_suffixes()
 
 
 def clean_context():
@@ -68,8 +69,8 @@ class VersionStringException(Exception):
 
 def explode_version(s):
     r = []
-    alpha_regexp = re.compile('^([a-zA-Z]+)(.*)$')
-    numeric_regexp = re.compile('^(\d+)(.*)$')
+    alpha_regexp = re.compile(r'^([a-zA-Z]+)(.*)$')
+    numeric_regexp = re.compile(r'^(\d+)(.*)$')
     while (s != ''):
         if s[0] in string.digits:
             m = numeric_regexp.match(s)
@@ -317,10 +318,13 @@ def better_compile(text, file, realfile, mode = "exec", lineno = 0):
         error = []
         # split the text into lines again
         body = text.split('\n')
-        error.append("Error in compiling python function in %s, line %s:\n" % (realfile, lineno))
+        error.append("Error in compiling python function in %s, line %s:\n" % (realfile, e.lineno))
         if hasattr(e, "lineno"):
             error.append("The code lines resulting in this error were:")
-            error.extend(_print_trace(body, e.lineno))
+            # e.lineno: line's position in reaflile
+            # lineno: function name's "position -1" in realfile
+            # e.lineno - lineno: line's relative position in function
+            error.extend(_print_trace(body, e.lineno - lineno))
         else:
             error.append("The function causing this error was:")
             for line in body:
@@ -704,15 +708,7 @@ def prunedir(topdir):
     # CAUTION:  This is dangerous!
     if _check_unsafe_delete_path(topdir):
         raise Exception('bb.utils.prunedir: called with dangerous path "%s", refusing to delete!' % topdir)
-    for root, dirs, files in os.walk(topdir, topdown = False):
-        for name in files:
-            os.remove(os.path.join(root, name))
-        for name in dirs:
-            if os.path.islink(os.path.join(root, name)):
-                os.remove(os.path.join(root, name))
-            else:
-                os.rmdir(os.path.join(root, name))
-    os.rmdir(topdir)
+    remove(topdir, recurse=True)
 
 #
 # Could also use return re.compile("(%s)" % "|".join(map(re.escape, suffixes))).sub(lambda mo: "", var)
@@ -1157,14 +1153,14 @@ def edit_metadata(meta_lines, variables, varfunc, match_overrides=False):
 
     var_res = {}
     if match_overrides:
-        override_re = '(_[a-zA-Z0-9-_$(){}]+)?'
+        override_re = r'(_[a-zA-Z0-9-_$(){}]+)?'
     else:
         override_re = ''
     for var in variables:
         if var.endswith('()'):
-            var_res[var] = re.compile('^(%s%s)[ \\t]*\([ \\t]*\)[ \\t]*{' % (var[:-2].rstrip(), override_re))
+            var_res[var] = re.compile(r'^(%s%s)[ \\t]*\([ \\t]*\)[ \\t]*{' % (var[:-2].rstrip(), override_re))
         else:
-            var_res[var] = re.compile('^(%s%s)[ \\t]*[?+:.]*=[+.]*[ \\t]*(["\'])' % (var, override_re))
+            var_res[var] = re.compile(r'^(%s%s)[ \\t]*[?+:.]*=[+.]*[ \\t]*(["\'])' % (var, override_re))
 
     updated = False
     varset_start = ''
@@ -1501,6 +1497,8 @@ def ioprio_set(who, cls, value):
       NR_ioprio_set = 251
     elif _unamearch[0] == "i" and _unamearch[2:3] == "86":
       NR_ioprio_set = 289
+    elif _unamearch == "aarch64":
+      NR_ioprio_set = 30
 
     if NR_ioprio_set:
         ioprio = value | (cls << IOPRIO_CLASS_SHIFT)
@@ -1544,12 +1542,9 @@ def export_proxies(d):
 def load_plugins(logger, plugins, pluginpath):
     def load_plugin(name):
         logger.debug(1, 'Loading plugin %s' % name)
-        fp, pathname, description = imp.find_module(name, [pluginpath])
-        try:
-            return imp.load_module(name, fp, pathname, description)
-        finally:
-            if fp:
-                fp.close()
+        spec = importlib.machinery.PathFinder.find_spec(name, path=[pluginpath] )
+        if spec:
+            return spec.loader.load_module()
 
     logger.debug(1, 'Loading plugins from %s...' % pluginpath)
 

bitbake/lib/bblayers/layerindex.py

@@ -28,7 +28,7 @@ class LayerIndexPlugin(ActionPlugin):
         layerdir = os.path.join(repodir, subdir)
         if not os.path.exists(repodir):
             if fetch_layer:
-                result = subprocess.call('git clone %s %s' % (url, repodir), shell = True)
+                result = subprocess.call(['git', 'clone', url, repodir])
                 if result:
                     logger.error("Failed to download %s" % url)
                     return None, None, None

documentation/brief-yoctoprojectqs/brief-yoctoprojectqs.xml

@@ -37,17 +37,30 @@
             hardware.
             You will use Yocto Project to build a reference embedded OS
             called Poky.
-            <note>
-                The examples in this paper assume you are using a native Linux
-                system running a recent Ubuntu Linux distribution.
-                If the machine you want to use
-                Yocto Project on to build an image is not a native Linux
-                system, you can still perform these steps by using CROss
-                PlatformS (CROPS) and setting up a Poky container.
-                See the
-                <ulink url='&YOCTO_DOCS_DEV_URL;#setting-up-to-use-crops'>Setting Up to Use CROss PlatformS (CROPS)</ulink>"
-                section in the Yocto Project Development Tasks Manual for more
-                information.
+            <note><title>Notes</title>
+                <itemizedlist>
+                    <listitem><para>
+                        The examples in this paper assume you are using a
+                        native Linux system running a recent Ubuntu Linux
+                        distribution.
+                        If the machine you want to use Yocto Project on to
+                        build an image
+                        (<ulink url='&YOCTO_DOCS_REF_URL;#hardware-build-system-term'>build host</ulink>)
+                        is not a native Linux system, you can
+                        still perform these steps by using CROss PlatformS
+                        (CROPS) and setting up a Poky container.
+                        See the
+                        <ulink url='&YOCTO_DOCS_DEV_URL;#setting-up-to-use-crops'>Setting Up to Use CROss PlatformS (CROPS)</ulink>"
+                        section in the Yocto Project Development Tasks Manual for more
+                        information.
+                        </para></listitem>
+                    <listitem><para>
+                        You cannot use a build host that is using the
+                        <ulink url='https://en.wikipedia.org/wiki/Windows_Subsystem_for_Linux'>Windows Subsystem for Linux</ulink>
+                        (WSL).
+                        The Yocto Project is not compatible with WSL.
+                        </para></listitem>
+                </itemizedlist>
             </note>
         </para>
 
@@ -75,6 +88,10 @@
                     Linux distributions that support the Yocto Project, see the
                     "<ulink url='&YOCTO_DOCS_REF_URL;#detailed-supported-distros'>Supported Linux Distributions</ulink>"
                     section in the Yocto Project Reference Manual.
+                    For detailed information on preparing your build host, see
+                    the
+                    "<ulink url='&YOCTO_DOCS_DEV_URL;#dev-preparing-the-build-host'>Preparing the Build Host</ulink>"
+                    section in the Yocto Project Development Tasks Manual.
                     </para></listitem>
                 <listitem><para>
                     <itemizedlist>

documentation/bsp-guide/bsp-guide.xml

@@ -123,9 +123,14 @@
             </revision>
             <revision>
                 <revnumber>2.6</revnumber>
-                <date>&REL_MONTH_YEAR;</date>
+                <date>November 2018</date>
                 <revremark>Released with the Yocto Project 2.6 Release.</revremark>
             </revision>
+            <revision>
+                <revnumber>2.7</revnumber>
+                <date>&REL_MONTH_YEAR;</date>
+                <revremark>Released with the Yocto Project 2.7 Release.</revremark>
+            </revision>
         </revhistory>
 
     <copyright>

documentation/dev-manual/dev-manual-common-tasks.xml

@@ -3237,30 +3237,22 @@
                 post-installation script to be delayed until the first boot.
                 For example, the script might need to be executed on the
                 device itself.
-                To delay script execution until boot time, use the following
-                structure in the post-installation script:
-                <literallayout class='monospaced'>
-     pkg_postinst_<replaceable>PACKAGENAME</replaceable>() {
-     if [ x"$D" = "x" ]; then
-          # Actions to carry out on the device go here
-     else
-          exit 1
-     fi
-     }
-                </literallayout>
+                To delay script execution until boot time, you must explicitly
+                mark post installs to defer to the target.
+                You can use <filename>pkg_postinst_ontarget()</filename> or
+                call
+                <filename>postinst-intercepts defer_to_first_boot</filename>
+                from <filename>pkg_postinst()</filename>.
+                Any failure of a <filename>pkg_postinst()</filename> script
+                (including exit 1) triggers an error during the
+                <ulink url='&YOCTO_DOCS_REF_URL;#ref-tasks-rootfs'><filename>do_rootfs</filename></ulink>
+                task.
             </para>
 
             <para>
-                The previous example delays execution until the image boots
-                again because the environment variable <filename>D</filename>
-                points to the directory containing the image when
-                the root filesystem is created at build time but is unset
-                when executed on the first boot.
-            </para>
-
-            <para>
-                If you have recipes that use <filename>pkg_postinst</filename>
-                scripts and they require the use of non-standard native
+                If you have recipes that use
+                <filename>pkg_postinst</filename> function
+                and they require the use of non-standard native
                 tools that have dependencies during rootfs construction, you
                 need to use the
                 <ulink url='&YOCTO_DOCS_REF_URL;#var-PACKAGE_WRITE_DEPS'><filename>PACKAGE_WRITE_DEPS</filename></ulink>
@@ -3279,8 +3271,8 @@
                 <filename>pkg_prerm</filename>, and
                 <filename>pkg_postrm</filename>, respectively.
                 These scrips work in exactly the same way as does
-                <filename>pkg_postinst</filename> with the exception that they
-                run at different times.
+                <filename>pkg_postinst</filename> with the exception
+                that they run at different times.
                 Also, because of when they run, they are not applicable to
                 being run at image creation time like
                 <filename>pkg_postinst</filename>.
@@ -7270,13 +7262,12 @@
                         easier-to-use and more flexible replacements for an
                         existing functionality in OE-Core's
                         <ulink url='&YOCTO_DOCS_REF_URL;#ref-classes-image-live'><filename>image-live</filename></ulink>
-                        class and <filename>mkefidisk.sh</filename> script.
-                        The difference between
-                        Wic and those examples is
-                        that with Wic the
-                        functionality of those scripts is implemented
-                        by a general-purpose partitioning language, which is
-                        based on Redhat kickstart syntax.</para></listitem>
+                        class.
+                        The difference between Wic and those examples is
+                        that with Wic the functionality of those scripts is
+                        implemented by a general-purpose partitioning language,
+                        which is based on Redhat kickstart syntax.
+                        </para></listitem>
                 </itemizedlist>
             </para>
         </section>
@@ -11798,16 +11789,15 @@
                         To run the tests automatically after the
                         OpenEmbedded build system successfully creates an image,
                         first set the
-                        <ulink url='&YOCTO_DOCS_REF_URL;#var-TEST_IMAGE'><filename>TEST_IMAGE</filename></ulink>
+                        <ulink url='&YOCTO_DOCS_REF_URL;#var-TESTIMAGE_AUTO'><filename>TESTIMAGE_AUTO</filename></ulink>
                         variable to "1" in your <filename>local.conf</filename>
                         file in the
                         <ulink url='&YOCTO_DOCS_REF_URL;#build-directory'>Build Directory</ulink>:
                         <literallayout class='monospaced'>
-     TEST_IMAGE = "1"
+     TESTIMAGE_AUTO = "1"
                         </literallayout>
                         Next, build your image.
-                        If the image successfully builds, the tests will be
-                        run:
+                        If the image successfully builds, the tests run:
                         <literallayout class='monospaced'>
      bitbake core-image-sato
                         </literallayout></para></listitem>

documentation/dev-manual/dev-manual-start.xml

@@ -408,6 +408,12 @@
         <ulink url='https://git.yoctoproject.org/cgit/cgit.cgi/crops/about/'>CROPS</ulink>,
         which leverages
         <ulink url='https://www.docker.com/'>Docker Containers</ulink>.
+        <note>
+            You cannot use a build host that is using the
+            <ulink url='https://en.wikipedia.org/wiki/Windows_Subsystem_for_Linux'>Windows Subsystem for Linux</ulink>
+            (WSL).
+            The Yocto Project is not compatible with WSL.
+        </note>
     </para>
 
     <para>

documentation/dev-manual/dev-manual.xml

@@ -108,9 +108,14 @@
             </revision>
             <revision>
                 <revnumber>2.6</revnumber>
-                <date>&REL_MONTH_YEAR;</date>
+                <date>November 2018</date>
                 <revremark>Released with the Yocto Project 2.6 Release.</revremark>
             </revision>
+            <revision>
+                <revnumber>2.7</revnumber>
+                <date>&REL_MONTH_YEAR;</date>
+                <revremark>Released with the Yocto Project 2.7 Release.</revremark>
+            </revision>
         </revhistory>
 
     <copyright>

documentation/kernel-dev/kernel-dev-common.xml

@@ -1603,8 +1603,11 @@
             <title>Creating a&nbsp;&nbsp;<filename>defconfig</filename> File</title>
 
             <para>
-                A <filename>defconfig</filename> file is simply a
-                <filename>.config</filename> renamed to "defconfig".
+                A <filename>defconfig</filename> file in the context of
+                the Yocto Project is often a <filename>.config</filename>
+                file that is copied from a build or a
+                <filename>defconfig</filename> taken from the kernel tree
+                and moved into recipe space.
                 You can use a <filename>defconfig</filename> file
                 to retain a known set of kernel configurations from which the
                 OpenEmbedded build system can draw to create the final

documentation/kernel-dev/kernel-dev.xml

@@ -93,9 +93,14 @@
             </revision>
             <revision>
                 <revnumber>2.6</revnumber>
-                <date>&REL_MONTH_YEAR;</date>
+                <date>November 2018</date>
                 <revremark>Released with the Yocto Project 2.6 Release.</revremark>
             </revision>
+            <revision>
+                <revnumber>2.7</revnumber>
+                <date>&REL_MONTH_YEAR;</date>
+                <revremark>Released with the Yocto Project 2.7 Release.</revremark>
+            </revision>
         </revhistory>
 
     <copyright>

documentation/mega-manual/mega-manual.xml

@@ -77,9 +77,14 @@
             </revision>
             <revision>
                 <revnumber>2.6</revnumber>
-                <date>&REL_MONTH_YEAR;</date>
+                <date>November 2018</date>
                 <revremark>Released with the Yocto Project 2.6 Release.</revremark>
             </revision>
+            <revision>
+                <revnumber>2.7</revnumber>
+                <date>&REL_MONTH_YEAR;</date>
+                <revremark>Released with the Yocto Project 2.7 Release.</revremark>
+            </revision>
        </revhistory>
 
     <copyright>

documentation/poky.ent

@@ -1,19 +1,19 @@
-<!ENTITY DISTRO "2.6">
-<!ENTITY DISTRO_COMPRESSED "26">
-<!ENTITY DISTRO_NAME_NO_CAP "thud">
-<!ENTITY DISTRO_NAME "Thud">
-<!ENTITY DISTRO_NAME_NO_CAP_MINUS_ONE "sumo">
-<!ENTITY DISTRO_NAME_MINUS_ONE "Sumo">
-<!ENTITY YOCTO_DOC_VERSION "2.6">
-<!ENTITY YOCTO_DOC_VERSION_MINUS_ONE "2.5">
-<!ENTITY DISTRO_REL_TAG "yocto-2.6">
+<!ENTITY DISTRO "2.7">
+<!ENTITY DISTRO_COMPRESSED "27">
+<!ENTITY DISTRO_NAME_NO_CAP "warrior">
+<!ENTITY DISTRO_NAME "Warrior">
+<!ENTITY DISTRO_NAME_NO_CAP_MINUS_ONE "thud">
+<!ENTITY DISTRO_NAME_MINUS_ONE "Thud">
+<!ENTITY YOCTO_DOC_VERSION "2.7">
+<!ENTITY YOCTO_DOC_VERSION_MINUS_ONE "2.6">
+<!ENTITY DISTRO_REL_TAG "yocto-2.7">
 <!ENTITY METAINTELVERSION "9.0">
-<!ENTITY REL_MONTH_YEAR "October 2018">
+<!ENTITY REL_MONTH_YEAR "TBD">
 <!ENTITY META_INTEL_REL_TAG "&METAINTELVERSION;-&DISTRO_NAME_NO_CAP;-&YOCTO_DOC_VERSION;">
-<!ENTITY POKYVERSION "21.0.0">
-<!ENTITY POKYVERSION_COMPRESSED "2100">
+<!ENTITY POKYVERSION "22.0.0">
+<!ENTITY POKYVERSION_COMPRESSED "2200">
 <!ENTITY YOCTO_POKY "poky-&DISTRO_NAME_NO_CAP;-&POKYVERSION;">
-<!ENTITY COPYRIGHT_YEAR "2010-2018">
+<!ENTITY COPYRIGHT_YEAR "2010-2019">
 <!ENTITY YOCTO_DL_URL "http://downloads.yoctoproject.org">
 <!ENTITY YOCTO_HOME_URL "http://www.yoctoproject.org">
 <!ENTITY YOCTO_LISTS_URL "http://lists.yoctoproject.org">

documentation/profile-manual/profile-manual.xml

@@ -93,9 +93,14 @@
             </revision>
             <revision>
                 <revnumber>2.6</revnumber>
-                <date>&REL_MONTH_YEAR;</date>
+                <date>November 2018</date>
                 <revremark>Released with the Yocto Project 2.6 Release.</revremark>
             </revision>
+            <revision>
+                <revnumber>2.7</revnumber>
+                <date>&REL_MONTH_YEAR;</date>
+                <revremark>Released with the Yocto Project 2.7 Release.</revremark>
+            </revision>
         </revhistory>
 
     <copyright>

documentation/ref-manual/migration.xml

@@ -3619,7 +3619,7 @@ $ runqemu qemux86-64 tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.
 
         <para>
             The
-            <link linkend='var-KERNEL_IMAGE_BASE_NAME'><filename>KERNEL_IMAGE_BASE_NAME</filename></link>
+            <filename>KERNEL_IMAGE_BASE_NAME</filename>
             variable no longer uses the
             <link linkend='var-KERNEL_IMAGETYPE'><filename>KERNEL_IMAGETYPE</filename></link>
             variable to create the image's base name.
@@ -5557,8 +5557,9 @@ id=f4d4f99cfbc2396e49c1613a7d237b9e57f06f81'>commit message</ulink>.
                     incompatible with other implementations.
                     </para></listitem>
                 <listitem><para>
-                    By default, the <filename>cmake</filename> class uses
-                    <filename>ninja</filename> instead of
+                    By default, the
+                    <link linkend='ref-classes-cmake'><filename>cmake</filename></link>
+                    class uses <filename>ninja</filename> instead of
                     <filename>make</filename> for building.
                     This improves build performance.
                     If a recipe is broken with <filename>ninja</filename>, then
@@ -5608,7 +5609,11 @@ id=f4d4f99cfbc2396e49c1613a7d237b9e57f06f81'>commit message</ulink>.
                     Any failure of a <filename>pkg_postinst()</filename>
                     script (including <filename>exit 1</filename>)
                     will trigger a warning during
-                    <filename>do_rootfs</filename>.
+                    <filename>do_rootfs</filename>.</para>
+
+                    <para>For more information, see the
+                    "<ulink url='&YOCTO_DOCS_DEV_URL;#new-recipe-post-installation-scripts'>Post-Installation Scripts</ulink>"
+                    section in the Yocto Project Development Tasks Manual.
                     </para></listitem>
                 <listitem><para>
                     The <filename>elf</filename> image type has been removed.
@@ -5678,6 +5683,648 @@ id=f4d4f99cfbc2396e49c1613a7d237b9e57f06f81'>commit message</ulink>.
         </para>
     </section>
 </section>
+
+<section id='moving-to-the-yocto-project-2.6-release'>
+    <title>Moving to the Yocto Project 2.6 Release</title>
+
+    <para>
+        This section provides migration information for moving to the
+        Yocto Project 2.6 Release from the prior release.
+    </para>
+
+    <section id='migration-2.6-gcc-changes'>
+        <title>GCC 8.2 is Now Used by Default</title>
+
+        <para>
+            The GNU Compiler Collection version 8.2 is now used by default
+            for compilation.
+            For more information on what has changed in the GCC 8.x release,
+            see
+            <ulink url='https://gcc.gnu.org/gcc-8/changes.html'></ulink>.
+        </para>
+
+        <para>
+            If you still need to compile with version 7.x, GCC 7.3 is
+            also provided.
+            You can select this version by setting the
+            and can be selected by setting the
+            <link linkend='var-GCCVERSION'><filename>GCCVERSION</filename></link>
+            variable to "7.%" in your configuration.
+        </para>
+    </section>
+
+    <section id='migration-2.6-removed-recipes'>
+        <title>Removed Recipes</title>
+
+        <para>
+            The following recipes have been removed:
+            <literallayout class='monospaced'>
+     <emphasis><filename>beecrypt</filename>:</emphasis> No longer needed since moving to RPM 4.
+     <emphasis><filename>bigreqsproto</filename>:</emphasis> Replaced by <filename>xorgproto</filename>.
+     <emphasis><filename>calibrateproto</filename>:</emphasis> Removed in favor of <filename>xinput</filename>.
+     <emphasis><filename>compositeproto</filename>:</emphasis> Replaced by <filename>xorgproto</filename>.
+     <emphasis><filename>damageproto</filename>:</emphasis> Replaced by <filename>xorgproto</filename>.
+     <emphasis><filename>dmxproto</filename>:</emphasis> Replaced by <filename>xorgproto</filename>.
+     <emphasis><filename>dri2proto</filename>:</emphasis> Replaced by <filename>xorgproto</filename>.
+     <emphasis><filename>dri3proto</filename>:</emphasis> Replaced by <filename>xorgproto</filename>.
+     <emphasis><filename>eee-acpi-scripts</filename>:</emphasis> Became obsolete.
+     <emphasis><filename>fixesproto</filename>:</emphasis> Replaced by <filename>xorgproto</filename>.
+     <emphasis><filename>fontsproto</filename>:</emphasis> Replaced by <filename>xorgproto</filename>.
+     <emphasis><filename>fstests</filename>:</emphasis> Became obsolete.
+     <emphasis><filename>gccmakedep</filename>:</emphasis> No longer used.
+     <emphasis><filename>glproto</filename>:</emphasis> Replaced by <filename>xorgproto</filename>.
+     <emphasis><filename>gnome-desktop3</filename>:</emphasis> No longer needed.  This recipe has moved to <filename>meta-oe</filename>.
+     <emphasis><filename>icon-naming-utils</filename>:</emphasis> No longer used since the Sato theme was removed in 2016.
+     <emphasis><filename>inputproto</filename>:</emphasis> Replaced by <filename>xorgproto</filename>.
+     <emphasis><filename>kbproto</filename>:</emphasis> Replaced by <filename>xorgproto</filename>.
+     <emphasis><filename>libusb-compat</filename>:</emphasis> Became obsolete.
+     <emphasis><filename>libuser</filename>:</emphasis> Became obsolete.
+     <emphasis><filename>libnfsidmap</filename>:</emphasis> No longer an external requirement since <filename>nfs-utils</filename> 2.2.1.  <filename>libnfsidmap</filename> is now integrated.
+     <emphasis><filename>libxcalibrate</filename>:</emphasis> No longer needed with <filename>xinput</filename>
+     <emphasis><filename>mktemp</filename>:</emphasis> Became obsolete. The <filename>mktemp</filename> command is provided by both <filename>busybox</filename> and <filename>coreutils</filename>.
+     <emphasis><filename>ossp-uuid</filename>:</emphasis> Is not being maintained and has mostly been replaced by <filename>uuid.h</filename> in <filename>util-linux</filename>.
+     <emphasis><filename>pax-utils</filename>:</emphasis> No longer needed.  Previous QA tests that did use this recipe are now done at build time.
+     <emphasis><filename>pcmciautils</filename>:</emphasis> Became obsolete.
+     <emphasis><filename>pixz</filename>:</emphasis> No longer needed. <filename>xz</filename> now supports multi-threaded compression.
+     <emphasis><filename>presentproto</filename>:</emphasis> Replaced by <filename>xorgproto</filename>.
+     <emphasis><filename>randrproto</filename>:</emphasis> Replaced by <filename>xorgproto</filename>.
+     <emphasis><filename>recordproto</filename>:</emphasis> Replaced by <filename>xorgproto</filename>.
+     <emphasis><filename>renderproto</filename>:</emphasis> Replaced by <filename>xorgproto</filename>.
+     <emphasis><filename>resourceproto</filename>:</emphasis> Replaced by <filename>xorgproto</filename>.
+     <emphasis><filename>scrnsaverproto</filename>:</emphasis> Replaced by <filename>xorgproto</filename>.
+     <emphasis><filename>trace-cmd</filename>:</emphasis> Became obsolete.  <filename>perf</filename> replaced this recipe's functionally.
+     <emphasis><filename>videoproto</filename>:</emphasis> Replaced by <filename>xorgproto</filename>.
+     <emphasis><filename>wireless-tools</filename>:</emphasis> Became obsolete.  Superseded by <filename>iw</filename>.
+     <emphasis><filename>xcmiscproto</filename>:</emphasis> Replaced by <filename>xorgproto</filename>.
+     <emphasis><filename>xextproto</filename>:</emphasis> Replaced by <filename>xorgproto</filename>.
+     <emphasis><filename>xf86dgaproto</filename>:</emphasis> Replaced by <filename>xorgproto</filename>.
+     <emphasis><filename>xf86driproto</filename>:</emphasis> Replaced by <filename>xorgproto</filename>.
+     <emphasis><filename>xf86miscproto</filename>:</emphasis> Replaced by <filename>xorgproto</filename>.
+     <emphasis><filename>xf86-video-omapfb</filename>:</emphasis> Became obsolete.  Use kernel modesetting driver instead.
+     <emphasis><filename>xf86-video-omap</filename>:</emphasis> Became obsolete.  Use kernel modesetting driver instead.
+     <emphasis><filename>xf86vidmodeproto</filename>:</emphasis> Replaced by <filename>xorgproto</filename>.
+     <emphasis><filename>xineramaproto</filename>:</emphasis> Replaced by <filename>xorgproto</filename>.
+     <emphasis><filename>xproto</filename>:</emphasis> Replaced by <filename>xorgproto</filename>.
+     <emphasis><filename>yasm</filename>:</emphasis> No longer needed since previous usages are now satisfied by <filename>nasm</filename>.
+            </literallayout>
+        </para>
+    </section>
+
+    <section id='migration-2.6-packaging-changes'>
+        <title>Packaging Changes</title>
+
+        <para>
+            The following packaging changes have been made:
+            <itemizedlist>
+                <listitem><para>
+                    <emphasis><filename>cmake</filename>:</emphasis>
+                    <filename>cmake.m4</filename> and
+                    <filename>toolchain</filename> files have been moved to the
+                    main package.
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis><filename>iptables</filename>:</emphasis>
+                    The <filename>iptables</filename> modules have been split
+                    into separate packages.
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis><filename>alsa-lib</filename>:</emphasis>
+                    <filename>libasound</filename> is now in the main
+                    <filename>alsa-lib</filename> package instead of
+                    <filename>libasound</filename>.
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis><filename>glibc</filename>:</emphasis>
+                    <filename>libnss-db</filename> is now in its own package
+                    along with a <filename>/var/db/makedbs.sh</filename>
+                    script to update databases.
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis><filename>python</filename> and <filename>python3</filename>:</emphasis>
+                    The main package has been removed from the recipe.
+                    You must install specific packages or
+                    <filename>python-modules</filename> /
+                    <filename>python3-modules</filename> for everything.
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis><filename>systemtap</filename>:</emphasis>
+                    Moved <filename>systemtap-exporter</filename> into its own
+                    package.
+                    </para></listitem>
+            </itemizedlist>
+        </para>
+    </section>
+
+    <section id='migration-2.6-xorg-protocol-dependencies'>
+        <title>XOrg Protocol dependencies</title>
+
+        <para>
+            The "*proto" upstream repositories have been combined into one
+            "xorgproto" repository.
+            Thus, the corresponding recipes have also been combined into a
+            single <filename>xorgproto</filename> recipe.
+            Any recipes that depend upon the older <filename>*proto</filename>
+            recipes need to be changed to depend on the newer
+            <filename>xorgproto</filename> recipe instead.
+        </para>
+
+        <para>
+            For names of recipes removed because of this repository change,
+            see the
+            <link linkend="migration-2.6-removed-recipes">Removed Recipes</link>
+            section.
+        </para>
+    </section>
+
+    <section id='migration-2.6-distutils-distutils3-fetching-dependencies'>
+        <title><filename>distutils</filename> and <filename>distutils3</filename> Now Prevent Fetching Dependencies During the <filename>do_configure</filename> Task</title>
+
+        <para>
+            Previously, it was possible for Python recipes that inherited
+            the
+            <link linkend='ref-classes-distutils'><filename>distutils</filename></link>
+            and
+            <link linkend='ref-classes-distutils3'><filename>distutils3</filename></link>
+            classes to fetch code during the
+            <link linkend='ref-tasks-configure'><filename>do_configure</filename></link>
+            task to satisfy dependencies mentioned in
+            <filename>setup.py</filename> if those dependencies were not
+            provided in the sysroot (i.e. recipes providing the dependencies
+            were missing from
+            <link linkend='var-DEPENDS'><filename>DEPENDS</filename></link>).
+            <note>
+                This change affects classes beyond just the two mentioned
+                (i.e. <filename>distutils</filename> and
+                <filename>distutils3</filename>).
+                Any recipe that inherits <filename>distutils*</filename>
+                classes are affected.
+                For example, the <filename>setuptools</filename> and
+                <filename>setuptools3</filename> recipes are affected since
+                they inherit the <filename>distutils*</filename> classes.
+            </note>
+        </para>
+
+        <para>
+            Fetching these types of dependencies that are not provided in the
+            sysroot negatively affects the ability to reproduce builds.
+            This type of fetching is now explicitly disabled.
+            Consequently, any missing dependencies in Python recipes that
+            use these classes now result in an error during the
+            <filename>do_configure</filename> task.
+        </para>
+    </section>
+
+    <section id='migration-2.6-linux-yocto-configuration-audit-issues-now-correctly-reported'>
+        <title><filename>linux-yocto</filename> Configuration Audit Issues Now Correctly Reported</title>
+
+        <para>
+            Due to a bug, the kernel configuration audit functionality was
+            not writing out any resulting warnings during the build.
+            This issue is now corrected.
+            You might notice these warnings now if you have a custom kernel
+            configuration with a <filename>linux-yocto</filename> style
+            kernel recipe.
+        </para>
+    </section>
+
+    <section id='migration-2.6-image-kernel-artifact-naming-changes'>
+        <title>Image/Kernel Artifact Naming Changes</title>
+
+        <para>
+            The following changes have been made:
+            <itemizedlist>
+                <listitem><para>
+                    Name variables (e.g.
+                    <link linkend='var-IMAGE_NAME'><filename>IMAGE_NAME</filename></link>)
+                    use a new <filename>IMAGE_VERSION_SUFFIX</filename>
+                    variable instead of
+                    <link linkend='var-DATETIME'><filename>DATETIME</filename></link>.
+                    Using <filename>IMAGE_VERSION_SUFFIX</filename> allows
+                    easier and more direct changes.</para>
+
+                    <para>The <filename>IMAGE_VERSION_SUFFIX</filename>
+                    variable is set in the
+                    <filename>bitbake.conf</filename> configuration file as
+                    follows:
+                    <literallayout class='monospaced'>
+     IMAGE_VERSION_SUFFIX = "-${DATETIME}"
+                    </literallayout>
+                    </para></listitem>
+                <listitem><para>
+                    Several variables have changed names for consistency:
+                    <literallayout class='monospaced'>
+    Old Variable Name               New Variable Name
+    ========================================================
+    KERNEL_IMAGE_BASE_NAME          <link linkend='var-KERNEL_IMAGE_NAME'>KERNEL_IMAGE_NAME</link>
+    KERNEL_IMAGE_SYMLINK_NAME       <link linkend='var-KERNEL_IMAGE_LINK_NAME'>KERNEL_IMAGE_LINK_NAME</link>
+    MODULE_TARBALL_BASE_NAME        <link linkend='var-MODULE_TARBALL_NAME'>MODULE_TARBALL_NAME</link>
+    MODULE_TARBALL_SYMLINK_NAME     <link linkend='var-MODULE_TARBALL_LINK_NAME'>MODULE_TARBALL_LINK_NAME</link>
+    INITRAMFS_BASE_NAME             <link linkend='var-INITRAMFS_NAME'>INITRAMFS_NAME</link>
+                    </literallayout>
+                    </para></listitem>
+                <listitem><para>
+                    The <filename>MODULE_IMAGE_BASE_NAME</filename> variable
+                    has been removed.
+                    The module tarball name is now controlled directly with the
+                    <link linkend='var-MODULE_TARBALL_NAME'><filename>MODULE_TARBALL_NAME</filename></link>
+                    variable.
+                    </para></listitem>
+                <listitem><para>
+                    The
+                    <link linkend='var-KERNEL_DTB_NAME'><filename>KERNEL_DTB_NAME</filename></link>
+                    and
+                    <link linkend='var-KERNEL_DTB_LINK_NAME'><filename>KERNEL_DTB_LINK_NAME</filename></link>
+                    variables have been introduced to control kernel Device
+                    Tree Binary (DTB) artifact names instead of mangling
+                    <filename>KERNEL_IMAGE_*</filename> variables.
+                    </para></listitem>
+                <listitem><para>
+                    The
+                    <link linkend='var-KERNEL_FIT_NAME'><filename>KERNEL_FIT_NAME</filename></link>
+                    and
+                    <link linkend='var-KERNEL_FIT_LINK_NAME'><filename>KERNEL_FIT_LINK_NAME</filename></link>
+                    variables have been introduced to specify the name of
+                    flattened image tree (FIT) kernel images similar to other
+                    deployed artifacts.
+                    </para></listitem>
+                <listitem><para>
+                    The
+                    <link linkend='var-MODULE_TARBALL_NAME'><filename>MODULE_TARBALL_NAME</filename></link>
+                    and
+                    <link linkend='var-MODULE_TARBALL_LINK_NAME'><filename>MODULE_TARBALL_LINK_NAME</filename></link>
+                    variable values no longer include the "module-" prefix or
+                    ".tgz" suffix.
+                    These parts are now hardcoded so that the values are
+                    consistent with other artifact naming variables.
+                    </para></listitem>
+                <listitem><para>
+                    Added the
+                    <link linkend='var-INITRAMFS_LINK_NAME'><filename>INITRAMFS_LINK_NAME</filename></link>
+                    variable so that the symlink can be controlled similarly
+                    to other artifact types.
+                    </para></listitem>
+                <listitem><para>
+                    <link linkend='var-INITRAMFS_NAME'><filename>INITRAMFS_NAME</filename></link>
+                    now uses
+                    "${PKGE}-${PKGV}-${PKGR}-${MACHINE}${IMAGE_VERSION_SUFFIX}"
+                    instead of
+                    "${PV}-${PR}-${MACHINE}-${DATETIME}", which
+                    makes it consistent with other variables.
+                    </para></listitem>
+            </itemizedlist>
+        </para>
+    </section>
+
+    <section id='migration-2.6-serial-console-deprecated'>
+        <title><filename>SERIAL_CONSOLE</filename> Deprecated</title>
+
+        <para>
+            The
+            <link linkend='var-SERIAL_CONSOLE'><filename>SERIAL_CONSOLE</filename></link>
+            variable has been functionally replaced by the
+            <link linkend='var-SERIAL_CONSOLES'><filename>SERIAL_CONSOLES</filename></link>
+            variable for some time.
+            With the Yocto Project 2.6 release,
+            <filename>SERIAL_CONSOLE</filename> has been officially deprecated.
+        </para>
+
+        <para>
+            <filename>SERIAL_CONSOLE</filename> will continue to work as
+            before for the 2.6 release.
+            However, for the sake of future compatibility, it is recommended
+            that you replace all instances of
+            <filename>SERIAL_CONSOLE</filename> with
+            <filename>SERIAL_CONSOLES</filename>.
+            <note>
+                The only difference in usage is that
+                <filename>SERIAL_CONSOLES</filename> expects entries to be
+                separated using semicolons as compared to
+                <filename>SERIAL_CONSOLE</filename>, which expects spaces.
+            </note>
+        </para>
+    </section>
+
+    <section id='migration-2.6-poky-sets-unknown-configure-option-to-qa-error'>
+        <title>Configure Script Reports Unknown Options as Errors</title>
+
+        <para>
+            If the configure script reports an unknown option, this now
+            triggers a QA error instead of a warning.
+            Any recipes that previously got away with specifying such unknown
+            options now need to be fixed.
+        </para>
+    </section>
+
+    <section id='migration-2.6-override-changes'>
+        <title>Override Changes</title>
+
+        <para>
+            The following changes have occurred:
+            <itemizedlist>
+                <listitem><para>
+                    <emphasis>The <filename>virtclass-native</filename> and
+                    <filename>virtclass-nativesdk</filename> Overrides Have
+                    Been Removed:</emphasis>
+                    The <filename>virtclass-native</filename> and
+                    <filename>virtclass-nativesdk</filename> overrides have
+                    been deprecated since 2012 in favor of
+                    <filename>class-native</filename> and
+                    <filename>class-nativesdk</filename>, respectively.
+                    Both <filename>virtclass-native</filename> and
+                    <filename>virtclass-nativesdk</filename> are now dropped.
+                    <note>
+                        The <filename>virtclass-multilib-</filename> overrides
+                        for multilib are still valid.
+                    </note>
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis>The <filename>forcevariable</filename>
+                    Override Now Has a Higher Priority Than
+                    <filename>libc</filename> Overrides:</emphasis>
+                    The <filename>forcevariable</filename> override is
+                    documented to be the highest priority override.
+                    However, due to a long-standing quirk of how
+                    <link linkend='var-OVERRIDES'><filename>OVERRIDES</filename></link>
+                    is set, the <filename>libc</filename> overrides (e.g.
+                    <filename>libc-glibc</filename>,
+                    <filename>libc-musl</filename>, and so forth) erroneously
+                    had a higher priority.
+                    This issue is now corrected.</para>
+
+                    <para>It is likely this change will not cause any
+                    problems.
+                    However, it is possible with some unusual configurations
+                    that you might see a change in behavior if you were
+                    relying on the previous behavior.
+                    Be sure to check how you use
+                    <filename>forcevariable</filename> and
+                    <filename>libc-*</filename> overrides in your custom
+                    layers and configuration files to ensure they make sense.
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis>The <filename>build-${BUILD_OS}</filename>
+                    Override Has Been Removed:</emphasis>
+                    The <filename>build-${BUILD_OS}</filename>, which is
+                    typically <filename>build-linux</filename>, override has
+                    been removed because building on a host operating system
+                    other than a recent version of Linux is neither supported
+                    nor recommended.
+                    Dropping the override avoids giving the impression that
+                    other host operating systems might be supported.
+                    </para></listitem>
+                <listitem><para>
+                    The "_remove" operator now preserves whitespace.
+                    Consequently, when specifying list items to remove, be
+                    aware that leading and trailing whitespace resulting from
+                    the removal is retained.</para>
+
+                    <para>See the
+                    "<ulink url='&YOCTO_DOCS_BB_URL;#removing-override-style-syntax'>Removal (Override Style Syntax)</ulink>"
+                    section in the BitBake User Manual for a detailed example.
+                    </para></listitem>
+            </itemizedlist>
+        </para>
+    </section>
+
+    <section id='migration-2.6-systemd-configuration-now-split-out-to-system-conf'>
+        <title><filename>systemd</filename> Configuration is Now Split Into <filename>systemd-conf</filename></title>
+
+        <para>
+            The configuration for the <filename>systemd</filename> recipe
+            has been moved into a <filename>system-conf</filename> recipe.
+            Moving this configuration to a separate recipe avoids the
+            <filename>systemd</filename> recipe from becoming machine-specific
+            for cases where machine-specific configurations need to be applied
+            (e.g. for <filename>qemu*</filename> machines).
+        </para>
+
+        <para>
+            Currently, the new recipe packages the following files:
+            <literallayout class='monospaced'>
+     ${sysconfdir}/machine-id
+     ${sysconfdir}/systemd/coredump.conf
+     ${sysconfdir}/systemd/journald.conf
+     ${sysconfdir}/systemd/logind.conf
+     ${sysconfdir}/systemd/system.conf
+     ${sysconfdir}/systemd/user.conf
+            </literallayout>
+            If you previously used bbappend files to append the
+            <filename>systemd</filename> recipe to change any of the
+            listed files, you must do so for the
+            <filename>systemd-conf</filename> recipe instead.
+        </para>
+    </section>
+
+    <section id='migration-2.6-automatic-testing-changes'>
+        <title>Automatic Testing Changes</title>
+
+        <para>
+            This section provides information about automatic testing
+            changes:
+            <itemizedlist>
+                <listitem><para>
+                    <emphasis><filename>TEST_IMAGE</filename> Variable Removed:</emphasis>
+                    Prior to this release, you set the
+                    <filename>TEST_IMAGE</filename> variable to "1" to
+                    enable automatic testing for successfully built images.
+                    The <filename>TEST_IMAGE</filename> variable no longer
+                    exists and has been replaced by the
+                    <link linkend='var-TESTIMAGE_AUTO'><filename>TESTIMAGE_AUTO</filename></link>
+                    variable.
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis>Inheriting the <filename>testimage</filename> and
+                    <filename>testsdk</filename> Classes:</emphasis>
+                    Best practices now dictate that you use the
+                    <link linkend='var-IMAGE_CLASSES'><filename>IMAGE_CLASSES</filename></link>
+                    variable rather than the
+                    <link linkend='var-INHERIT'><filename>INHERIT</filename></link>
+                    variable when you inherit the
+                    <link linkend='ref-classes-testimage*'><filename>testimage</filename></link>
+                    and
+                    <link linkend='ref-classes-testsdk'><filename>testsdk</filename></link>
+                    classes used for automatic testing.
+                    </para></listitem>
+            </itemizedlist>
+        </para>
+    </section>
+
+    <section id='migration-2.6-openssl-changes'>
+        <title>OpenSSL Changes</title>
+
+        <para>
+            <ulink url='https://www.openssl.org/'>OpenSSL</ulink> has been
+            upgraded from 1.0 to 1.1.
+            By default, this upgrade could cause problems for recipes that
+            have both versions in their dependency chains.
+            The problem is that both versions cannot be installed together
+            at build time.
+            <note>
+                It is possible to have both versions of the library at runtime.
+            </note>
+        </para>
+    </section>
+
+    <section id='migration-2.6-bitbake-changes'>
+        <title>BitBake Changes</title>
+
+        <para>
+            The server logfile <filename>bitbake-cookerdaemon.log</filename> is
+            now always placed in the
+            <link linkend='build-directory'>Build Directory</link>
+            instead of the current directory.
+        </para>
+    </section>
+
+    <section id='migration-2.6-security-changes'>
+        <title>Security Changes</title>
+
+        <para>
+            The Poky distribution now uses security compiler flags by
+            default.
+            Inclusion of these flags could cause new failures due to stricter
+            checking for various potential security issues in code.
+        </para>
+    </section>
+
+    <section id='migration-2.6-post-installation-changes'>
+        <title>Post Installation Changes</title>
+
+        <para>
+            You must explicitly mark post installs to defer to the target.
+            If you want to explicitly defer a postinstall to first boot on
+            the target rather than at rootfs creation time, use
+            <filename>pkg_postinst_ontarget()</filename> or call
+            <filename>postinst-intercepts defer_to_first_boot</filename> from
+            <filename>pkg_postinst()</filename>.
+            Any failure of a <filename>pkg_postinst()</filename> script
+            (including exit 1) triggers an error during the
+            <link linkend='ref-tasks-rootfs'><filename>do_rootfs</filename></link> task.
+        </para>
+
+        <para>
+            For more information on post-installation behavior, see the
+            "<ulink url='&YOCTO_DOCS_DEV_URL;#new-recipe-post-installation-scripts'>Post-Installation Scripts</ulink>"
+            section in the Yocto Project Development Tasks Manual.
+        </para>
+    </section>
+
+    <section id='migration-2.6-python-3-profile-guided-optimizations'>
+        <title>Python 3 Profile-Guided Optimization</title>
+
+        <para>
+            The <filename>python3</filename> recipe now enables profile-guided
+            optimization.
+            Using this optimization requires a little extra build time in
+            exchange for improved performance on the target at runtime.
+            Additionally, the optimization is only enabled if the current
+            <link linkend='var-MACHINE'><filename>MACHINE</filename></link>
+            has support for user-mode emulation in QEMU (i.e. "qemu-usermode"
+            is in
+            <link linkend='var-MACHINE_FEATURES'><filename>MACHINE_FEATURES</filename></link>,
+            which it is by default).
+        </para>
+
+        <para>
+            If you wish to disable Python profile-guided optimization
+            regardless of the value of
+            <filename>MACHINE_FEATURES</filename>, then ensure that
+            <link linkend='var-PACKAGECONFIG'><filename>PACKAGECONFIG</filename></link>
+            for the <filename>python3</filename> recipe does not contain "pgo".
+            You could accomplish the latter using the following at the
+            configuration level:
+            <literallayout class='monospaced'>
+     PACKAGECONFIG_remove_pn-python3 = "pgo"
+            </literallayout>
+            Alternatively, you can set
+            <filename>PACKAGECONFIG</filename> using an append file for the
+            <filename>python3</filename> recipe.
+        </para>
+    </section>
+
+    <section id='migration-2.6-miscellaneous-changes'>
+        <title>Miscellaneous Changes</title>
+
+        <para>
+            The following miscellaneous changes occurred:
+            <itemizedlist>
+                <listitem><para>
+                    Default to using the Thumb-2 instruction set for armv7a
+                    and above.
+                    If you have any custom recipes that build software that
+                    needs to be built with the ARM instruction set, change the
+                    recipe to set the instruction set as follows:
+                    <literallayout class='monospaced'>
+     ARM_INSTRUCTION_SET = "arm"
+                    </literallayout>
+                    </para></listitem>
+                <listitem><para>
+                    <filename>run-postinsts</filename> no longer uses
+                    <filename>/etc/*-postinsts</filename> for
+                    <filename>dpkg/opkg</filename> in favor of built-in
+                    postinst support.
+                    RPM behavior remains unchanged.
+                    </para></listitem>
+                <listitem><para>
+                    The <filename>NOISO</filename> and
+                    <filename>NOHDD</filename> variables are no longer used.
+                    You now control building <filename>*.iso</filename> and
+                    <filename>*.hddimg</filename> image types directly
+                    by using the
+                    <link linkend='var-IMAGE_FSTYPES'><filename>IMAGE_FSTYPES</filename></link>
+                    variable.
+                    </para></listitem>
+                <listitem><para>
+                    The <filename>scripts/contrib/mkefidisk.sh</filename>
+                    has been removed in favor of Wic.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>kernel-modules</filename> has been removed from
+                    <link linkend='var-RRECOMMENDS'><filename>RRECOMMENDS</filename></link>
+                    for <filename>qemumips</filename> and
+                    <filename>qemumips64</filename> machines.
+                    Removal also impacts the <filename>x86-base.inc</filename>
+                    file.
+                    <note>
+                        <filename>genericx86</filename> and
+                        <filename>genericx86-64</filename> retain
+                        <filename>kernel-modules</filename> as part of the
+                        <filename>RRECOMMENDS</filename> variable setting.
+                    </note>
+                    </para></listitem>
+                <listitem><para>
+                    The <filename>LGPLv2_WHITELIST_GPL-3.0</filename>
+                    variable has been removed.
+                    If you are setting this variable in your configuration,
+                    set or append it to the
+                    <filename>WHITELIST_GPL-3.0</filename> variable instead.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>${ASNEEDED}</filename> is now included in
+                    the
+                    <link linkend='var-TARGET_LDFLAGS'><filename>TARGET_LDFLAGS</filename></link>
+                    variable directly.
+                    The remaining definitions from
+                    <filename>meta/conf/distro/include/as-needed.inc</filename>
+                    have been moved to corresponding recipes.
+                    </para></listitem>
+                <listitem><para>
+                    Support for DSA host keys has been dropped from the
+                    OpenSSH recipes.
+                    If you are still using DSA keys, you must switch over to a
+                    more secure algorithm as recommended by OpenSSH upstream.
+                    </para></listitem>
+                <listitem><para>
+                    The <filename>dhcp</filename> recipe now uses the
+                    <filename>dhcpd6.conf</filename> configuration file in
+                    <filename>dhcpd6.service</filename> for IPv6 DHCP rather
+                    than re-using <filename>dhcpd.conf</filename>, which is
+                    now reserved for IPv4.
+                    </para></listitem>
+            </itemizedlist>
+        </para>
+    </section>
+</section>
 </chapter>
 <!--
 vim: expandtab tw=80 ts=4

documentation/ref-manual/ref-classes.xml

@@ -449,12 +449,13 @@
     <title><filename>cmake.bbclass</filename></title>
 
     <para>
-        The <filename>cmake</filename> class allows for
-        recipes that need to build software using the CMake build system.
+        The <filename>cmake</filename> class allows for recipes that need to
+        build software using the
+        <ulink url='https://cmake.org/overview/'>CMake</ulink> build system.
         You can use the
         <link linkend='var-EXTRA_OECMAKE'><filename>EXTRA_OECMAKE</filename></link>
-        variable to specify additional configuration options to be passed on
-        the <filename>cmake</filename> command line.
+        variable to specify additional configuration options to be passed
+        using the <filename>cmake</filename> command line.
     </para>
 </section>
 
@@ -645,6 +646,54 @@
     </para>
 </section>
 
+<section id='ref-classes-devupstream'>
+    <title><filename>devupstream.bbclass</filename></title>
+
+    <para>
+        The <filename>devupstream</filename> class uses
+        <link linkend='var-BBCLASSEXTEND'><filename>BBCLASSEXTEND</filename></link>
+        to add a variant of the recipe that fetches from an alternative URI
+        (e.g. Git) instead of a tarball.
+        Following is an example:
+        <literallayout class='monospaced'>
+     BBCLASSEXTEND = "devupstream:target"
+     SRC_URI_class-devupstream = "git://git.example.com/example"
+     SRCREV_class-devupstream = "abcd1234"
+        </literallayout>
+        Adding the above statements to your recipe creates a variant that has
+        <link linkend='var-DEFAULT_PREFERENCE'><filename>DEFAULT_PREFERENCE</filename></link>
+        set to "-1".
+        Consequently, you need to select the variant of the recipe to use it.
+        Any development-specific adjustments can be done by using the
+        <filename>class-devupstream</filename> override.
+        Here is an example:
+        <literallayout class='monospaced'>
+     DEPENDS_append_class-devupstream = " gperf-native"
+
+     do_configure_prepend_class-devupstream() {
+        touch ${S}/README
+     }
+        </literallayout>
+        The class currently only supports creating a development variant of
+        the target recipe, not <filename>native</filename> or
+        <filename>nativesdk</filename> variants.
+    </para>
+
+    <para>
+        The <filename>BBCLASSEXTEND</filename> syntax
+        (i.e. <filename>devupstream:target</filename>) provides support for
+        <filename>native</filename> and <filename>nativesdk</filename>
+        variants.
+        Consequently, this functionality can be added in a future release.
+    </para>
+
+    <para>
+        Support for other version control systems such as Subversion is
+        limited due to BitBake's automatic fetch dependencies (e.g.
+        <filename>subversion-native</filename>).
+    </para>
+</section>
+
 <section id='ref-classes-distro_features_check'>
     <title><filename>distro_features_check.bbclass</filename></title>
 
@@ -1266,28 +1315,35 @@
     <title><filename>image_types.bbclass</filename></title>
 
     <para>
-        The <filename>image_types</filename> class defines all of
-        the standard image output types that you can enable through the
+        The <filename>image_types</filename> class defines all of the
+        standard image output types that you can enable through the
         <link linkend='var-IMAGE_FSTYPES'><filename>IMAGE_FSTYPES</filename></link>
         variable.
-        You can use this class as a reference on how to add support for custom
-        image output types.
+        You can use this class as a reference on how to add support for
+        custom image output types.
     </para>
 
     <para>
-        By default, this class is enabled through the
-        <link linkend='var-IMAGE_CLASSES'><filename>IMAGE_CLASSES</filename></link>
-        variable in
-        <link linkend='ref-classes-image'><filename>image.bbclass</filename></link>.
-        If you define your own image types using a custom BitBake class and
-        then use <filename>IMAGE_CLASSES</filename> to enable it, the custom
-        class must either inherit <filename>image_types</filename> or
-        <filename>image_types</filename> must also appear in
-        <filename>IMAGE_CLASSES</filename>.
+        By default, the
+        <link linkend='ref-classes-image'><filename>image</filename></link>
+        class automatically enables the <filename>image_types</filename> class.
+        The <filename>image</filename> class uses the
+        <filename>IMGCLASSES</filename> variable as follows:
+        <literallayout class='monospaced'>
+     IMGCLASSES = "rootfs_${IMAGE_PKGTYPE} image_types ${IMAGE_CLASSES}"
+     IMGCLASSES += "${@['populate_sdk_base', 'populate_sdk_ext']['linux' in d.getVar("SDK_OS")]}"
+     IMGCLASSES += "${@bb.utils.contains_any('IMAGE_FSTYPES', 'live iso hddimg', 'image-live', '', d)}"
+     IMGCLASSES += "${@bb.utils.contains('IMAGE_FSTYPES', 'container', 'image-container', '', d)}"
+     IMGCLASSES += "image_types_wic"
+     IMGCLASSES += "rootfs-postcommands"
+     IMGCLASSES += "image-postinst-intercepts"
+     inherit ${IMGCLASSES}
+        </literallayout>
     </para>
 
     <para>
-        This class also handles conversion and compression of images.
+        The <filename>image_types</filename> class also handles conversion and
+        compression of images.
         <note>
             To build a VMware VMDK image, you need to add "wic.vmdk" to
             <filename>IMAGE_FSTYPES</filename>.
@@ -1314,14 +1370,6 @@
         Normally, you do not use this class directly.
         Instead, you add "live" to
         <link linkend='var-IMAGE_FSTYPES'><filename>IMAGE_FSTYPES</filename></link>.
-        You can selectively build just one of these types through the
-        <link linkend='var-NOISO'><filename>NOISO</filename></link>
-        and
-        <link linkend='var-NOHDD'><filename>NOHDD</filename></link> variables.
-        For example, if you were building an ISO image, you would add "live"
-        to <filename>IMAGE_FSTYPES</filename>, set the
-        <filename>NOISO</filename> variable to "0" and the build system would
-        use the <filename>image-live</filename> class to build the ISO image.
     </para>
 </section>
 
@@ -3502,6 +3550,14 @@ This check was removed for YP 2.3 release
         The classes handle loading the tests and starting the image.
         To use the classes, you need to perform steps to set up the
         environment.
+        <note><title>Tip</title>
+            Best practices include using
+            <link linkend='var-IMAGE_CLASSES'><filename>IMAGE_CLASSES</filename></link>
+            rather than
+            <link linkend='var-INHERIT'><filename>INHERIT</filename></link> to
+            inherit the <filename>testimage</filename> class for automated
+            image testing.
+        </note>
     </para>
 
     <para>
@@ -3519,7 +3575,7 @@ This check was removed for YP 2.3 release
         </literallayout>
         The <filename>testimage-auto</filename> class runs tests on an image
         after the image is constructed (i.e.
-        <link linkend='var-TEST_IMAGE'><filename>TEST_IMAGE</filename></link>
+        <link linkend='var-TESTIMAGE_AUTO'><filename>TESTIMAGE_AUTO</filename></link>
         must be set to "1").
     </para>
 
@@ -3541,6 +3597,14 @@ This check was removed for YP 2.3 release
         <literallayout class='monospaced'>
      $ bitbake -c testsdk image
         </literallayout>
+        <note><title>Tip</title>
+            Best practices include using
+            <link linkend='var-IMAGE_CLASSES'><filename>IMAGE_CLASSES</filename></link>
+            rather than
+            <link linkend='var-INHERIT'><filename>INHERIT</filename></link> to
+            inherit the <filename>testsdk</filename> class for automated
+            SDK testing.
+        </note>
     </para>
 </section>
 

documentation/ref-manual/ref-manual.xml

@@ -124,9 +124,14 @@
             </revision>
             <revision>
                 <revnumber>2.6</revnumber>
-                <date>&REL_MONTH_YEAR;</date>
+                <date>November 2018</date>
                 <revremark>Released with the Yocto Project 2.6 Release.</revremark>
             </revision>
+            <revision>
+                <revnumber>2.7</revnumber>
+                <date>&REL_MONTH_YEAR;</date>
+                <revremark>Released with the Yocto Project 2.7 Release.</revremark>
+            </revision>
         </revhistory>
 
     <copyright>

documentation/ref-manual/ref-system-requirements.xml

@@ -65,6 +65,14 @@
                         that our priority is on the supported platforms listed
                         below.
                         </para></listitem>
+                    <listitem><para>
+                        The Yocto Project is not compatible with the
+                        <ulink url='https://en.wikipedia.org/wiki/Windows_Subsystem_for_Linux'>Windows Subsystem for Linux</ulink>
+                        (WSL).
+                        You cannot use a
+                        <link linkend='hardware-build-system-term'>build host</link>
+                        that is running WSL.
+                        </para></listitem>
                     <listitem><para>
                         If you encounter problems, please go to
                         <ulink url='&YOCTO_BUGZILLA_URL;'>Yocto Project Bugzilla</ulink>
@@ -80,46 +88,49 @@
                 </itemizedlist>
             </note>
             <itemizedlist>
-<!--
-                <listitem><para>Ubuntu 10.04</para></listitem>
+<!--            <listitem><para>Ubuntu 10.04</para></listitem>
                 <listitem><para>Ubuntu 11.10</para></listitem>
                 <listitem><para>Ubuntu 12.04 (LTS)</para></listitem>
                 <listitem><para>Ubuntu 13.10</para></listitem>
-                <listitem><para>Ubuntu 14.04 (LTS)</para></listitem> -->
+                <listitem><para>Ubuntu 14.04 (LTS)</para></listitem>
                 <listitem><para>Ubuntu 14.10</para></listitem>
                 <listitem><para>Ubuntu 15.04</para></listitem>
-                <listitem><para>Ubuntu 15.10</para></listitem>
+                <listitem><para>Ubuntu 15.10</para></listitem> -->
                 <listitem><para>Ubuntu 16.04 (LTS)</para></listitem>
-<!--                <listitem><para>Fedora 16 (Verne)</para></listitem>
+                <listitem><para>Ubuntu 16.10 (LTS)</para></listitem>
+                <listitem><para>Ubuntu 17.04</para></listitem>
+<!--            <listitem><para>Fedora 16 (Verne)</para></listitem>
                 <listitem><para>Fedora 17 (Spherical)</para></listitem>
-                <listitem><para>Fedora release 19 (Schrödinger's Cat)</para></listitem>
-                <listitem><para>Fedora release 20 (Heisenbug)</para></listitem> -->
+                <listitem><para>Fedora 19 (Schrödinger's Cat)</para></listitem>
+                <listitem><para>Fedora release 20 (Heisenbug)</para></listitem>
                 <listitem><para>Fedora release 22</para></listitem>
                 <listitem><para>Fedora release 23</para></listitem>
-<!--                <listitem><para>Fedora release 24</para></listitem>
-                <listitem><para>CentOS release 5.6 (Final)</para></listitem>
+                <listitem><para>Fedora release 24</para></listitem> -->
+                <listitem><para>Fedora release 26</para></listitem>
+<!--            <listitem><para>CentOS release 5.6 (Final)</para></listitem>
                 <listitem><para>CentOS release 5.7 (Final)</para></listitem>
                 <listitem><para>CentOS release 5.8 (Final)</para></listitem>
                 <listitem><para>CentOS release 6.3 (Final)</para></listitem>
                 <listitem><para>CentOS release 6.x</para></listitem> -->
                 <listitem><para>CentOS release 7.x</para></listitem>
-<!--                <listitem><para>Debian GNU/Linux 6.0 (Squeeze)</para></listitem>
+<!--            <listitem><para>Debian GNU/Linux 6.0 (Squeeze)</para></listitem>
                 <listitem><para>Debian GNU/Linux 7.x (Wheezy)</para></listitem> -->
                 <listitem><para>Debian GNU/Linux 8.x (Jessie)</para></listitem>
                 <listitem><para>Debian GNU/Linux 9.x (Stretch)</para></listitem>
-<!--                <listitem><para>Debian GNU/Linux 7.1 (Wheezy)</para></listitem>
+<!--            <listitem><para>Debian GNU/Linux 7.1 (Wheezy)</para></listitem>
                 <listitem><para>Debian GNU/Linux 7.2 (Wheezy)</para></listitem>
                 <listitem><para>Debian GNU/Linux 7.3 (Wheezy)</para></listitem>
                 <listitem><para>Debian GNU/Linux 7.4 (Wheezy)</para></listitem>
                 <listitem><para>Debian GNU/Linux 7.5 (Wheezy)</para></listitem>
-                <listitem><para>Debian GNU/Linux 7.6 (Wheezy)</para></listitem> -->
-<!--                <listitem><para>openSUSE 11.4</para></listitem>
+                <listitem><para>Debian GNU/Linux 7.6 (Wheezy)</para></listitem>
+                <listitem><para>openSUSE 11.4</para></listitem>
                 <listitem><para>openSUSE 12.1</para></listitem>
                 <listitem><para>openSUSE 12.2</para></listitem>
                 <listitem><para>openSUSE 12.3</para></listitem>
-                <listitem><para>openSUSE 13.1</para></listitem> -->
-                <listitem><para>openSUSE 13.2</para></listitem>
+                <listitem><para>openSUSE 13.1</para></listitem>
+                <listitem><para>openSUSE 13.2</para></listitem> -->
                 <listitem><para>openSUSE 42.1</para></listitem>
+                <listitem><para>openSUSE 42.2</para></listitem>
             </itemizedlist>
         </para>
 

documentation/ref-manual/ref-tasks.xml

@@ -886,7 +886,7 @@
             Boots an image and performs runtime tests within the image
             immediately after it has been built.
             This task is enabled when you set
-            <link linkend='var-TEST_IMAGE'><filename>TEST_IMAGE</filename></link>
+            <link linkend='var-TESTIMAGE_AUTO'><filename>TESTIMAGE_AUTO</filename></link>
             equal to "1".
         </para>
 

documentation/ref-manual/ref-variables.xml

@@ -680,11 +680,18 @@
      BB_ALLOWED_NETWORKS = "*.gnu.org"
                             </literallayout>
                             <note><title>Important</title>
-                                The use of the "<filename>*</filename>"
+                                <para>The use of the "<filename>*</filename>"
                                 character only works at the beginning of
-                                a host name.
+                                a host name and it must be isolated from
+                                the remainder of the host name.
                                 You cannot use the wildcard character in any
-                                other location of the name.
+                                other location of the name or combined with
+                                the front part of the name.</para>
+
+                                <para>For example,
+                                <filename>*.foo.bar</filename> is supported,
+                                while <filename>*aa.foo.bar</filename> is not.
+                                </para>
                             </note>
                             </para></listitem>
                         <listitem><para>
@@ -1152,7 +1159,7 @@
                 <para>
                     When specifying recipe files, you can pattern match using
                     Python's
-                    <ulink url='https://docs.python.org/2/library/glob.html'><filename>glob</filename></ulink>
+                    <ulink url='https://docs.python.org/3/library/glob.html'><filename>glob</filename></ulink>
                     syntax.
                     For details on the syntax, see the documentation by
                     following the previous link.
@@ -1296,7 +1303,7 @@
                     the files.
                     For complete syntax information, see Python's
                     documentation at
-                    <ulink url='http://docs.python.org/release/2.3/lib/re-syntax.html'></ulink>.
+                    <ulink url='http://docs.python.org/3/library/re.html#re'></ulink>.
                 </para>
 
                 <para>
@@ -4626,7 +4633,12 @@
             <glossdef>
                 <para role="glossdeffirst">
 <!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
-                    Additional <filename>cmake</filename> options.
+                    Additional
+                    <ulink url='https://cmake.org/overview/'>CMake</ulink>
+                    options.
+                    See the
+                    <link linkend='ref-classes-cmake'><filename>cmake</filename></link>
+                    class for additional information.
                 </para>
             </glossdef>
         </glossentry>
@@ -5221,6 +5233,28 @@
             </glossdef>
         </glossentry>
 
+        <glossentry id='var-GCCVERSION'><glossterm>GCCVERSION</glossterm>
+            <info>
+                GCCVERSION[doc] = "Specifies the default version of the GNU C Compiler (GCC) to use."
+            </info>
+            <glossdef>
+                <para role="glossdeffirst">
+<!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
+                    Specifies the default version of the GNU C Compiler (GCC)
+                    used for compilation.
+                    By default, <filename>GCCVERSION</filename> is set to
+                    "8.x" in the
+                    <filename>meta/conf/distro/include/tcmode-default.inc</filename>
+                    include file:
+                    <literallayout class='monospaced'>
+     GCCVERSION ?= "8.%"
+                    </literallayout>
+                    You can override this value by setting it in a configuration
+                    file such as the <filename>local.conf</filename>.
+                </para>
+            </glossdef>
+        </glossentry>
+
         <glossentry id='var-GDB'><glossterm>GDB</glossterm>
             <info>
                 GDB[doc] = "The minimal command and arguments to run the GNU Debugger."
@@ -6995,6 +7029,61 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
             </glossdef>
         </glossentry>
 
+        <glossentry id='var-INITRAMFS_LINK_NAME'><glossterm>INITRAMFS_LINK_NAME</glossterm>
+            <info>
+                INITRAMFS_LINK_NAME[doc] = "The link name of the initial RAM filesystem image."
+            </info>
+            <glossdef>
+                <para role="glossdeffirst">
+<!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
+                    The link name of the initial RAM filesystem image.
+                    This variable is set in the
+                    <filename>meta/classes/kernel-artifact-names.bbclass</filename>
+                    file as follows:
+                    <literallayout class='monospaced'>
+     INITRAMFS_LINK_NAME ?= "initramfs-${KERNEL_ARTIFACT_LINK_NAME}"
+                    </literallayout>
+                    The value of the <filename>KERNEL_ARTIFACT_LINK_NAME</filename>
+                    variable, which is set in the same file, has the following
+                    value:
+                    <literallayout class='monospaced'>
+     KERNEL_ARTIFACT_LINK_NAME ?= "${MACHINE}"
+                    </literallayout>
+                </para>
+
+                <para>
+                    See the
+                    <link linkend='var-MACHINE'><filename>MACHINE</filename></link>
+                    variable for additional information.
+                </para>
+            </glossdef>
+        </glossentry>
+
+        <glossentry id='var-INITRAMFS_NAME'><glossterm>INITRAMFS_NAME</glossterm>
+            <info>
+                INITRAMFS_NAME[doc] = "The base name of the initial RAM filesystem image."
+            </info>
+            <glossdef>
+                <para role="glossdeffirst">
+<!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
+                    The base name of the initial RAM filesystem image.
+                    This variable is set in the
+                    <filename>meta/classes/kernel-artifact-names.bbclass</filename>
+                    file as follows:
+                    <literallayout class='monospaced'>
+     INITRAMFS_NAME ?= "initramfs-${KERNEL_ARTIFACT_NAME}"
+                    </literallayout>
+                    The value of the
+                    <link linkend='var-KERNEL_ARTIFACT_NAME'><filename>KERNEL_ARTIFACT_NAME</filename></link>
+                    variable, which is set in the same file, has the following
+                    value:
+                    <literallayout class='monospaced'>
+     KERNEL_ARTIFACT_NAME ?= "${PKGE}-${PKGV}-${PKGR}-${MACHINE}${IMAGE_VERSION_SUFFIX}"
+                    </literallayout>
+                </para>
+            </glossdef>
+        </glossentry>
+
         <glossentry id='var-INITRD'><glossterm>INITRD</glossterm>
             <info>
                 INITRD[doc] = "Indicates a list of filesystem images to concatenate and use as an initial RAM disk (initrd)."
@@ -7371,6 +7460,45 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
             </glossdef>
         </glossentry>
 
+        <glossentry id='var-KERNEL_ARTIFACT_NAME'><glossterm>KERNEL_ARTIFACT_NAME</glossterm>
+            <info>
+                KERNEL_ARTIFACT_NAME[doc] = "Specifies the name of all of the build artifacts."
+            </info>
+            <glossdef>
+                <para role="glossdeffirst">
+<!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
+                    Specifies the name of all of the build artifacts.
+                    You can change the name of the artifacts by changing the
+                    <filename>KERNEL_ARTIFACT_NAME</filename> variable.
+                </para>
+
+                <para>
+                    The value of <filename>KERNEL_ARTIFACT_NAME</filename>,
+                    which is set in the
+                    <filename> meta/classes/kernel-artifact-names.bbclass</filename>
+                    file, has the following default value:
+                    <literallayout class='monospaced'>
+     KERNEL_ARTIFACT_NAME ?= "${PKGE}-${PKGV}-${PKGR}-${MACHINE}${IMAGE_VERSION_SUFFIX}"
+                    </literallayout>
+                </para>
+
+                <para>
+                    See the
+                    <link linkend='var-PKGE'><filename>PKGE</filename></link>,
+                    <link linkend='var-PKGV'><filename>PKGV</filename></link>,
+                    <link linkend='var-PKGR'><filename>PKGR</filename></link>,
+                    and
+                    <link linkend='var-MACHINE'><filename>MACHINE</filename></link>
+                    variables for additional information.
+                    <note>
+                        The <filename>IMAGE_VERSION_SUFFIX</filename> variable
+                        is set to
+                        <link linkend='var-DATETIME'><filename>DATETIME</filename></link>.
+                    </note>
+                </para>
+            </glossdef>
+        </glossentry>
+
         <glossentry id='var-KERNEL_CLASSES'><glossterm>KERNEL_CLASSES</glossterm>
             <info>
                 KERNEL_CLASSES[doc] = "A list of classes defining kernel image types that kernel class should inherit."
@@ -7420,6 +7548,61 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
             </glossdef>
         </glossentry>
 
+        <glossentry id='var-KERNEL_DTB_LINK_NAME'><glossterm>KERNEL_DTB_LINK_NAME</glossterm>
+            <info>
+                KERNEL_DTB_LINK_NAME[doc] = "The link name of the kernel device tree binary (DTB)."
+            </info>
+            <glossdef>
+                <para role="glossdeffirst">
+<!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
+                    The link name of the kernel device tree binary (DTB).
+                    This variable is set in the
+                    <filename>meta/classes/kernel-artifact-names.bbclass</filename>
+                    file as follows:
+                    <literallayout class='monospaced'>
+     KERNEL_DTB_LINK_NAME ?= "${KERNEL_ARTIFACT_LINK_NAME}"
+                    </literallayout>
+                    The value of the <filename>KERNEL_ARTIFACT_LINK_NAME</filename>
+                    variable, which is set in the same file, has the following
+                    value:
+                    <literallayout class='monospaced'>
+     KERNEL_ARTIFACT_LINK_NAME ?= "${MACHINE}"
+                    </literallayout>
+                </para>
+
+                <para>
+                    See the
+                    <link linkend='var-MACHINE'><filename>MACHINE</filename></link>
+                    variable for additional information.
+                </para>
+            </glossdef>
+        </glossentry>
+
+        <glossentry id='var-KERNEL_DTB_NAME'><glossterm>KERNEL_DTB_NAME</glossterm>
+            <info>
+                KERNEL_DTB_NAME[doc] = "The base name of the kernel device tree binary (DTB)."
+            </info>
+            <glossdef>
+                <para role="glossdeffirst">
+<!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
+                    The base name of the kernel device tree binary (DTB).
+                    This variable is set in the
+                    <filename>meta/classes/kernel-artifact-names.bbclass</filename>
+                    file as follows:
+                    <literallayout class='monospaced'>
+     KERNEL_DTB_NAME ?= "${KERNEL_ARTIFACT_NAME}"
+                    </literallayout>
+                    The value of the
+                    <link linkend='var-KERNEL_ARTIFACT_NAME'><filename>KERNEL_ARTIFACT_NAME</filename></link>
+                    variable, which is set in the same file, has the following
+                    value:
+                    <literallayout class='monospaced'>
+     KERNEL_ARTIFACT_NAME ?= "${PKGE}-${PKGV}-${PKGR}-${MACHINE}${IMAGE_VERSION_SUFFIX}"
+                    </literallayout>
+                </para>
+            </glossdef>
+        </glossentry>
+
         <glossentry id='var-KERNEL_EXTRA_ARGS'><glossterm>KERNEL_EXTRA_ARGS</glossterm>
             <info>
                 KERNEL_EXTRA_ARGS[doc] = "Specifies additional make command-line arguments the OpenEmbedded build system passes on when compiling the kernel."
@@ -7477,38 +7660,93 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
                     <literallayout class='monospaced'>
      KERNEL_EXTRA_FEATURES ?= "features/netfilter/netfilter.scc features/taskstats/taskstats.scc"
      KERNEL_FEATURES_append = " ${KERNEL_EXTRA_FEATURES}"
-     KERNEL_FEATURES_append_qemuall=" cfg/virtio.scc"
-     KERNEL_FEATURES_append_qemux86=" cfg/sound.scc cfg/paravirt_kvm.scc"
-     KERNEL_FEATURES_append_qemux86-64=" cfg/sound.scc"
-                    </literallayout></para>
+     KERNEL_FEATURES_append_qemuall = " cfg/virtio.scc"
+     KERNEL_FEATURES_append_qemux86 = " cfg/sound.scc cfg/paravirt_kvm.scc"
+     KERNEL_FEATURES_append_qemux86-64 = " cfg/sound.scc"                    </literallayout></para>
             </glossdef>
         </glossentry>
 
-        <glossentry id='var-KERNEL_IMAGE_BASE_NAME'><glossterm>KERNEL_IMAGE_BASE_NAME</glossterm>
+        <glossentry id='var-KERNEL_FIT_LINK_NAME'><glossterm>KERNEL_FIT_LINK_NAME</glossterm>
             <info>
-                KERNEL_IMAGE_BASE_NAME[doc] = "The base name of the kernel image."
+                KERNEL_FIT_LINK_NAME[doc] = "The link name of the kernel flattened image tree (FIT) image."
             </info>
             <glossdef>
                 <para role="glossdeffirst">
 <!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
-                    The base name of the kernel image.
+                    The link name of the kernel flattened image tree (FIT) image.
                     This variable is set in the
-                    <link linkend='ref-classes-kernel'>kernel</link> class
-                    as follows:
+                    <filename>meta/classes/kernel-artifact-names.bbclass</filename>
+                    file as follows:
                     <literallayout class='monospaced'>
-     KERNEL_IMAGE_BASE_NAME ?= "${PKGE}-${PKGV}-${PKGR}-${MACHINE}-${DATETIME}"
+     KERNEL_FIT_LINK_NAME ?= "${KERNEL_ARTIFACT_LINK_NAME}"
+                    </literallayout>
+                    The value of the <filename>KERNEL_ARTIFACT_LINK_NAME</filename>
+                    variable, which is set in the same file, has the following
+                    value:
+                    <literallayout class='monospaced'>
+     KERNEL_ARTIFACT_LINK_NAME ?= "${MACHINE}"
                     </literallayout>
                 </para>
 
                 <para>
                     See the
-                    <link linkend='var-PKGE'><filename>PKGE</filename></link>,
-                    <link linkend='var-PKGV'><filename>PKGV</filename></link>,
-                    <link linkend='var-PKGR'><filename>PKGR</filename></link>,
-                    <link linkend='var-MACHINE'><filename>MACHINE</filename></link>,
-                    and
-                    <link linkend='var-DATETIME'><filename>DATETIME</filename></link>
-                    variables for additional information.
+                    <link linkend='var-MACHINE'><filename>MACHINE</filename></link>
+                    variable for additional information.
+                </para>
+            </glossdef>
+        </glossentry>
+
+        <glossentry id='var-KERNEL_FIT_NAME'><glossterm>KERNEL_FIT_NAME</glossterm>
+            <info>
+                KERNEL_FIT_NAME[doc] = "The base name of the kernel flattened image tree (FIT) image."
+            </info>
+            <glossdef>
+                <para role="glossdeffirst">
+<!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
+                    The base name of the kernel flattened image tree (FIT) image.
+                    This variable is set in the
+                    <filename>meta/classes/kernel-artifact-names.bbclass</filename>
+                    file as follows:
+                    <literallayout class='monospaced'>
+     KERNEL_FIT_NAME ?= "${KERNEL_ARTIFACT_NAME}"
+                    </literallayout>
+                    The value of the
+                    <link linkend='var-KERNEL_ARTIFACT_NAME'><filename>KERNEL_ARTIFACT_NAME</filename></link>
+                    variable, which is set in the same file, has the following
+                    value:
+                    <literallayout class='monospaced'>
+     KERNEL_ARTIFACT_NAME ?= "${PKGE}-${PKGV}-${PKGR}-${MACHINE}${IMAGE_VERSION_SUFFIX}"
+                    </literallayout>
+                </para>
+            </glossdef>
+        </glossentry>
+
+        <glossentry id='var-KERNEL_IMAGE_LINK_NAME'><glossterm>KERNEL_IMAGE_LINK_NAME</glossterm>
+            <info>
+                KERNEL_IMAGE_LINK_NAME[doc] = "The link name for the kernel image."
+            </info>
+            <glossdef>
+                <para role="glossdeffirst">
+<!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
+                    The link name for the kernel image.
+                    This variable is set in the
+                    <filename>meta/classes/kernel-artifact-names.bbclass</filename>
+                    file as follows:
+                    <literallayout class='monospaced'>
+     KERNEL_IMAGE_LINK_NAME ?= "${KERNEL_ARTIFACT_LINK_NAME}"
+                    </literallayout>
+                    The value of the <filename>KERNEL_ARTIFACT_LINK_NAME</filename>
+                    variable, which is set in the same file, has the following
+                    value:
+                    <literallayout class='monospaced'>
+     KERNEL_ARTIFACT_LINK_NAME ?= "${MACHINE}"
+                    </literallayout>
+                </para>
+
+                <para>
+                    See the
+                    <link linkend='var-MACHINE'><filename>MACHINE</filename></link>
+                    variable for additional information.
                 </para>
             </glossdef>
         </glossentry>
@@ -7544,6 +7782,31 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
             </glossdef>
         </glossentry>
 
+        <glossentry id='var-KERNEL_IMAGE_NAME'><glossterm>KERNEL_IMAGE_NAME</glossterm>
+            <info>
+                KERNEL_IMAGE_NAME[doc] = "The base name of the kernel image."
+            </info>
+            <glossdef>
+                <para role="glossdeffirst">
+<!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
+                    The base name of the kernel image.
+                    This variable is set in the
+                    <filename>meta/classes/kernel-artifact-names.bbclass</filename>
+                    file as follows:
+                    <literallayout class='monospaced'>
+     KERNEL_IMAGE_NAME ?= "${KERNEL_ARTIFACT_NAME}"
+                    </literallayout>
+                    The value of the
+                    <link linkend='var-KERNEL_ARTIFACT_NAME'><filename>KERNEL_ARTIFACT_NAME</filename></link>
+                    variable, which is set in the same file, has the following
+                    value:
+                    <literallayout class='monospaced'>
+     KERNEL_ARTIFACT_NAME ?= "${PKGE}-${PKGV}-${PKGR}-${MACHINE}${IMAGE_VERSION_SUFFIX}"
+                    </literallayout>
+                </para>
+            </glossdef>
+        </glossentry>
+
         <glossentry id='var-KERNEL_IMAGETYPE'><glossterm>KERNEL_IMAGETYPE</glossterm>
             <info>
                 KERNEL_IMAGETYPE[doc] = "The type of kernel to build for a device, usually set by the machine configuration files and defaults to 'zImage'."
@@ -8957,35 +9220,6 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
             </glossdef>
         </glossentry>
 
-        <glossentry id='var-MODULE_IMAGE_BASE_NAME'><glossterm>MODULE_IMAGE_BASE_NAME</glossterm>
-            <info>
-                MODULE_IMAGE_BASE_NAME[doc] = "The base name of the kernel modules tarball."
-            </info>
-            <glossdef>
-                <para role="glossdeffirst">
-<!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
-                    The base name of the kernel modules tarball.
-                    This variable is set in the
-                    <link linkend='ref-classes-kernel'>kernel</link> class
-                    as follows:
-                    <literallayout class='monospaced'>
-     MODULE_IMAGE_BASE_NAME ?= "modules-${PKGE}-${PKGV}-${PKGR}-${MACHINE}-${DATETIME}"
-                    </literallayout>
-                </para>
-
-                <para>
-                    See the
-                    <link linkend='var-PKGE'><filename>PKGE</filename></link>,
-                    <link linkend='var-PKGV'><filename>PKGV</filename></link>,
-                    <link linkend='var-PKGR'><filename>PKGR</filename></link>,
-                    <link linkend='var-MACHINE'><filename>MACHINE</filename></link>,
-                    and
-                    <link linkend='var-DATETIME'><filename>DATETIME</filename></link>
-                    variables for additional information.
-                </para>
-            </glossdef>
-        </glossentry>
-
         <glossentry id='var-MODULE_TARBALL_DEPLOY'><glossterm>MODULE_TARBALL_DEPLOY</glossterm>
             <info>
                 MODULE_TARBALL_DEPLOY[doc] = "Controls creation of the modules-*.tgz file. Set this variable to "0" to disable creation of this file, which contains all of the kernel modules resulting from a kernel build."
@@ -9002,6 +9236,61 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
             </glossdef>
         </glossentry>
 
+        <glossentry id='var-MODULE_TARBALL_LINK_NAME'><glossterm>MODULE_TARBALL_LINK_NAME</glossterm>
+            <info>
+                MODULE_TARBALL_LINK_NAME[doc] = "The link name of the kernel module tarball."
+            </info>
+            <glossdef>
+                <para role="glossdeffirst">
+<!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
+                    The link name of the kernel module tarball.
+                    This variable is set in the
+                    <filename>meta/classes/kernel-artifact-names.bbclass</filename>
+                    file as follows:
+                    <literallayout class='monospaced'>
+     MODULE_TARBALL_LINK_NAME ?= "${KERNEL_ARTIFACT_LINK_NAME}"
+                    </literallayout>
+                    The value of the <filename>KERNEL_ARTIFACT_LINK_NAME</filename>
+                    variable, which is set in the same file, has the following
+                    value:
+                    <literallayout class='monospaced'>
+     KERNEL_ARTIFACT_LINK_NAME ?= "${MACHINE}"
+                    </literallayout>
+                </para>
+
+                <para>
+                    See the
+                    <link linkend='var-MACHINE'><filename>MACHINE</filename></link>
+                    variable for additional information.
+                </para>
+            </glossdef>
+        </glossentry>
+
+        <glossentry id='var-MODULE_TARBALL_NAME'><glossterm>MODULE_TARBALL_NAME</glossterm>
+            <info>
+                MODULE_TARBALL_NAME[doc] = "The base name of the kernel module tarball."
+            </info>
+            <glossdef>
+                <para role="glossdeffirst">
+<!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
+                    The base name of the kernel module tarball.
+                    This variable is set in the
+                    <filename>meta/classes/kernel-artifact-names.bbclass</filename>
+                    file as follows:
+                    <literallayout class='monospaced'>
+     MODULE_TARBALL_NAME ?= "${KERNEL_ARTIFACT_NAME}"
+                    </literallayout>
+                    The value of the
+                    <link linkend='var-KERNEL_ARTIFACT_NAME'><filename>KERNEL_ARTIFACT_NAME</filename></link>
+                    variable, which is set in the same file, has the following
+                    value:
+                    <literallayout class='monospaced'>
+     KERNEL_ARTIFACT_NAME ?= "${PKGE}-${PKGV}-${PKGR}-${MACHINE}${IMAGE_VERSION_SUFFIX}"
+                    </literallayout>
+                </para>
+            </glossdef>
+        </glossentry>
+
 <!--
         <glossentry id='var-MULTIMACH_HOST_SYS'><glossterm>MULTIMACH_HOST_SYS</glossterm>
             <info>
@@ -9230,44 +9519,6 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
                 </para>
             </glossdef>
         </glossentry>
-
-        <glossentry id='var-NOHDD'><glossterm>NOHDD</glossterm>
-            <info>
-                NOHDD[doc] = "Causes the OpenEmbedded build system to skip building the .hddimg image."
-            </info>
-            <glossdef>
-                <para role="glossdeffirst">
-<!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
-                    Causes the OpenEmbedded build system to skip building the
-                    <filename>.hddimg</filename> image.
-                    The <filename>NOHDD</filename> variable is used with the
-                    <link linkend='ref-classes-image-live'><filename>image-live</filename></link>
-                    class.
-                    Set the variable to "1" to prevent the
-                    <filename>.hddimg</filename> image from being built.
-                </para>
-            </glossdef>
-        </glossentry>
-
-        <glossentry id='var-NOISO'><glossterm>NOISO</glossterm>
-            <info>
-                NOISO[doc] = "Causes the OpenEmbedded build system to skip building the ISO image."
-            </info>
-            <glossdef>
-                <para role="glossdeffirst">
-<!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
-                    Causes the OpenEmbedded build system to skip building the
-                    ISO image.
-                    The <filename>NOISO</filename> variable is used with the
-                    <link linkend='ref-classes-image-live'><filename>image-live</filename></link>
-                    class.
-                    Set the variable to "1" to prevent the ISO image from
-                    being built.
-                    To enable building an ISO image, set the variable to "0".
-                </para>
-            </glossdef>
-        </glossentry>
-
     </glossdiv>
 
     <glossdiv id='var-glossary-o'><title>O</title>
@@ -10086,9 +10337,9 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
                     Here is the basic block structure:
                     <literallayout class='monospaced'>
      PACKAGECONFIG ??= "f1 f2 f3 ..."
-     PACKAGECONFIG[f1] = "--with-f1,--without-f1,build-deps-f1,rt-deps-f1"
-     PACKAGECONFIG[f2] = "--with-f2,--without-f2,build-deps-f2,rt-deps-f2"
-     PACKAGECONFIG[f3] = "--with-f3,--without-f3,build-deps-f3,rt-deps-f3"
+     PACKAGECONFIG[f1] = "--with-f1,--without-f1,build-deps-f1,rt-deps-f1,rt-recs-f1"
+     PACKAGECONFIG[f2] = "--with-f2,--without-f2,build-deps-f2,rt-deps-f2,rt-recs-f2"
+     PACKAGECONFIG[f3] = "--with-f3,--without-f3,build-deps-f3,rt-deps-f3,rt-recs-f3"
                     </literallayout>
                 </para>
 
@@ -10097,7 +10348,7 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
                     variable itself specifies a space-separated list of the
                     features to enable.
                     Following the features, you can determine the behavior of
-                    each feature by providing up to four order-dependent
+                    each feature by providing up to five order-dependent
                     arguments, which are separated by commas.
                     You can omit any argument you like but must retain the
                     separating commas.
@@ -10123,6 +10374,10 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
                             (<link linkend='var-RDEPENDS'><filename>RDEPENDS</filename></link>)
                             that should be added if the feature is enabled.
                             </para></listitem>
+                        <listitem><para>Additional runtime recommendations
+                            (<link linkend='var-RRECOMMENDS'><filename>RRECOMMENDS</filename></link>)
+                            that should be added if the feature is enabled.
+                            </para></listitem>
                     </orderedlist>
                 </para>
 
@@ -10171,7 +10426,7 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
                             <filename>PACKAGECONFIG</filename>.
                             You can either completely override the variable:
                             <literallayout class='monospaced'>
-     PACKAGECONFIG="f4 f5"
+      PACKAGECONFIG = "f4 f5"
                             </literallayout>
                             Or, you can just append the variable:
                             <literallayout class='monospaced'>
@@ -10185,7 +10440,7 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
                             As with append files previously described,
                             you can either completely override the variable:
                             <literallayout class='monospaced'>
-     PACKAGECONFIG_pn-<replaceable>recipename</replaceable>="f4 f5"
+     PACKAGECONFIG_pn-<replaceable>recipename</replaceable> = "f4 f5"
                             </literallayout>
                             Or, you can just amend the variable:
                             <literallayout class='monospaced'>
@@ -13657,7 +13912,7 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
                     <link linkend='var-MIRRORS'><filename>MIRRORS</filename></link>
                     and <link linkend='var-PREMIRRORS'><filename>PREMIRRORS</filename></link>
                     and points to the cache locations to check for the shared
-                    objects.
+                    state (sstate) objects.
                 </para>
 
                 <para>
@@ -13669,6 +13924,25 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
                     other machines.
                 </para>
 
+                <para>
+                    When pointing to sstate build artifacts on another machine
+                    that uses a different GCC version for native builds,
+                    you must configure <filename>SSTATE_MIRROR</filename>
+                    with a regular expression that maps local search paths
+                    to server paths.
+                    The paths need to take into account
+                    <link linkend='var-NATIVELSBSTRING'><filename>NATIVELSBSTRING</filename></link>
+                    set by the
+                    <link linkend='ref-classes-uninative'><filename>uninative</filename></link>
+                    class.
+                    For example, the following maps the local search path
+                    <filename>universal-4.9</filename> to the server-provided
+                    path <replaceable>server_url_sstate_path</replaceable>:
+                    <literallayout class='monospaced'>
+     SSTATE_MIRRORS ?= file://universal-4.9/(.*) http://<replaceable>server_url_sstate_path</replaceable>/universal-4.8/\1 \n
+                    </literallayout>
+                </para>
+
                 <para>
                     If a mirror uses the same structure as
                     <link linkend='var-SSTATE_DIR'><filename>SSTATE_DIR</filename></link>,
@@ -15169,41 +15443,6 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
             </glossdef>
         </glossentry>
 
-        <glossentry id='var-TEST_IMAGE'><glossterm>TEST_IMAGE</glossterm>
-            <info>
-                TEST_IMAGE[doc] = "Enables test booting of virtual machine images under the QEMU emulator after any root filesystems are created and runs tests against those images."
-            </info>
-            <glossdef>
-                <para role="glossdeffirst">
-<!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
-                    Automatically runs the series of automated tests for
-                    images when an image is successfully built.
-                </para>
-
-                <para>
-                    These tests are written in Python making use of the
-                    <filename>unittest</filename> module, and the majority of
-                    them run commands on the target system over
-                    <filename>ssh</filename>.
-                    You can set this variable to "1" in your
-                    <filename>local.conf</filename> file in the
-                    <link linkend='build-directory'>Build Directory</link>
-                    to have the OpenEmbedded build system automatically run
-                    these tests after an image successfully builds:
-                    <literallayout class='monospaced'>
-     TEST_IMAGE = "1"
-                    </literallayout>
-                    For more information on enabling, running, and writing
-                    these tests, see the
-                    "<ulink url='&YOCTO_DOCS_DEV_URL;#performing-automated-runtime-testing'>Performing Automated Runtime Testing</ulink>"
-                    section in the Yocto Project Development Tasks Manual and
-                    the
-                    "<link linkend='ref-classes-testimage*'><filename>testimage*.bbclass</filename></link>"
-                    section.
-                </para>
-            </glossdef>
-        </glossentry>
-
         <glossentry id='var-TEST_LOG_DIR'><glossterm>TEST_LOG_DIR</glossterm>
             <info>
                 TEST_LOG_DIR[doc] = "Holds the SSH log and the boot log for QEMU machines. The TEST_LOG_DIR variable defaults to "${WORKDIR}/testimage"."
@@ -15504,6 +15743,47 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
             </glossdef>
         </glossentry>
 
+        <glossentry id='var-TESTIMAGE_AUTO'><glossterm>TESTIMAGE_AUTO</glossterm>
+            <info>
+                TESTIMAGE_AUTO[doc] = "Enables automatic testing of an image once it is built."
+            </info>
+            <glossdef>
+                <para role="glossdeffirst">
+<!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
+                    Automatically runs the series of automated tests for
+                    images when an image is successfully built.
+                    Setting <filename>TESTIMAGE_AUTO</filename> to "1"
+                    causes any image that successfully builds to automatically
+                    boot under QEMU.
+                    Using the variable also adds in dependencies so that any
+                    SDK for which testing is requested is automatically built
+                    first.
+                </para>
+
+                <para>
+                    These tests are written in Python making use of the
+                    <filename>unittest</filename> module, and the majority of
+                    them run commands on the target system over
+                    <filename>ssh</filename>.
+                    You can set this variable to "1" in your
+                    <filename>local.conf</filename> file in the
+                    <link linkend='build-directory'>Build Directory</link>
+                    to have the OpenEmbedded build system automatically run
+                    these tests after an image successfully builds:
+                    <literallayout class='monospaced'>
+     TESTIMAGE_AUTO = "1"
+                    </literallayout>
+                    For more information on enabling, running, and writing
+                    these tests, see the
+                    "<ulink url='&YOCTO_DOCS_DEV_URL;#performing-automated-runtime-testing'>Performing Automated Runtime Testing</ulink>"
+                    section in the Yocto Project Development Tasks Manual and
+                    the
+                    "<link linkend='ref-classes-testimage*'><filename>testimage*.bbclass</filename></link>"
+                    section.
+                </para>
+            </glossdef>
+        </glossentry>
+
         <glossentry id='var-THISDIR'><glossterm>THISDIR</glossterm>
             <info>
                 THISDIR[doc] = "The directory in which the file BitBake is currently parsing is located."

documentation/sdk-manual/sdk-manual.xml

@@ -58,9 +58,14 @@
             </revision>
             <revision>
                 <revnumber>2.6</revnumber>
-                <date>&REL_MONTH_YEAR;</date>
+                <date>November 2018</date>
                 <revremark>Released with the Yocto Project 2.6 Release.</revremark>
             </revision>
+            <revision>
+                <revnumber>2.7</revnumber>
+                <date>&REL_MONTH_YEAR;</date>
+                <revremark>Released with the Yocto Project 2.7 Release.</revremark>
+            </revision>
        </revhistory>
 
     <copyright>

documentation/toaster-manual/toaster-manual.xml

@@ -68,9 +68,14 @@
             </revision>
             <revision>
                 <revnumber>2.6</revnumber>
-                <date>&REL_MONTH_YEAR;</date>
+                <date>November 2018</date>
                 <revremark>Released with the Yocto Project 2.6 Release.</revremark>
             </revision>
+            <revision>
+                <revnumber>2.7</revnumber>
+                <date>&REL_MONTH_YEAR;</date>
+                <revremark>Released with the Yocto Project 2.7 Release.</revremark>
+            </revision>
        </revhistory>
 
     <copyright>

documentation/tools/mega-manual.sed

@@ -2,39 +2,39 @@
 # This style is for manual folders like "yocto-project-qs" and "poky-ref-manual".
 # This is the old way that did it.  Can't do that now that we have "bitbake-user-manual" strings
 # in the mega-manual.
-# s@"ulink" href="http://www.yoctoproject.org/docs/2.6/[a-z]*-[a-z]*-[a-z]*/[a-z]*-[a-z]*-[a-z]*.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.6/yocto-project-qs/yocto-project-qs.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.6/poky-ref-manual/poky-ref-manual.html#@"link" href="#@g
+# s@"ulink" href="http://www.yoctoproject.org/docs/2.7/[a-z]*-[a-z]*-[a-z]*/[a-z]*-[a-z]*-[a-z]*.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/2.7/yocto-project-qs/yocto-project-qs.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/2.7/poky-ref-manual/poky-ref-manual.html#@"link" href="#@g
 
 # Processes all other manuals (<word>-<word> style) except for the BitBake User Manual because
 # it is not included in the mega-manual.
 # This style is for manual folders that use two word, which is the standard now (e.g. "ref-manual").
 # This was the one-liner that worked before we introduced the BitBake User Manual, which is
 # not in the mega-manual.
-# s@"ulink" href="http://www.yoctoproject.org/docs/2.6/[a-z]*-[a-z]*/[a-z]*-[a-z]*.html#@"link" href="#@g
+# s@"ulink" href="http://www.yoctoproject.org/docs/2.7/[a-z]*-[a-z]*/[a-z]*-[a-z]*.html#@"link" href="#@g
 
-s@"ulink" href="http://www.yoctoproject.org/docs/2.6/sdk-manual/sdk-manual.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.6/bsp-guide/bsp-guide.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.6/dev-manual/dev-manual.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.6/overview-manual/overview-manual.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.6/brief-yoctoprojectqs/brief-yoctoprojectqs.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.6/kernel-dev/kernel-dev.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.6/profile-manual/profile-manual.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.6/ref-manual/ref-manual.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.6/toaster-manual/toaster-manual.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/2.7/sdk-manual/sdk-manual.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/2.7/bsp-guide/bsp-guide.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/2.7/dev-manual/dev-manual.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/2.7/overview-manual/overview-manual.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/2.7/brief-yoctoprojectqs/brief-yoctoprojectqs.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/2.7/kernel-dev/kernel-dev.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/2.7/profile-manual/profile-manual.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/2.7/ref-manual/ref-manual.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/2.7/toaster-manual/toaster-manual.html#@"link" href="#@g
 
 # Process cases where just an external manual is referenced without an id anchor
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.6/brief-yoctoprojectqs/brief-yoctoprojectqs.html" target="_top">Yocto Project Quick Build</a>@Yocto Project Quick Build@g
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.6/yocto-project-qs/yocto-project-qs.html" target="_top">Yocto Project Quick Start</a>@Yocto Project Quick Start@g
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.6/dev-manual/dev-manual.html" target="_top">Yocto Project Development Tasks Manual</a>@Yocto Project Development Tasks Manual@g
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.6/overview-manual/overview-manual.html" target="_top">Yocto Project Overview and Concepts Manual</a>@Yocto project Overview and Concepts Manual@g
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.6/sdk-manual/sdk-manual.html" target="_top">Yocto Project Application Development and the Extensible Software Development Kit (eSDK)</a>@Yocto Project Application Development and the Extensible Software Development Kit (eSDK)@g
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.6/bsp-guide/bsp-guide.html" target="_top">Yocto Project Board Support Package (BSP) Developer's Guide</a>@Yocto Project Board Support Package (BSP) Developer's Guide@g
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.6/profile-manual/profile-manual.html" target="_top">Yocto Project Profiling and Tracing Manual</a>@Yocto Project Profiling and Tracing Manual@g
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.6/kernel-dev/kernel-dev.html" target="_top">Yocto Project Linux Kernel Development Manual</a>@Yocto Project Linux Kernel Development Manual@g
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.6/ref-manual/ref-manual.html" target="_top">Yocto Project Reference Manual</a>@Yocto Project Reference Manual@g
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.6/toaster-manual/toaster-manual.html" target="_top">Toaster User Manual</a>@Toaster User Manual@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.7/brief-yoctoprojectqs/brief-yoctoprojectqs.html" target="_top">Yocto Project Quick Build</a>@Yocto Project Quick Build@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.7/yocto-project-qs/yocto-project-qs.html" target="_top">Yocto Project Quick Start</a>@Yocto Project Quick Start@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.7/dev-manual/dev-manual.html" target="_top">Yocto Project Development Tasks Manual</a>@Yocto Project Development Tasks Manual@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.7/overview-manual/overview-manual.html" target="_top">Yocto Project Overview and Concepts Manual</a>@Yocto project Overview and Concepts Manual@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.7/sdk-manual/sdk-manual.html" target="_top">Yocto Project Application Development and the Extensible Software Development Kit (eSDK)</a>@Yocto Project Application Development and the Extensible Software Development Kit (eSDK)@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.7/bsp-guide/bsp-guide.html" target="_top">Yocto Project Board Support Package (BSP) Developer's Guide</a>@Yocto Project Board Support Package (BSP) Developer's Guide@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.7/profile-manual/profile-manual.html" target="_top">Yocto Project Profiling and Tracing Manual</a>@Yocto Project Profiling and Tracing Manual@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.7/kernel-dev/kernel-dev.html" target="_top">Yocto Project Linux Kernel Development Manual</a>@Yocto Project Linux Kernel Development Manual@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.7/ref-manual/ref-manual.html" target="_top">Yocto Project Reference Manual</a>@Yocto Project Reference Manual@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.7/toaster-manual/toaster-manual.html" target="_top">Toaster User Manual</a>@Toaster User Manual@g
 
 # Process a single, rouge occurrence of a linked reference to the Mega-Manual.
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.6/mega-manual/mega-manual.html" target="_top">Yocto Project Mega-Manual</a>@Yocto Project Mega-Manual@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.7/mega-manual/mega-manual.html" target="_top">Yocto Project Mega-Manual</a>@Yocto Project Mega-Manual@g
 

meta-poky/conf/distro/poky.conf

@@ -1,6 +1,6 @@
 DISTRO = "poky"
 DISTRO_NAME = "Poky (Yocto Project Reference Distro)"
-DISTRO_VERSION = "2.5+snapshot-${DATE}"
+DISTRO_VERSION = "2.6+snapshot-${DATE}"
 DISTRO_CODENAME = "master"
 SDK_VENDOR = "-pokysdk"
 SDK_VERSION := "${@'${DISTRO_VERSION}'.replace('snapshot-${DATE}','snapshot')}"
@@ -23,7 +23,7 @@ DISTRO_FEATURES ?= "${DISTRO_FEATURES_DEFAULT} ${DISTRO_FEATURES_LIBC} ${POKY_DE
 
 PREFERRED_VERSION_linux-yocto ?= "4.18%"
 
-SDK_NAME = "${DISTRO}-${TCLIBC}-${SDK_ARCH}-${IMAGE_BASENAME}-${TUNE_PKGARCH}"
+SDK_NAME = "${DISTRO}-${TCLIBC}-${SDKMACHINE}-${IMAGE_BASENAME}-${TUNE_PKGARCH}-${MACHINE}"
 SDKPATH = "/opt/${DISTRO}/${SDK_VERSION}"
 
 DISTRO_EXTRA_RDEPENDS += " ${POKY_DEFAULT_EXTRA_RDEPENDS}"
@@ -64,18 +64,15 @@ https://.*/.*    http://downloads.yoctoproject.org/mirror/sources/ \n"
 CONNECTIVITY_CHECK_URIS ?= "https://www.example.com/"
 
 SANITY_TESTED_DISTROS ?= " \
-            poky-2.4 \n \
             poky-2.5 \n \
-            ubuntu-15.04 \n \
+            poky-2.6 \n \
             ubuntu-16.04 \n \
-            ubuntu-16.10 \n \
-            ubuntu-17.04 \n \
-            fedora-26 \n \
+            ubuntu-18.04 \n \
+            fedora-28 \n \
             centos-7 \n \
             debian-8 \n \
             debian-9 \n \
-            opensuse-42.1 \n \
-            opensuse-42.2 \n \
+            opensuse-42.3 \n \
             "
 #
 # OELAYOUT_ABI allows us to notify users when the format of TMPDIR changes in

meta-poky/conf/local.conf.sample

@@ -157,10 +157,12 @@ USER_CLASSES ?= "buildstats image-mklibs image-prelink"
 # Runtime testing of images
 #
 # The build system can test booting virtual machine images under qemu (an emulator)
-# after any root filesystems are created and run tests against those images. To
-# enable this uncomment this line. See classes/testimage(-auto).bbclass for
-# further details.
-#TEST_IMAGE = "1"
+# after any root filesystems are created and run tests against those images. It can also
+# run tests against any SDK that are built. To enable this uncomment these lines.
+# See classes/test{image,sdk}.bbclass for further details.
+#IMAGE_CLASSES += "testimage testsdk"
+#TESTIMAGE_AUTO_qemuall = "1"
+
 #
 # Interactive shell configuration
 #

meta-selftest/recipes-devtools/python/python-async-test.inc

@@ -0,0 +1,16 @@
+SUMMARY = "Python framework to process interdependent tasks in a pool of workers"
+HOMEPAGE = "http://github.com/gitpython-developers/async"
+SECTION = "devel/python"
+LICENSE = "BSD"
+LIC_FILES_CHKSUM = "file://PKG-INFO;beginline=8;endline=8;md5=88df8e78b9edfd744953862179f2d14e"
+
+inherit pypi
+
+PYPI_PACKAGE = "async"
+
+SRC_URI[md5sum] = "9b06b5997de2154f3bc0273f80bcef6b"
+SRC_URI[sha256sum] = "ac6894d876e45878faae493b0cf61d0e28ec417334448ac0a6ea2229d8343051"
+
+RDEPENDS_${PN} += "${PYTHON_PN}-threading"
+
+BBCLASSEXTEND = "nativesdk"

meta-selftest/recipes-devtools/python/python3-async-test_0.6.2.bb

@@ -0,0 +1,2 @@
+inherit setuptools3
+require python-async-test.inc

meta-selftest/recipes-test/delay/delay.bb

@@ -0,0 +1,12 @@
+SUMMARY = "Recipe with a fixed delay task"
+DESCRIPTION = "Contains a delay task to be used to for testing."
+LICENSE = "MIT"
+
+INHIBIT_DEFAULT_DEPS = "1"
+EXCLUDE_FROM_WORLD = "1"
+
+do_delay() {
+	sleep 5
+}
+do_delay[nostamp] = "1"
+addtask delay

meta-selftest/recipes-test/error/error.bb

@@ -2,7 +2,7 @@ SUMMARY = "Error Test case that fails on do_compile"
 DESCRIPTION = "This generates a compile time error to be used to for testing."
 LICENSE = "MIT"
 
-INHIBIT_DEFAULT_DEPS = "1"                                                                                                                                                                  
+INHIBIT_DEFAULT_DEPS = "1"
 EXCLUDE_FROM_WORLD = "1"
 
 do_compile() {

meta-selftest/recipes-test/recipeutils/recipeutils-test.inc

@@ -0,0 +1,5 @@
+SRC_URI = "http://xorg.freedesktop.org/releases/individual/lib/libxshmfence-${PV}.tar.bz2"
+
+SRC_URI[md5sum] = "66662e76899112c0f99e22f2fc775a7e"
+SRC_URI[sha256sum] = "d21b2d1fd78c1efbe1f2c16dae1cb23f8fd231dcf891465b8debe636a9054b0c"
+

meta-selftest/recipes-test/recipeutils/recipeutils-test_1.2.bb

@@ -0,0 +1,13 @@
+SUMMARY = "Test recipe for recipeutils.patch_recipe()"
+
+require recipeutils-test.inc
+
+LICENSE = "Proprietary"
+LIC_FILES_CHKSUM = "file://${WORKDIR}/somefile;md5=d41d8cd98f00b204e9800998ecf8427e"
+DEPENDS += "virtual/libx11"
+
+BBCLASSEXTEND = "native nativesdk"
+
+SRC_URI += "file://somefile"
+
+SRC_URI_append = " file://anotherfile"

meta-yocto-bsp/conf/machine/beaglebone-yocto.conf

@@ -20,7 +20,8 @@ WKS_FILE ?= "beaglebone-yocto.wks"
 IMAGE_INSTALL_append = " kernel-devicetree kernel-image-zimage"
 do_image_wic[depends] += "mtools-native:do_populate_sysroot dosfstools-native:do_populate_sysroot"
 
-SERIAL_CONSOLES = "115200;ttyO0"
+SERIAL_CONSOLES ?= "115200;ttyS0 115200;ttyO0"
+SERIAL_CONSOLES_CHECK = "${SERIAL_CONSOLES}"
 
 PREFERRED_PROVIDER_virtual/kernel ?= "linux-yocto"
 PREFERRED_VERSION_linux-yocto ?= "4.18%"

meta/classes/archiver.bbclass

@@ -23,9 +23,6 @@
 #    COPYLEFT_RECIPE_TYPES = 'target'
 #
 
-# Don't filter the license by default
-COPYLEFT_LICENSE_INCLUDE ?= ''
-COPYLEFT_LICENSE_EXCLUDE ?= ''
 # Create archive for all the recipe types
 COPYLEFT_RECIPE_TYPES ?= 'target native nativesdk cross crosssdk cross-canadian'
 inherit copyleft_filter
@@ -275,6 +272,14 @@ python do_ar_configured() {
         create_tarball(d, srcdir, 'configured', ar_outdir)
 }
 
+def exclude_useless_paths(tarinfo):
+    if tarinfo.isdir():
+        if tarinfo.name.endswith('/temp') or tarinfo.name.endswith('/patches') or tarinfo.name.endswith('/.pc'):
+            return None
+        elif tarinfo.name == 'temp' or tarinfo.name == 'patches' or tarinfo.name == '.pc':
+            return None
+    return tarinfo
+
 def create_tarball(d, srcdir, suffix, ar_outdir):
     """
     create the tarball from srcdir
@@ -299,7 +304,7 @@ def create_tarball(d, srcdir, suffix, ar_outdir):
 
     bb.note('Creating %s' % tarname)
     tar = tarfile.open(tarname, 'w:gz')
-    tar.add(srcdir, arcname=os.path.basename(srcdir))
+    tar.add(srcdir, arcname=os.path.basename(srcdir), filter=exclude_useless_paths)
     tar.close()
 
 # creating .diff.gz between source.orig and source

meta/classes/base.bbclass

@@ -122,12 +122,22 @@ def setup_hosttools_dir(dest, toolsvar, d, fatal=True):
         desttool = os.path.join(dest, tool)
         if not os.path.exists(desttool):
             srctool = bb.utils.which(path, tool, executable=True)
+            # gcc/g++ may link to ccache on some hosts, e.g.,
+            # /usr/local/bin/ccache/gcc -> /usr/bin/ccache, then which(gcc)
+            # would return /usr/local/bin/ccache/gcc, but what we need is
+            # /usr/bin/gcc, this code can check and fix that.
             if "ccache" in srctool:
                 srctool = bb.utils.which(path, tool, executable=True, direction=1)
             if srctool:
                 os.symlink(srctool, desttool)
             else:
                 notfound.append(tool)
+    # Force "python" -> "python2"
+    desttool = os.path.join(dest, "python")
+    if not os.path.exists(desttool):
+        srctool = "python2"
+        os.symlink(srctool, desttool)
+
     if notfound and fatal:
         bb.fatal("The following required tools (as specified by HOSTTOOLS) appear to be unavailable in PATH, please install them in order to proceed:\n  %s" % " ".join(notfound))
 
@@ -303,7 +313,9 @@ base_do_configure() {
 			if [ "${CLEANBROKEN}" != "1" -a \( -e Makefile -o -e makefile -o -e GNUmakefile \) ]; then
 				oe_runmake clean
 			fi
-			find ${B} -ignore_readdir_race -name \*.la -delete
+			# -ignore_readdir_race does not work correctly with -delete;
+			# use xargs to avoid spurious build failures
+			find ${B} -ignore_readdir_race -name \*.la -type f -print0 | xargs -0 rm -f
 		fi
 	fi
 	if [ -n "${CONFIGURESTAMPFILE}" ]; then
@@ -459,12 +471,15 @@ python () {
 
     if bb.data.inherits_class('license', d):
         check_license_format(d)
-        unmatched_license_flag = check_license_flags(d)
-        if unmatched_license_flag:
-            bb.debug(1, "Skipping %s because it has a restricted license not"
-                 " whitelisted in LICENSE_FLAGS_WHITELIST" % pn)
-            raise bb.parse.SkipRecipe("because it has a restricted license not"
-                 " whitelisted in LICENSE_FLAGS_WHITELIST")
+        unmatched_license_flags = check_license_flags(d)
+        if unmatched_license_flags:
+            if len(unmatched_license_flags) == 1:
+                message = "because it has a restricted license '{0}'. Which is not whitelisted in LICENSE_FLAGS_WHITELIST".format(unmatched_license_flags[0])
+            else:
+                message = "because it has restricted licenses {0}. Which are not whitelisted in LICENSE_FLAGS_WHITELIST".format(
+                    ", ".join("'{0}'".format(f) for f in unmatched_license_flags))
+            bb.debug(1, "Skipping %s %s" % (pn, message))
+            raise bb.parse.SkipRecipe(message)
 
     # If we're building a target package we need to use fakeroot (pseudo)
     # in order to capture permissions, owners, groups and special files

meta/classes/binconfig.bbclass

@@ -31,7 +31,7 @@ BINCONFIG_GLOB ?= "*-config"
 PACKAGE_PREPROCESS_FUNCS += "binconfig_package_preprocess"
 
 binconfig_package_preprocess () {
-	for config in `find ${PKGD} -name '${BINCONFIG_GLOB}'`; do
+	for config in `find ${PKGD} -type f -name '${BINCONFIG_GLOB}'`; do
 		sed -i \
 		    -e 's:${STAGING_BASELIBDIR}:${base_libdir}:g;' \
 		    -e 's:${STAGING_LIBDIR}:${libdir}:g;' \
@@ -40,7 +40,7 @@ binconfig_package_preprocess () {
 		    -e 's:${STAGING_DIR_HOST}${prefix}:${prefix}:' \
                     $config
 	done
-	for lafile in `find ${PKGD} -name "*.la"` ; do
+	for lafile in `find ${PKGD} -type f -name "*.la"` ; do
 		sed -i \
 		    -e 's:${STAGING_BASELIBDIR}:${base_libdir}:g;' \
 		    -e 's:${STAGING_LIBDIR}:${libdir}:g;' \
@@ -54,7 +54,7 @@ binconfig_package_preprocess () {
 SYSROOT_PREPROCESS_FUNCS += "binconfig_sysroot_preprocess"
 
 binconfig_sysroot_preprocess () {
-	for config in `find ${S} -name '${BINCONFIG_GLOB}'` `find ${B} -name '${BINCONFIG_GLOB}'`; do
+	for config in `find ${S} -type f -name '${BINCONFIG_GLOB}'` `find ${B} -type f -name '${BINCONFIG_GLOB}'`; do
 		configname=`basename $config`
 		install -d ${SYSROOT_DESTDIR}${bindir_crossscripts}
 		sed ${@get_binconfig_mangle(d)} $config > ${SYSROOT_DESTDIR}${bindir_crossscripts}/$configname

meta/classes/bugzilla.bbclass

@@ -1,187 +0,0 @@
-#
-# Small event handler to automatically open URLs and file
-# bug reports at a bugzilla of your choiche
-# it uses XML-RPC interface, so you must have it enabled
-#
-# Before using you must define BUGZILLA_USER, BUGZILLA_PASS credentials,
-# BUGZILLA_XMLRPC - uri of xmlrpc.cgi,
-# BUGZILLA_PRODUCT, BUGZILLA_COMPONENT - a place in BTS for build bugs
-# BUGZILLA_VERSION - version against which to report new bugs
-#
-
-def bugzilla_find_bug_report(debug_file, server, args, bugname):
-    args['summary'] = bugname
-    bugs = server.Bug.search(args)
-    if len(bugs['bugs']) == 0:
-        print >> debug_file, "Bugs not found"
-        return (False,None)
-    else: # silently pick the first result
-        print >> debug_file, "Result of bug search is "
-        print >> debug_file, bugs
-        status = bugs['bugs'][0]['status']
-        id = bugs['bugs'][0]['id']
-        return (not status in ["CLOSED", "RESOLVED", "VERIFIED"],id)
-
-def bugzilla_file_bug(debug_file, server, args, name, text, version):
-    args['summary'] = name
-    args['comment'] = text
-    args['version'] = version
-    args['op_sys'] = 'Linux'
-    args['platform'] = 'Other'
-    args['severity'] = 'normal'
-    args['priority'] = 'Normal'
-    try:
-        return server.Bug.create(args)['id']
-    except Exception, e:
-        print >> debug_file, repr(e)
-        return None
-
-def bugzilla_reopen_bug(debug_file, server, args, bug_number):
-    args['ids'] = [bug_number]
-    args['status'] = "CONFIRMED"
-    try:
-        server.Bug.update(args)
-        return True
-    except Exception, e:
-        print >> debug_file, repr(e)
-        return False
-
-def bugzilla_create_attachment(debug_file, server, args, bug_number, text, file_name, log, logdescription):
-    args['ids'] = [bug_number]
-    args['file_name'] = file_name
-    args['summary'] = logdescription
-    args['content_type'] = "text/plain"
-    args['data'] = log
-    args['comment'] = text
-    try:
-        server.Bug.add_attachment(args)
-        return True
-    except Exception, e:
-        print >> debug_file, repr(e)
-        return False
-
-def bugzilla_add_comment(debug_file, server, args, bug_number, text):
-    args['id'] = bug_number
-    args['comment'] = text
-    try:
-        server.Bug.add_comment(args)
-        return True
-    except Exception, e:
-        print >> debug_file, repr(e)
-        return False
-
-addhandler bugzilla_eventhandler
-bugzilla_eventhandler[eventmask] = "bb.event.MsgNote bb.build.TaskFailed"
-python bugzilla_eventhandler() {
-    import glob
-    import xmlrpclib, httplib
-
-    class ProxiedTransport(xmlrpclib.Transport):
-        def __init__(self, proxy, use_datetime = 0):
-            xmlrpclib.Transport.__init__(self, use_datetime)
-            self.proxy = proxy
-            self.user = None
-            self.password = None
-
-        def set_user(self, user):
-            self.user = user
-
-        def set_password(self, password):
-            self.password = password
-
-        def make_connection(self, host):
-            self.realhost = host
-            return httplib.HTTP(self.proxy)
-
-        def send_request(self, connection, handler, request_body):
-            connection.putrequest("POST", 'http://%s%s' % (self.realhost, handler))
-            if self.user != None:
-                if self.password != None:
-                    auth = "%s:%s" % (self.user, self.password)
-                else:
-                    auth = self.user
-                connection.putheader("Proxy-authorization", "Basic " + base64.encodestring(auth))
-
-    event = e
-    data = e.data
-    name = bb.event.getName(event)
-    if name == "MsgNote":
-        # avoid recursion
-        return
-
-    if name == "TaskFailed":
-        xmlrpc  = data.getVar("BUGZILLA_XMLRPC")
-        user    = data.getVar("BUGZILLA_USER")
-        passw   = data.getVar("BUGZILLA_PASS")
-        product = data.getVar("BUGZILLA_PRODUCT")
-        compon  = data.getVar("BUGZILLA_COMPONENT")
-        version = data.getVar("BUGZILLA_VERSION")
-
-        proxy   = data.getVar('http_proxy')
-        if (proxy):
-            import urllib2
-            s, u, p, hostport = urllib2._parse_proxy(proxy)
-            transport = ProxiedTransport(hostport)
-        else:
-            transport = None
-
-        server = xmlrpclib.ServerProxy(xmlrpc, transport=transport, verbose=0)
-        args = {
-            'Bugzilla_login': user,
-            'Bugzilla_password': passw,
-            'product': product,
-            'component': compon}
-
-        # evil hack to figure out what is going on
-        debug_file = open(os.path.join(data.getVar("TMPDIR"),"..","bugzilla-log"),"a")
-
-        file = None
-        bugname = "%(package)s-%(pv)s-autobuild" % { "package" : data.getVar("PN"),
-                                                           "pv"      : data.getVar("PV"),
-                                                           }
-        log_file = glob.glob("%s/log.%s.*" % (event.data.getVar('T'), event.task))
-        text     = "The %s step in %s failed at %s for machine %s" % (e.task, data.getVar("PN"), data.getVar('DATETIME'), data.getVar('MACHINE') )
-        if len(log_file) != 0:
-            print >> debug_file, "Adding log file %s" % log_file[0]
-            file = open(log_file[0], 'r')
-            log = file.read()
-            file.close();
-        else:
-            print >> debug_file, "No log file found for the glob"
-            log = None
-
-        (bug_open, bug_number) = bugzilla_find_bug_report(debug_file, server, args.copy(), bugname)
-        print >> debug_file, "Bug is open: %s and bug number: %s" % (bug_open, bug_number)
-
-        # The bug is present and still open, attach an error log
-        if not bug_number:
-            bug_number = bugzilla_file_bug(debug_file, server, args.copy(), bugname, text, version)
-            if not bug_number:
-                print >> debug_file, "Couldn't acquire a new bug_numer, filing a bugreport failed"
-            else:
-                print >> debug_file, "The new bug_number: '%s'" % bug_number
-        elif not bug_open:
-            if not bugzilla_reopen_bug(debug_file, server, args.copy(), bug_number):
-                print >> debug_file, "Failed to reopen the bug #%s" % bug_number
-            else:
-                print >> debug_file, "Reopened the bug #%s" % bug_number
-
-        if bug_number and log:
-            print >> debug_file, "The bug is known as '%s'" % bug_number
-            desc = "Build log for machine %s" % (data.getVar('MACHINE'))
-            if not bugzilla_create_attachment(debug_file, server, args.copy(), bug_number, text, log_file[0], log, desc):
-                 print >> debug_file, "Failed to attach the build log for bug #%s" % bug_number
-            else:
-                 print >> debug_file, "Created an attachment for '%s' '%s' '%s'" % (product, compon, bug_number)
-        else:
-            print >> debug_file, "Not trying to create an attachment for bug #%s" % bug_number
-            if not bugzilla_add_comment(debug_file, server, args.copy(), bug_number, text, ):
-                 print >> debug_file, "Failed to create a comment the build log for bug #%s" % bug_number
-            else:
-                 print >> debug_file, "Created an attachment for '%s' '%s' '%s'" % (product, compon, bug_number)
-
-        # store bug number for oestats-client
-        if bug_number:
-            data.setVar('OESTATS_BUG_NUMBER', bug_number)
-}
-

meta/classes/clutter.bbclass

@@ -1,11 +1,11 @@
 def get_minor_dir(v):
     import re
-    m = re.match("^([0-9]+)\.([0-9]+)", v)
+    m = re.match(r"^([0-9]+)\.([0-9]+)", v)
     return "%s.%s" % (m.group(1), m.group(2))
 
 def get_real_name(n):
     import re
-    m = re.match("^([a-z]+(-[a-z]+)?)(-[0-9]+\.[0-9]+)?", n)
+    m = re.match(r"^([a-z]+(-[a-z]+)?)(-[0-9]+\.[0-9]+)?", n)
     return "%s" % (m.group(1))
 
 VERMINOR = "${@get_minor_dir("${PV}")}"

meta/classes/cmake.bbclass

@@ -20,7 +20,7 @@ python() {
     elif generator == "Ninja":
         d.appendVar("DEPENDS", " ninja-native")
         d.setVar("OECMAKE_GENERATOR_ARGS", "-G Ninja -DCMAKE_MAKE_PROGRAM=ninja")
-        d.setVarFlag("do_compile", "progress", "outof:^\[(\d+)/(\d+)\]\s+")
+        d.setVarFlag("do_compile", "progress", r"outof:^\[(\d+)/(\d+)\]\s+")
     else:
         bb.fatal("Unknown CMake Generator %s" % generator)
 }
@@ -108,6 +108,10 @@ list(APPEND CMAKE_MODULE_PATH "${STAGING_DATADIR}/cmake/Modules/")
 # add for non /usr/lib libdir, e.g. /usr/lib64
 set( CMAKE_LIBRARY_PATH ${libdir} ${base_libdir})
 
+# add include dir to implicit includes in case it differs from /usr/include
+list(APPEND CMAKE_C_IMPLICIT_INCLUDE_DIRECTORIES ${includedir})
+list(APPEND CMAKE_CXX_IMPLICIT_INCLUDE_DIRECTORIES ${includedir})
+
 EOF
 }
 

meta/classes/cpan.bbclass

@@ -16,7 +16,7 @@ export PERL_ARCHLIB = "${STAGING_LIBDIR}${PERL_OWN_DIR}/perl/${@get_perl_version
 export PERLHOSTLIB = "${STAGING_LIBDIR_NATIVE}/perl-native/perl/${@get_perl_version(d)}/"
 
 cpan_do_configure () {
-	yes '' | perl ${EXTRA_PERLFLAGS} Makefile.PL INSTALLDIRS=vendor NO_PERLLOCAL=1 NO_PACKLIST=1 ${EXTRA_CPANFLAGS}
+	yes '' | perl ${EXTRA_PERLFLAGS} Makefile.PL INSTALLDIRS=vendor NO_PERLLOCAL=1 NO_PACKLIST=1 PERL=$(which perl) ${EXTRA_CPANFLAGS}
 
 	# Makefile.PLs can exit with success without generating a
 	# Makefile, e.g. in cases of missing configure time

meta/classes/crosssdk.bbclass

@@ -4,8 +4,8 @@ CLASSOVERRIDE = "class-crosssdk"
 MACHINEOVERRIDES = ""
 PACKAGE_ARCH = "${SDK_ARCH}"
 python () {
-	# set TUNE_PKGARCH to SDK_ARCH
-	d.setVar('TUNE_PKGARCH', d.getVar('SDK_ARCH'))
+    # set TUNE_PKGARCH to SDK_ARCH
+    d.setVar('TUNE_PKGARCH', d.getVar('SDK_ARCH'))
 }
 
 STAGING_BINDIR_TOOLCHAIN = "${STAGING_DIR_NATIVE}${bindir_native}/${TARGET_ARCH}${TARGET_VENDOR}-${TARGET_OS}"
@@ -21,10 +21,10 @@ TARGET_PREFIX = "${SDK_PREFIX}"
 TARGET_CC_ARCH = "${SDK_CC_ARCH}"
 TARGET_LD_ARCH = "${SDK_LD_ARCH}"
 TARGET_AS_ARCH = "${SDK_AS_ARCH}"
-TARGET_CPPFLAGS = "${BUILD_CPPFLAGS}"
-TARGET_CFLAGS = "${BUILD_CFLAGS}"
-TARGET_CXXFLAGS = "${BUILD_CXXFLAGS}"
-TARGET_LDFLAGS = "${BUILD_LDFLAGS}"
+TARGET_CPPFLAGS = ""
+TARGET_CFLAGS = ""
+TARGET_CXXFLAGS = ""
+TARGET_LDFLAGS = ""
 TARGET_FPU = ""
 
 

meta/classes/cve-check.bbclass

@@ -190,7 +190,8 @@ def check_cves(d, patched_cves):
         # Write the faux CSV file to be used with cve-check-tool
         fd, faux = tempfile.mkstemp(prefix="cve-faux-")
         with os.fdopen(fd, "w") as f:
-            f.write("%s,%s,%s," % (bpn, pv, cves))
+            for pn in bpn.split():
+                f.write("%s,%s,%s,\n" % (pn, pv, cves))
         cmd.append(faux)
 
         output = subprocess.check_output(cmd).decode("utf-8")

meta/classes/go.bbclass

@@ -8,7 +8,7 @@ GOROOT = "${STAGING_LIBDIR}/go"
 export GOROOT
 export GOROOT_FINAL = "${libdir}/go"
 
-DEPENDS_GOLANG_class-target = "virtual/${TARGET_PREFIX}go virtual/${TARGET_PREFIX}go-runtime"
+DEPENDS_GOLANG_class-target = "virtual/${TUNE_PKGARCH}-go virtual/${TARGET_PREFIX}go-runtime"
 DEPENDS_GOLANG_class-native = "go-native"
 DEPENDS_GOLANG_class-nativesdk = "virtual/${TARGET_PREFIX}go-crosssdk virtual/${TARGET_PREFIX}go-runtime"
 

meta/classes/gobject-introspection.bbclass

@@ -25,7 +25,7 @@ DEPENDS_append_class-native = " gobject-introspection-native"
 DEPENDS_append_class-nativesdk = " gobject-introspection-native"
 
 # This is used by introspection tools to find .gir includes
-export XDG_DATA_DIRS = "${STAGING_DATADIR}"
+export XDG_DATA_DIRS = "${STAGING_DATADIR}:${STAGING_LIBDIR}"
 
 do_configure_prepend_class-target () {
     # introspection.m4 pre-packaged with upstream tarballs does not yet

meta/classes/icecc.bbclass

@@ -33,7 +33,7 @@ BB_HASHBASE_WHITELIST += "ICECC_PARALLEL_MAKE ICECC_DISABLED ICECC_USER_PACKAGE_
     ICECC_CARET_WORKAROUND ICECC_CFLAGS ICECC_ENV_VERSION \
     ICECC_DEBUG ICECC_LOGFILE ICECC_REPEAT_RATE ICECC_PREFERRED_HOST \
     ICECC_CLANG_REMOTE_CPP ICECC_IGNORE_UNVERIFIED ICECC_TEST_SOCKET \
-    ICECC_ENV_DEBUG \
+    ICECC_ENV_DEBUG ICECC_SYSTEM_PACKAGE_BL ICECC_SYSTEM_CLASS_BL \
     "
 
 ICECC_ENV_EXEC ?= "${STAGING_BINDIR_NATIVE}/icecc-create-env"
@@ -61,6 +61,31 @@ CXXFLAGS += "${ICECC_CFLAGS}"
 # Debug flags when generating environments
 ICECC_ENV_DEBUG ??= ""
 
+# "system" recipe blacklist contains a list of packages that can not distribute
+# compile tasks for one reason or the other. When adding new entry, please
+# document why (how it failed) so that we can re-evaluate it later e.g. when
+# there is new version
+#
+# libgcc-initial - fails with CPP sanity check error if host sysroot contains
+#                  cross gcc built for another target tune/variant
+# target-sdk-provides-dummy - ${HOST_PREFIX} is empty which triggers the "NULL
+#                             prefix" error.
+ICECC_SYSTEM_PACKAGE_BL += "\
+    libgcc-initial \
+    target-sdk-provides-dummy \
+    "
+
+# "system" classes that should be blacklisted. When adding new entry, please
+# document why (how it failed) so that we can re-evaluate it later
+#
+# image - Image aren't compiling, but the testing framework for images captures
+#         PARALLEL_MAKE as part of the test environment. Many tests won't use
+#         icecream, but leaving the high level of parallelism can cause them to
+#         consume an unnecessary amount of resources.
+ICECC_SYSTEM_CLASS_BL += "\
+    image \
+    "
+
 def icecc_dep_prepend(d):
     # INHIBIT_DEFAULT_DEPS doesn't apply to the patch command.  Whether or  not
     # we need that built is the responsibility of the patch function / class, not
@@ -134,7 +159,7 @@ def use_icecc(bb,d):
 
     pn = d.getVar('PN')
 
-    system_class_blacklist = []
+    system_class_blacklist = (d.getVar('ICECC_SYSTEM_CLASS_BL') or "").split()
     user_class_blacklist = (d.getVar('ICECC_USER_CLASS_BL') or "none").split()
     package_class_blacklist = system_class_blacklist + user_class_blacklist
 
@@ -143,15 +168,7 @@ def use_icecc(bb,d):
             bb.debug(1, "%s: class %s found in blacklist, disable icecc" % (pn, black))
             return "no"
 
-    # "system" recipe blacklist contains a list of packages that can not distribute compile tasks
-    # for one reason or the other
-    # this is the old list (which doesn't seem to be valid anymore, because I was able to build
-    # all these with icecc enabled)
-    # system_package_blacklist = [ "glibc", "gcc", "bind", "u-boot", "dhcp-forwarder", "enchant", "connman", "orbit2" ]
-    # when adding new entry, please document why (how it failed) so that we can re-evaluate it later
-    # e.g. when there is new version
-    # building libgcc-initial with icecc fails with CPP sanity check error if host sysroot contains cross gcc built for another target tune/variant
-    system_package_blacklist = ["libgcc-initial"]
+    system_package_blacklist = (d.getVar('ICECC_SYSTEM_PACKAGE_BL') or "").split()
     user_package_blacklist = (d.getVar('ICECC_USER_PACKAGE_BL') or "").split()
     user_package_whitelist = (d.getVar('ICECC_USER_PACKAGE_WL') or "").split()
     package_blacklist = system_package_blacklist + user_package_blacklist
@@ -381,7 +398,7 @@ set_icecc_env() {
     # Don't let ccache find the icecream compiler links that have been created, otherwise
     # it can end up invoking icecream recursively.
     export CCACHE_PATH="$PATH"
-    export CCACHE_DISBALE="1"
+    export CCACHE_DISABLE="1"
 
     export ICECC_VERSION ICECC_CC ICECC_CXX
     export PATH="$ICE_PATH:$PATH"

meta/classes/image-buildinfo.bbclass

@@ -16,9 +16,8 @@ IMAGE_BUILDINFO_VARS ?= "DISTRO DISTRO_VERSION"
 IMAGE_BUILDINFO_FILE ??= "${sysconfdir}/build"
 
 # From buildhistory.bbclass
-def image_buildinfo_outputvars(vars, listvars, d): 
+def image_buildinfo_outputvars(vars, d):
     vars = vars.split()
-    listvars = listvars.split()
     ret = ""
     for var in vars:
         value = d.getVar(var) or ""
@@ -59,11 +58,12 @@ def buildinfo_target(d):
                 return ""
         # Single and list variables to be read
         vars = (d.getVar("IMAGE_BUILDINFO_VARS") or "")
-        listvars = (d.getVar("IMAGE_BUILDINFO_LVARS") or "")
-        return image_buildinfo_outputvars(vars, listvars, d)
+        return image_buildinfo_outputvars(vars, d)
 
 # Write build information to target filesystem
 python buildinfo () {
+    if not d.getVar('IMAGE_BUILDINFO_FILE'):
+        return
     with open(d.expand('${IMAGE_ROOTFS}${IMAGE_BUILDINFO_FILE}'), 'w') as build:
         build.writelines((
             '''-----------------------

meta/classes/image.bbclass

@@ -24,7 +24,7 @@ POPULATE_SDK_POST_TARGET_COMMAND += "rootfs_sysroot_relativelinks; "
 LICENSE ?= "MIT"
 PACKAGES = ""
 DEPENDS += "${@' '.join(["%s-qemuwrapper-cross" % m for m in d.getVar("MULTILIB_VARIANTS").split()])} qemuwrapper-cross depmodwrapper-cross cross-localedef-native"
-RDEPENDS += "${PACKAGE_INSTALL} ${LINGUAS_INSTALL}"
+RDEPENDS += "${PACKAGE_INSTALL} ${LINGUAS_INSTALL} ${IMAGE_INSTALL_DEBUGFS}"
 RRECOMMENDS += "${PACKAGE_INSTALL_ATTEMPTONLY}"
 PATH_prepend = "${@":".join(all_multilib_tune_values(d, 'STAGING_BINDIR_CROSS').split())}:"
 
@@ -176,6 +176,11 @@ IMAGE_LINGUAS ?= "de-de fr-fr en-gb"
 
 LINGUAS_INSTALL ?= "${@" ".join(map(lambda s: "locale-base-%s" % s, d.getVar('IMAGE_LINGUAS').split()))}"
 
+python () {
+    if not bb.utils.contains('DISTRO_FEATURES', 'libc-charsets libc-locale-code libc-locales', True, False, d):
+        d.setVar('IMAGE_LINGUAS', '')
+}
+
 # Prefer image, but use the fallback files for lookups if the image ones
 # aren't yet available.
 PSEUDO_PASSWD = "${IMAGE_ROOTFS}:${STAGING_DIR_NATIVE}"
@@ -497,10 +502,10 @@ python () {
         d.setVarFlag(task, 'fakeroot', '1')
 
         d.appendVarFlag(task, 'prefuncs', ' ' + debug + ' set_image_size')
-        d.prependVarFlag(task, 'postfuncs', ' create_symlinks')
+        d.prependVarFlag(task, 'postfuncs', 'create_symlinks ')
         d.appendVarFlag(task, 'subimages', ' ' + ' '.join(subimages))
         d.appendVarFlag(task, 'vardeps', ' ' + ' '.join(vardeps))
-        d.appendVarFlag(task, 'vardepsexclude', 'DATETIME DATE ' + ' '.join(vardepsexclude))
+        d.appendVarFlag(task, 'vardepsexclude', ' DATETIME DATE ' + ' '.join(vardepsexclude))
 
         bb.debug(2, "Adding task %s before %s, after %s" % (task, 'do_image_complete', after))
         bb.build.addtask(task, 'do_image_complete', after, d)

meta/classes/image_types.bbclass

@@ -239,6 +239,7 @@ EXTRA_IMAGECMD_ext4 ?= "-i 4096"
 EXTRA_IMAGECMD_btrfs ?= "-n 4096"
 EXTRA_IMAGECMD_f2fs ?= ""
 
+do_image_cpio[depends] += "cpio-native:do_populate_sysroot"
 do_image_jffs2[depends] += "mtd-utils-native:do_populate_sysroot"
 do_image_cramfs[depends] += "util-linux-native:do_populate_sysroot"
 do_image_ext2[depends] += "e2fsprogs-native:do_populate_sysroot"
@@ -308,7 +309,7 @@ CONVERSION_DEPENDS_lzo = "lzop-native"
 CONVERSION_DEPENDS_zip = "zip-native"
 CONVERSION_DEPENDS_sum = "mtd-utils-native"
 CONVERSION_DEPENDS_bmap = "bmap-tools-native"
-CONVERSION_DEPENDS_u-boot = "u-boot-mkimage-native"
+CONVERSION_DEPENDS_u-boot = "u-boot-tools-native"
 CONVERSION_DEPENDS_vmdk = "qemu-native"
 CONVERSION_DEPENDS_vdi = "qemu-native"
 CONVERSION_DEPENDS_qcow2 = "qemu-native"

meta/classes/insane.bbclass

@@ -111,7 +111,7 @@ def package_qa_check_rpath(file,name, d, elf, messages):
     phdrs = elf.run_objdump("-p", d)
 
     import re
-    rpath_re = re.compile("\s+RPATH\s+(.*)")
+    rpath_re = re.compile(r"\s+RPATH\s+(.*)")
     for line in phdrs.split("\n"):
         m = rpath_re.match(line)
         if m:
@@ -140,7 +140,7 @@ def package_qa_check_useless_rpaths(file, name, d, elf, messages):
     phdrs = elf.run_objdump("-p", d)
 
     import re
-    rpath_re = re.compile("\s+RPATH\s+(.*)")
+    rpath_re = re.compile(r"\s+RPATH\s+(.*)")
     for line in phdrs.split("\n"):
         m = rpath_re.match(line)
         if m:
@@ -203,8 +203,8 @@ def package_qa_check_libdir(d):
     # The re's are purposely fuzzy, as some there are some .so.x.y.z files
     # that don't follow the standard naming convention. It checks later
     # that they are actual ELF files
-    lib_re = re.compile("^/lib.+\.so(\..+)?$")
-    exec_re = re.compile("^%s.*/lib.+\.so(\..+)?$" % exec_prefix)
+    lib_re = re.compile(r"^/lib.+\.so(\..+)?$")
+    exec_re = re.compile(r"^%s.*/lib.+\.so(\..+)?$" % exec_prefix)
 
     for root, dirs, files in os.walk(pkgdest):
         if root == pkgdest:
@@ -302,15 +302,15 @@ def package_qa_check_arch(path,name,d, elf, messages):
     # Check the architecture and endiannes of the binary
     is_32 = (("virtual/kernel" in provides) or bb.data.inherits_class("module", d)) and \
             (target_os == "linux-gnux32" or target_os == "linux-muslx32" or \
-            target_os == "linux-gnu_ilp32" or re.match('mips64.*32', d.getVar('DEFAULTTUNE')))
+            target_os == "linux-gnu_ilp32" or re.match(r'mips64.*32', d.getVar('DEFAULTTUNE')))
     is_bpf = (oe.qa.elf_machine_to_string(elf.machine()) == "BPF")
     if not ((machine == elf.machine()) or is_32 or is_bpf):
         package_qa_add_message(messages, "arch", "Architecture did not match (%s, expected %s) on %s" % \
                  (oe.qa.elf_machine_to_string(elf.machine()), oe.qa.elf_machine_to_string(machine), package_qa_clean_path(path,d)))
-    elif not ((bits == elf.abiSize()) or is_32):
+    elif not ((bits == elf.abiSize()) or is_32 or is_bpf):
         package_qa_add_message(messages, "arch", "Bit size did not match (%d to %d) %s on %s" % \
                  (bits, elf.abiSize(), bpn, package_qa_clean_path(path,d)))
-    elif not littleendian == elf.isLittleEndian():
+    elif not ((littleendian == elf.isLittleEndian()) or is_bpf):
         package_qa_add_message(messages, "arch", "Endiannes did not match (%d to %d) on %s" % \
                  (littleendian, elf.isLittleEndian(), package_qa_clean_path(path,d)))
 
@@ -342,7 +342,7 @@ def package_qa_textrel(path, name, d, elf, messages):
     sane = True
 
     import re
-    textrel_re = re.compile("\s+TEXTREL\s+")
+    textrel_re = re.compile(r"\s+TEXTREL\s+")
     for line in phdrs.split("\n"):
         if textrel_re.match(line):
             sane = False
@@ -383,7 +383,7 @@ def package_qa_hash_style(path, name, d, elf, messages):
             sane = True
 
     if has_syms and not sane:
-        package_qa_add_message(messages, "ldflags", "No GNU_HASH in the elf binary: '%s'" % path)
+        package_qa_add_message(messages, "ldflags", "No GNU_HASH in the ELF binary %s, didn't pass LDFLAGS?" % path)
 
 
 QAPATHTEST[buildpaths] = "package_qa_check_buildpaths"
@@ -952,7 +952,7 @@ python do_package_qa () {
 
     import re
     # The package name matches the [a-z0-9.+-]+ regular expression
-    pkgname_pattern = re.compile("^[a-z0-9.+-]+$")
+    pkgname_pattern = re.compile(r"^[a-z0-9.+-]+$")
 
     taskdepdata = d.getVar("BB_TASKDEPDATA", False)
     taskdeps = set()
@@ -1017,6 +1017,13 @@ do_package_qa[vardepsexclude] = "BB_TASKDEPDATA"
 do_package_qa[rdeptask] = "do_packagedata"
 addtask do_package_qa after do_packagedata do_package before do_build
 
+# Add the package specific INSANE_SKIPs to the sstate dependencies
+python() {
+    pkgs = (d.getVar('PACKAGES') or '').split()
+    for pkg in pkgs:
+        d.appendVarFlag("do_package_qa", "vardeps", " INSANE_SKIP_{}".format(pkg))
+}
+
 SSTATETASKS += "do_package_qa"
 do_package_qa[sstate-inputdirs] = ""
 do_package_qa[sstate-outputdirs] = ""
@@ -1160,7 +1167,7 @@ python () {
     if pn in overrides:
         msg = 'Recipe %s has PN of "%s" which is in OVERRIDES, this can result in unexpected behaviour.' % (d.getVar("FILE"), pn)
         package_qa_handle_error("pn-overrides", msg, d)
-    prog = re.compile('[A-Z]')
+    prog = re.compile(r'[A-Z]')
     if prog.search(pn):
         package_qa_handle_error("uppercase-pn", 'PN: %s is upper case, this can result in unexpected behavior.' % pn, d)
 

meta/classes/kernel-fitimage.bbclass

@@ -4,7 +4,7 @@ python __anonymous () {
     kerneltypes = d.getVar('KERNEL_IMAGETYPES') or ""
     if 'fitImage' in kerneltypes.split():
         depends = d.getVar("DEPENDS")
-        depends = "%s u-boot-mkimage-native dtc-native" % depends
+        depends = "%s u-boot-tools-native dtc-native" % depends
         d.setVar("DEPENDS", depends)
 
         uarch = d.getVar("UBOOT_ARCH")
@@ -19,9 +19,9 @@ python __anonymous () {
         else:
             replacementtype = "zImage"
 
-	# Override KERNEL_IMAGETYPE_FOR_MAKE variable, which is internal
-	# to kernel.bbclass . We have to override it, since we pack zImage
-	# (at least for now) into the fitImage .
+        # Override KERNEL_IMAGETYPE_FOR_MAKE variable, which is internal
+        # to kernel.bbclass . We have to override it, since we pack zImage
+        # (at least for now) into the fitImage .
         typeformake = d.getVar("KERNEL_IMAGETYPE_FOR_MAKE") or ""
         if 'fitImage' in typeformake.split():
             d.setVar('KERNEL_IMAGETYPE_FOR_MAKE', typeformake.replace('fitImage', replacementtype))
@@ -35,7 +35,7 @@ python __anonymous () {
         # the fitImage:
         if d.getVar('UBOOT_SIGN_ENABLE') == "1":
             uboot_pn = d.getVar('PREFERRED_PROVIDER_u-boot') or 'u-boot'
-            d.appendVarFlag('do_assemble_fitimage', 'depends', ' %s:do_deploy' % uboot_pn)
+            d.appendVarFlag('do_assemble_fitimage', 'depends', ' %s:do_populate_sysroot' % uboot_pn)
 }
 
 # Options for the device tree compiler passed to mkimage '-D' feature:
@@ -456,10 +456,17 @@ fitimage_assemble() {
 	# Step 7: Sign the image and add public key to U-Boot dtb
 	#
 	if [ "x${UBOOT_SIGN_ENABLE}" = "x1" ] ; then
+		add_key_to_u_boot=""
+		if [ -n "${UBOOT_DTB_BINARY}" ]; then
+			# The u-boot.dtb is a symlink to UBOOT_DTB_IMAGE, so we need copy
+			# both of them, and don't dereference the symlink.
+			cp -P ${STAGING_DATADIR}/u-boot*.dtb ${B}
+			add_key_to_u_boot="-K ${B}/${UBOOT_DTB_BINARY}"
+		fi
 		uboot-mkimage \
 			${@'-D "${UBOOT_MKIMAGE_DTCOPTS}"' if len('${UBOOT_MKIMAGE_DTCOPTS}') else ''} \
 			-F -k "${UBOOT_SIGN_KEYDIR}" \
-			${@'-K "${DEPLOY_DIR_IMAGE}/${UBOOT_DTB_BINARY}"' if len('${UBOOT_DTB_BINARY}') else ''} \
+			$add_key_to_u_boot \
 			-r arch/${ARCH}/boot/${2}
 	fi
 }
@@ -505,5 +512,11 @@ kernel_do_deploy_append() {
 			install -m 0644 ${B}/arch/${ARCH}/boot/fitImage-${INITRAMFS_IMAGE} ${DEPLOYDIR}/fitImage-${INITRAMFS_IMAGE_NAME}-${KERNEL_FIT_NAME}.bin
 			ln -snf fitImage-${INITRAMFS_IMAGE_NAME}-${KERNEL_FIT_NAME}.bin ${DEPLOYDIR}/fitImage-${INITRAMFS_IMAGE_NAME}-${KERNEL_FIT_LINK_NAME}
 		fi
+		if [ "${UBOOT_SIGN_ENABLE}" = "1" -a -n "${UBOOT_DTB_BINARY}" ] ; then
+			# UBOOT_DTB_IMAGE is a realfile, but we can't use
+			# ${UBOOT_DTB_IMAGE} since it contains ${PV} which is aimed
+			# for u-boot, but we are in kernel env now.
+			install -m 0644 ${B}/u-boot-${MACHINE}*.dtb ${DEPLOYDIR}/
+		fi
 	fi
 }

meta/classes/kernel-uimage.bbclass

@@ -3,7 +3,7 @@ inherit kernel-uboot
 python __anonymous () {
     if "uImage" in d.getVar('KERNEL_IMAGETYPES'):
         depends = d.getVar("DEPENDS")
-        depends = "%s u-boot-mkimage-native" % depends
+        depends = "%s u-boot-tools-native" % depends
         d.setVar("DEPENDS", depends)
 
         # Override KERNEL_IMAGETYPE_FOR_MAKE variable, which is internal

meta/classes/kernel.bbclass

@@ -157,14 +157,14 @@ PACKAGES_DYNAMIC += "^${KERNEL_PACKAGE_NAME}-firmware-.*"
 export OS = "${TARGET_OS}"
 export CROSS_COMPILE = "${TARGET_PREFIX}"
 export KBUILD_BUILD_VERSION = "1"
-export KBUILD_BUILD_USER = "oe-user"
-export KBUILD_BUILD_HOST = "oe-host"
+export KBUILD_BUILD_USER ?= "oe-user"
+export KBUILD_BUILD_HOST ?= "oe-host"
 
 KERNEL_RELEASE ?= "${KERNEL_VERSION}"
 
 # The directory where built kernel lies in the kernel tree
 KERNEL_OUTPUT_DIR ?= "arch/${ARCH}/boot"
-KERNEL_IMAGEDEST = "boot"
+KERNEL_IMAGEDEST ?= "boot"
 
 #
 # configuration
@@ -492,7 +492,7 @@ sysroot_stage_all () {
 	:
 }
 
-KERNEL_CONFIG_COMMAND ?= "oe_runmake_call -C ${S} CC="${KERNEL_CC}" O=${B} oldnoconfig"
+KERNEL_CONFIG_COMMAND ?= "oe_runmake_call -C ${S} CC="${KERNEL_CC}" O=${B} olddefconfig || oe_runmake -C ${S} O=${B} CC="${KERNEL_CC}" oldnoconfig"
 
 python check_oldest_kernel() {
     oldest_kernel = d.getVar('OLDEST_KERNEL')

meta/classes/libc-package.bbclass

@@ -39,9 +39,7 @@ python __anonymous () {
                 break
 
     # try to fix disable charsets/locales/locale-code compile fail
-    if bb.utils.contains('DISTRO_FEATURES', 'libc-charsets', True, False, d) and \
-            bb.utils.contains('DISTRO_FEATURES', 'libc-locales', True, False, d) and \
-            bb.utils.contains('DISTRO_FEATURES', 'libc-locale-code', True, False, d):
+    if bb.utils.contains('DISTRO_FEATURES', 'libc-charsets libc-locales libc-locale-code', True, False, d):
         d.setVar('PACKAGE_NO_GCONV', '0')
     else:
         d.setVar('PACKAGE_NO_GCONV', '1')
@@ -49,13 +47,7 @@ python __anonymous () {
 
 OVERRIDES_append = ":${TARGET_ARCH}-${TARGET_OS}"
 
-locale_base_postinst() {
-#!/bin/sh
-
-if [ "x$D" != "x" ]; then
-	exit 1
-fi
-
+locale_base_postinst_ontarget() {
 localedef --inputfile=${datadir}/i18n/locales/%s --charmap=%s %s
 }
 
@@ -215,7 +207,7 @@ python package_do_split_gconvs () {
     def output_locale_source(name, pkgname, locale, encoding):
         d.setVar('RDEPENDS_%s' % pkgname, '%slocaledef %s-localedata-%s %s-charmap-%s' % \
         (mlprefix, mlprefix+bpn, legitimize_package_name(locale), mlprefix+bpn, legitimize_package_name(encoding)))
-        d.setVar('pkg_postinst_%s' % pkgname, d.getVar('locale_base_postinst') \
+        d.setVar('pkg_postinst_ontarget_%s' % pkgname, d.getVar('locale_base_postinst_ontarget') \
         % (locale, encoding, locale))
         d.setVar('pkg_postrm_%s' % pkgname, d.getVar('locale_base_postrm') % \
         (locale, encoding, locale))
@@ -242,6 +234,8 @@ python package_do_split_gconvs () {
         if use_cross_localedef == "1":
             target_arch = d.getVar('TARGET_ARCH')
             locale_arch_options = { \
+                "arc":     " --uint32-align=4 --little-endian ", \
+                "arceb":   " --uint32-align=4 --big-endian ",    \
                 "arm":     " --uint32-align=4 --little-endian ", \
                 "armeb":   " --uint32-align=4 --big-endian ",    \
                 "aarch64": " --uint32-align=4 --little-endian ",    \

meta/classes/license.bbclass

@@ -309,15 +309,15 @@ def incompatible_license(d, dont_want_licenses, package=None):
     except oe.license.LicenseError as exc:
         bb.fatal('%s: %s' % (d.getVar('P'), exc))
     return any(not oe.license.license_ok(canonical_license(d, l), \
-		dont_want_licenses) for l in licenses)
+               dont_want_licenses) for l in licenses)
 
 def check_license_flags(d):
     """
     This function checks if a recipe has any LICENSE_FLAGS that
     aren't whitelisted.
 
-    If it does, it returns the first LICENSE_FLAGS item missing from the
-    whitelist, or all of the LICENSE_FLAGS if there is no whitelist.
+    If it does, it returns the all LICENSE_FLAGS missing from the whitelist, or
+    all of the LICENSE_FLAGS if there is no whitelist.
 
     If everything is is properly whitelisted, it returns None.
     """
@@ -354,22 +354,23 @@ def check_license_flags(d):
         return False
 
     def all_license_flags_match(license_flags, whitelist):
-        """ Return first unmatched flag, None if all flags match """
+        """ Return all unmatched flags, None if all flags match """
         pn = d.getVar('PN')
         split_whitelist = whitelist.split()
+        flags = []
         for flag in license_flags.split():
             if not license_flag_matches(flag, split_whitelist, pn):
-                return flag
-        return None
+                flags.append(flag)
+        return flags if flags else None
 
     license_flags = d.getVar('LICENSE_FLAGS')
     if license_flags:
         whitelist = d.getVar('LICENSE_FLAGS_WHITELIST')
         if not whitelist:
-            return license_flags
-        unmatched_flag = all_license_flags_match(license_flags, whitelist)
-        if unmatched_flag:
-            return unmatched_flag
+            return license_flags.split()
+        unmatched_flags = all_license_flags_match(license_flags, whitelist)
+        if unmatched_flags:
+            return unmatched_flags
     return None
 
 def check_license_format(d):

meta/classes/license_image.bbclass

@@ -32,10 +32,10 @@ python license_create_manifest() {
 
     rootfs_license_manifest = os.path.join(d.getVar('LICENSE_DIRECTORY'),
                         d.getVar('IMAGE_NAME'), 'license.manifest')
-    write_license_files(d, rootfs_license_manifest, pkg_dic)
+    write_license_files(d, rootfs_license_manifest, pkg_dic, rootfs=True)
 }
 
-def write_license_files(d, license_manifest, pkg_dic):
+def write_license_files(d, license_manifest, pkg_dic, rootfs=True):
     import re
 
     bad_licenses = (d.getVar("INCOMPATIBLE_LICENSE") or "").split()
@@ -52,8 +52,8 @@ def write_license_files(d, license_manifest, pkg_dic):
                 except oe.license.LicenseError as exc:
                     bb.fatal('%s: %s' % (d.getVar('P'), exc))
             else:
-                pkg_dic[pkg]["LICENSES"] = re.sub('[|&()*]', ' ', pkg_dic[pkg]["LICENSE"])
-                pkg_dic[pkg]["LICENSES"] = re.sub('  *', ' ', pkg_dic[pkg]["LICENSES"])
+                pkg_dic[pkg]["LICENSES"] = re.sub(r'[|&()*]', ' ', pkg_dic[pkg]["LICENSE"])
+                pkg_dic[pkg]["LICENSES"] = re.sub(r'  *', ' ', pkg_dic[pkg]["LICENSES"])
                 pkg_dic[pkg]["LICENSES"] = pkg_dic[pkg]["LICENSES"].split()
 
             if not "IMAGE_MANIFEST" in pkg_dic[pkg]:
@@ -78,7 +78,7 @@ def write_license_files(d, license_manifest, pkg_dic):
             for lic in pkg_dic[pkg]["LICENSES"]:
                 lic_file = os.path.join(d.getVar('LICENSE_DIRECTORY'),
                                         pkg_dic[pkg]["PN"], "generic_%s" % 
-                                        re.sub('\+', '', lic))
+                                        re.sub(r'\+', '', lic))
                 # add explicity avoid of CLOSED license because isn't generic
                 if lic == "CLOSED":
                    continue
@@ -94,7 +94,7 @@ def write_license_files(d, license_manifest, pkg_dic):
     # With both options set we see a .5 M increase in core-image-minimal
     copy_lic_manifest = d.getVar('COPY_LIC_MANIFEST')
     copy_lic_dirs = d.getVar('COPY_LIC_DIRS')
-    if copy_lic_manifest == "1":
+    if rootfs and copy_lic_manifest == "1":
         rootfs_license_dir = os.path.join(d.getVar('IMAGE_ROOTFS'), 
                                 'usr', 'share', 'common-licenses')
         bb.utils.mkdirhier(rootfs_license_dir)
@@ -119,14 +119,14 @@ def write_license_files(d, license_manifest, pkg_dic):
                     pkg_license = os.path.join(pkg_license_dir, lic)
                     pkg_rootfs_license = os.path.join(pkg_rootfs_license_dir, lic)
 
-                    if re.match("^generic_.*$", lic):
+                    if re.match(r"^generic_.*$", lic):
                         generic_lic = canonical_license(d,
-                                re.search("^generic_(.*)$", lic).group(1))
+                                re.search(r"^generic_(.*)$", lic).group(1))
 
                         # Do not copy generic license into package if isn't
                         # declared into LICENSES of the package.
-                        if not re.sub('\+$', '', generic_lic) in \
-                                [re.sub('\+', '', lic) for lic in \
+                        if not re.sub(r'\+$', '', generic_lic) in \
+                                [re.sub(r'\+', '', lic) for lic in \
                                  pkg_manifest_licenses]:
                             continue
 
@@ -146,6 +146,13 @@ def write_license_files(d, license_manifest, pkg_dic):
                             continue
 
                         os.link(pkg_license, pkg_rootfs_license)
+            # Fixup file ownership
+            for walkroot, dirs, files in os.walk(rootfs_license_dir):
+                for f in files:
+                    os.lchown(os.path.join(walkroot, f), 0, 0)
+                for dir in dirs:
+                    os.lchown(os.path.join(walkroot, dir), 0, 0)
+
 
 
 def license_deployed_manifest(d):
@@ -176,7 +183,7 @@ def license_deployed_manifest(d):
                                     d.getVar('IMAGE_NAME'))
     bb.utils.mkdirhier(lic_manifest_dir)
     image_license_manifest = os.path.join(lic_manifest_dir, 'image_license.manifest')
-    write_license_files(d, image_license_manifest, man_dic)
+    write_license_files(d, image_license_manifest, man_dic, rootfs=False)
 
 def get_deployed_dependencies(d):
     """

meta/classes/linux-kernel-base.bbclass

@@ -34,8 +34,8 @@ def get_kernelversion_file(p):
         return None
 
 def linux_module_packages(s, d):
-	suffix = ""
-	return " ".join(map(lambda s: "kernel-module-%s%s" % (s.lower().replace('_', '-').replace('@', '+'), suffix), s.split()))
+    suffix = ""
+    return " ".join(map(lambda s: "kernel-module-%s%s" % (s.lower().replace('_', '-').replace('@', '+'), suffix), s.split()))
 
 # that's all
 

meta/classes/meson.bbclass

@@ -52,8 +52,12 @@ def meson_cpu_family(var, d):
     arch = d.getVar(var)
     if arch == 'powerpc':
         return 'ppc'
+    elif arch == 'powerpc64':
+        return 'ppc64'
     elif arch == 'mipsel':
         return 'mips'
+    elif arch == 'mips64el':
+        return 'mips64'
     elif re.match(r"i[3-6]86", arch):
         return "x86"
     else:

meta/classes/metadata_scm.bbclass

@@ -3,55 +3,15 @@ METADATA_REVISION ?= "${@base_detect_revision(d)}"
 
 def base_detect_revision(d):
     path = base_get_scmbasepath(d)
-
-    scms = [base_get_metadata_git_revision]
-
-    for scm in scms:
-        rev = scm(path, d)
-        if rev != "<unknown>":
-            return rev
-
-    return "<unknown>"
+    return base_get_metadata_git_revision(path, d)
 
 def base_detect_branch(d):
     path = base_get_scmbasepath(d)
-
-    scms = [base_get_metadata_git_branch]
-
-    for scm in scms:
-        rev = scm(path, d)
-        if rev != "<unknown>":
-            return rev.strip()
-
-    return "<unknown>"
+    return base_get_metadata_git_branch(path, d)
 
 def base_get_scmbasepath(d):
     return os.path.join(d.getVar('COREBASE'), 'meta')
 
-def base_get_metadata_monotone_branch(path, d):
-    monotone_branch = "<unknown>"
-    try:
-        with open("%s/_MTN/options" % path) as f:
-            monotone_branch = f.read().strip()
-            if monotone_branch.startswith( "database" ):
-                monotone_branch_words = monotone_branch.split()
-                monotone_branch = monotone_branch_words[ monotone_branch_words.index( "branch" )+1][1:-1]
-    except:
-        pass
-    return monotone_branch
-
-def base_get_metadata_monotone_revision(path, d):
-    monotone_revision = "<unknown>"
-    try:
-        with open("%s/_MTN/revision" % path) as f:
-            monotone_revision = f.read().strip()
-            if monotone_revision.startswith( "format_version" ):
-                monotone_revision_words = monotone_revision.split()
-                monotone_revision = monotone_revision_words[ monotone_revision_words.index( "old_revision" )+1][1:-1]
-    except IOError:
-        pass
-    return monotone_revision
-
 def base_get_metadata_svn_revision(path, d):
     # This only works with older subversion. For newer versions 
     # this function will need to be fixed by someone interested

meta/classes/module.bbclass

@@ -48,6 +48,7 @@ module_do_compile() {
 module_do_install() {
 	unset CFLAGS CPPFLAGS CXXFLAGS LDFLAGS
 	oe_runmake DEPMOD=echo MODLIB="${D}${nonarch_base_libdir}/modules/${KERNEL_VERSION}" \
+	           INSTALL_FW_PATH="${D}${nonarch_base_libdir}/firmware" \
 	           CC="${KERNEL_CC}" LD="${KERNEL_LD}" \
 	           O=${STAGING_KERNEL_BUILDDIR} \
 	           ${MODULES_INSTALL_TARGET}

meta/classes/multilib_script.bbclass

@@ -24,9 +24,11 @@ python () {
 
     for entry in (d.getVar("MULTILIB_SCRIPTS", False) or "").split():
        pkg, script = entry.split(":")
+       epkg = d.expand(pkg)
        scriptname = os.path.basename(script)
-       d.appendVar("ALTERNATIVE_" + pkg, scriptname + " ")
+       d.appendVar("ALTERNATIVE_" + epkg, " " + scriptname + " ")
        d.setVarFlag("ALTERNATIVE_LINK_NAME", scriptname, script)
        d.setVarFlag("ALTERNATIVE_TARGET", scriptname, script + "-${MULTILIB_SUFFIX}")
        d.appendVar("multilibscript_rename",  "\n	mv ${PKGD}" + script + " ${PKGD}" + script + "-${MULTILIB_SUFFIX}")
+       d.appendVar("FILES_" + epkg, " " + script + "-${MULTILIB_SUFFIX}")
 }

meta/classes/npm.bbclass

@@ -10,7 +10,7 @@ def node_pkgname(d):
 
 NPMPN ?= "${@node_pkgname(d)}"
 
-NPM_INSTALLDIR = "${D}${libdir}/node/${NPMPN}"
+NPM_INSTALLDIR = "${libdir}/node/${NPMPN}"
 
 # function maps arch names to npm arch names
 def npm_oe_arch_map(target_arch, d):
@@ -22,7 +22,7 @@ def npm_oe_arch_map(target_arch, d):
     return target_arch
 
 NPM_ARCH ?= "${@npm_oe_arch_map(d.getVar('TARGET_ARCH'), d)}"
-NPM_INSTALL_DEV = "0"
+NPM_INSTALL_DEV ?= "0"
 
 npm_do_compile() {
 	# Copy in any additionally fetched modules
@@ -52,9 +52,10 @@ npm_do_install() {
 	# changing the home directory to the working directory, the .npmrc will
 	# be created in this directory
 	export HOME=${WORKDIR}
-	mkdir -p ${NPM_INSTALLDIR}/
+	mkdir -p ${D}${libdir}/node_modules
 	npm pack .
 	npm install --prefix ${D}${prefix} -g --arch=${NPM_ARCH} --target_arch=${NPM_ARCH} --production --no-registry ${NPMPN}-${PV}.tgz
+	mv ${D}${libdir}/node_modules ${D}${libdir}/node
 	if [ -d ${D}${prefix}/etc ] ; then
 		# This will be empty
 		rmdir ${D}${prefix}/etc
@@ -62,13 +63,13 @@ npm_do_install() {
 }
 
 python populate_packages_prepend () {
-    instdir = d.expand('${D}${libdir}/node_modules/${NPMPN}')
+    instdir = d.expand('${D}${NPM_INSTALLDIR}')
     extrapackages = oe.package.npm_split_package_dirs(instdir)
     pkgnames = extrapackages.keys()
     d.prependVar('PACKAGES', '%s ' % ' '.join(pkgnames))
     for pkgname in pkgnames:
         pkgrelpath, pdata = extrapackages[pkgname]
-        pkgpath = '${libdir}/node_modules/${NPMPN}/' + pkgrelpath
+        pkgpath = '${NPM_INSTALLDIR}/' + pkgrelpath
         # package names can't have underscores but npm packages sometimes use them
         oe_pkg_name = pkgname.replace('_', '-')
         expanded_pkgname = d.expand(oe_pkg_name)
@@ -84,7 +85,7 @@ python populate_packages_prepend () {
 }
 
 FILES_${PN} += " \
-    ${libdir}/node_modules/${NPMPN} \
+    ${NPM_INSTALLDIR} \
 "
 
 EXPORT_FUNCTIONS do_compile do_install

meta/classes/perl-version.bbclass

@@ -13,7 +13,7 @@ def get_perl_version(d):
         return None
     l = f.readlines();
     f.close();
-    r = re.compile("^version='(\d*\.\d*\.\d*)'")
+    r = re.compile(r"^version='(\d*\.\d*\.\d*)'")
     for s in l:
         m = r.match(s)
         if m:

meta/classes/ptest.bbclass

@@ -3,6 +3,9 @@ DESCRIPTION_${PN}-ptest ?= "${DESCRIPTION}  \
 This package contains a test directory ${PTEST_PATH} for package test purposes."
 
 PTEST_PATH ?= "${libdir}/${BPN}/ptest"
+PTEST_BUILD_HOST_FILES ?= "Makefile"
+PTEST_BUILD_HOST_PATTERN ?= ""
+
 FILES_${PN}-ptest = "${PTEST_PATH}"
 SECTION_${PN}-ptest = "devel"
 ALLOW_EMPTY_${PN}-ptest = "1"
@@ -45,6 +48,21 @@ do_install_ptest_base() {
     fi
     do_install_ptest
     chown -R root:root ${D}${PTEST_PATH}
+
+    # Strip build host paths from any installed Makefile
+    for filename in ${PTEST_BUILD_HOST_FILES}; do
+        for installed_ptest_file in $(find ${D}${PTEST_PATH} -type f -name $filename); do
+            bbnote "Stripping host paths from: $installed_ptest_file"
+            sed -e 's#${HOSTTOOLS_DIR}/*##g' \
+                -e 's#${WORKDIR}/*=#.=#g' \
+                -e 's#${WORKDIR}/*##g' \
+                -i $installed_ptest_file
+            if [ -n "${PTEST_BUILD_HOST_PATTERN}" ]; then
+               sed -E '/${PTEST_BUILD_HOST_PATTERN}/d' \
+                   -i $installed_ptest_file
+            fi
+        done
+    done
 }
 
 do_configure_ptest_base[dirs] = "${B}"
@@ -59,6 +77,7 @@ addtask install_ptest_base   after do_install   before do_package do_populate_sy
 python () {
     if not bb.data.inherits_class('native', d) and not bb.data.inherits_class('cross', d):
         d.setVarFlag('do_install_ptest_base', 'fakeroot', '1')
+        d.setVarFlag('do_install_ptest_base', 'umask', '022')
 
     # Remove all '*ptest_base' tasks when ptest is not enabled
     if not(d.getVar('PTEST_ENABLED') == "1"):

meta/classes/reproducible_build.bbclass

@@ -22,9 +22,15 @@
 # 3. Use the mtime of "known" files such as NEWS, CHANGLELOG, ...
 #    This works for well-kept repositories distributed via tarball.
 #
-# 4. If the above steps fail, use the modification time of the youngest file in the source tree.
+# 4. Use the modification time of the youngest file in the source tree, if there is one.
+#    This will be the newest file from the distribution tarball, if any.
+#
+# 5. Fall back to a fixed timestamp.
+#
+# Once the value of SOURCE_DATE_EPOCH is determined, it is stored in the recipe's SDE_FILE.
+# If none of these mechanisms are suitable, replace the do_deploy_source_date_epoch task
+# with recipe-specific functionality to write the appropriate SOURCE_DATE_EPOCH into the SDE_FILE.
 #
-# Once the value of SOURCE_DATE_EPOCH is determined, it is stored in the recipe's ${SDE_FILE}.
 # If this file is found by other tasks, the value is exported in the SOURCE_DATE_EPOCH variable.
 # SOURCE_DATE_EPOCH is set for all tasks that might use it (do_configure, do_compile, do_package, ...)
 
@@ -101,15 +107,15 @@ def get_source_date_epoch_from_git(d, sourcedir):
     return source_date_epoch
 
 def get_source_date_epoch_from_youngest_file(d, sourcedir):
+    if sourcedir == d.getVar('WORKDIR'):
+       # These sources are almost certainly not from a tarball
+       return None
+
     # Do it the hard way: check all files and find the youngest one...
     source_date_epoch = None
     newest_file = None
-    # Just in case S = WORKDIR
-    exclude = set(["build", "image", "license-destdir", "patches", "pseudo",
-                   "recipe-sysroot", "recipe-sysroot-native", "sysroot-destdir", "temp"])
     for root, dirs, files in os.walk(sourcedir, topdown=True):
         files = [f for f in files if not f[0] == '.']
-        dirs[:] = [d for d in dirs if d not in exclude]
 
         for fname in files:
             filename = os.path.join(root, fname)
@@ -125,6 +131,10 @@ def get_source_date_epoch_from_youngest_file(d, sourcedir):
         bb.debug(1, "Newest file found: %s" % newest_file)
     return source_date_epoch
 
+def fixed_source_date_epoch():
+    bb.debug(1, "No tarball or git repo found to determine SOURCE_DATE_EPOCH")
+    return 0
+
 python do_create_source_date_epoch_stamp() {
     epochfile = d.getVar('SDE_FILE')
     if os.path.isfile(epochfile):
@@ -136,11 +146,8 @@ python do_create_source_date_epoch_stamp() {
         get_source_date_epoch_from_git(d, sourcedir) or
         get_source_date_epoch_from_known_files(d, sourcedir) or
         get_source_date_epoch_from_youngest_file(d, sourcedir) or
-        0       # Last resort
+        fixed_source_date_epoch()       # Last resort
     )
-    if source_date_epoch == 0:
-        # empty folder, not a single file ...
-        bb.debug(1, "No files found to determine SOURCE_DATE_EPOCH")
 
     bb.debug(1, "SOURCE_DATE_EPOCH: %d" % source_date_epoch)
     bb.utils.mkdirhier(d.getVar('SDE_DIR'))

meta/classes/sanity.bbclass

@@ -511,6 +511,16 @@ def check_make_version(sanity_data):
     return None
 
 
+# Check if we're running on WSL (Windows Subsystem for Linux). Its known not to
+# work but we should tell the user that upfront.
+def check_wsl(d):
+    with open("/proc/version", "r") as f:
+        verdata = f.readlines()
+    for l in verdata:
+        if "Microsoft" in l:
+            return "OpenEmbedded doesn't work under WSL at this time, sorry"
+    return None
+
 # Tar version 1.24 and onwards handle overwriting symlinks correctly
 # but earlier versions do not; this needs to work properly for sstate
 def check_tar_version(sanity_data):
@@ -625,6 +635,7 @@ def check_sanity_version_change(status, d):
     status.addresult(check_tar_version(d))
     status.addresult(check_git_version(d))
     status.addresult(check_perl_modules(d))
+    status.addresult(check_wsl(d))
 
     missing = ""
 

meta/classes/scons.bbclass

@@ -2,16 +2,25 @@ DEPENDS += "python-scons-native"
 
 EXTRA_OESCONS ?= ""
 
-do_configure[noexec] = "1"
+do_configure() {
+	if [ -n "${CONFIGURESTAMPFILE}" ]; then
+		if [ -e "${CONFIGURESTAMPFILE}" -a "`cat ${CONFIGURESTAMPFILE}`" != "${BB_TASKHASH}" -a "${CLEANBROKEN}" != "1" ]; then
+			${STAGING_BINDIR_NATIVE}/scons --clean PREFIX=${prefix} prefix=${prefix} ${EXTRA_OESCONS}
+		fi
+
+		mkdir -p `dirname ${CONFIGURESTAMPFILE}`
+		echo ${BB_TASKHASH} > ${CONFIGURESTAMPFILE}
+	fi
+}
 
 scons_do_compile() {
-        ${STAGING_BINDIR_NATIVE}/scons ${PARALLEL_MAKE} PREFIX=${prefix} prefix=${prefix} ${EXTRA_OESCONS} || \
-        die "scons build execution failed."
+	${STAGING_BINDIR_NATIVE}/scons ${PARALLEL_MAKE} PREFIX=${prefix} prefix=${prefix} ${EXTRA_OESCONS} || \
+	die "scons build execution failed."
 }
 
 scons_do_install() {
-        ${STAGING_BINDIR_NATIVE}/scons install_root=${D}${prefix} PREFIX=${prefix} prefix=${prefix} ${EXTRA_OESCONS} install || \
-        die "scons install execution failed."
+	${STAGING_BINDIR_NATIVE}/scons install_root=${D}${prefix} PREFIX=${prefix} prefix=${prefix} ${EXTRA_OESCONS} install || \
+	die "scons install execution failed."
 }
 
 EXPORT_FUNCTIONS do_compile do_install

meta/classes/siteconfig.bbclass

@@ -1,13 +1,13 @@
 python siteconfig_do_siteconfig () {
-	shared_state = sstate_state_fromvars(d)
-	if shared_state['task'] != 'populate_sysroot':
-		return
-	if not os.path.isdir(os.path.join(d.getVar('FILE_DIRNAME'), 'site_config')):
-		bb.debug(1, "No site_config directory, skipping do_siteconfig")
-		return
-	sstate_install(shared_state, d)
-	bb.build.exec_func('do_siteconfig_gencache', d)
-	sstate_clean(shared_state, d)
+    shared_state = sstate_state_fromvars(d)
+    if shared_state['task'] != 'populate_sysroot':
+        return
+    if not os.path.isdir(os.path.join(d.getVar('FILE_DIRNAME'), 'site_config')):
+        bb.debug(1, "No site_config directory, skipping do_siteconfig")
+        return
+    sstate_install(shared_state, d)
+    bb.build.exec_func('do_siteconfig_gencache', d)
+    sstate_clean(shared_state, d)
 }
 
 EXTRASITECONFIG ?= ""