build-appliance-image: Update to master head revision

(From OE-Core rev: 61d80b07bcfa4adf5f1feb2904fec0a8d09c89f6)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

bitbake/bin/bitbake-worker

@@ -65,7 +65,6 @@ if 0:
     format_str = "%(levelname)s: %(message)s"
     conlogformat = bb.msg.BBLogFormatter(format_str)
     consolelog = logging.FileHandler(logfilename)
-    bb.msg.addDefaultlogFilter(consolelog)
     consolelog.setFormatter(conlogformat)
     logger.addHandler(consolelog)
 
@@ -414,7 +413,7 @@ class BitbakeWorker(object):
 
     def handle_workerdata(self, data):
         self.workerdata = pickle.loads(data)
-        bb.msg.loggerDefaultDebugLevel = self.workerdata["logdefaultdebug"]
+        bb.msg.loggerDefaultLogLevel = self.workerdata["logdefaultlevel"]
         bb.msg.loggerDefaultVerbose = self.workerdata["logdefaultverbose"]
         bb.msg.loggerVerboseLogs = self.workerdata["logdefaultverboselogs"]
         bb.msg.loggerDefaultDomains = self.workerdata["logdefaultdomain"]

bitbake/contrib/autobuilderlog.json

@@ -0,0 +1,13 @@
+{
+    "version": 1,
+    "loggers": {
+        "BitBake.SigGen.HashEquiv": {
+            "level": "VERBOSE",
+            "handlers": ["BitBake.verbconsole"]
+        },
+        "BitBake.RunQueue.HashEquiv": {
+            "level": "VERBOSE",
+            "handlers": ["BitBake.verbconsole"]
+        }
+    }
+}

bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.xml

@@ -929,4 +929,101 @@
             section.
         </para>
     </section>
+
+    <section id="logging">
+        <title>Logging</title>
+        <para>
+            In addition to the standard command line option to control how
+            verbose builds are when execute, bitbake also supports user defined
+            configuration of the
+            <ulink url='https://docs.python.org/3/library/logging.html'>Python logging</ulink>
+            facilities through the
+            <link linkend="var-bb-BB_LOGCONFIG"><filename>BB_LOGCONFIG</filename></link>
+            variable. This variable defines a json or yaml
+            <ulink url='https://docs.python.org/3/library/logging.config.html'>logging configuration</ulink>
+            that will be intelligently merged into the default configuration.
+            The logging configuration is merged using the following rules:
+            <itemizedlist>
+                <listitem><para>
+                    The user defined configuration will completely replace the default
+                    configuration if top level key
+                    <filename>bitbake_merge</filename> is set to the value
+                    <filename>False</filename>. In this case, all other rules
+                    are ignored.
+                </para></listitem>
+                <listitem><para>
+                    The user configuration must have a top level
+                    <filename>version</filename> which must match the value of
+                    the default configuration.
+                </para></listitem>
+                <listitem><para>
+                    Any keys defined in the <filename>handlers</filename>,
+                    <filename>formatters</filename>, or <filename>filters</filename>,
+                    will be merged into the same section in the default
+                    configuration, with the user specified keys taking
+                    replacing a default one if there is a conflict. In
+                    practice, this means that if both the default configuration
+                    and user configuration specify a handler named
+                    <filename>myhandler</filename>, the user defined one will
+                    replace the default. To prevent the user from inadvertently
+                    replacing a default handler, formatter, or filter, all of
+                    the default ones are named with a prefix of
+                    "<filename>BitBake.</filename>"
+                </para></listitem>
+                <listitem><para>
+                    If a logger is defined by the user with the key
+                    <filename>bitbake_merge</filename> set to
+                    <filename>False</filename>, that logger will be completely
+                    replaced by user configuration. In this case, no other
+                    rules will apply to that logger.
+                </listitem></para>
+                <listitem><para>
+                    All user defined <filename>filter</filename> and
+                    <filename>handlers</filename> properties for a given logger
+                    will be merged with corresponding properties from the
+                    default logger. For example, if the user configuration adds
+                    a filter called <filename>myFilter</filename> to the
+                    <filename>BitBake.SigGen</filename>, and the default
+                    configuration adds a filter called
+                    <filename>BitBake.defaultFilter</filename>, both filters
+                    will be applied to the logger
+                </listitem></para>
+            </itemizedlist>
+        </para>
+
+        <para>
+            As an example, consider the following user logging configuration
+            file which logs all Hash Equivalence related messages of VERBOSE or
+            higher to a file called <filename>hashequiv.log</filename>
+            <literallayout class='monospaced'>
+    {
+        "version": 1,
+        "handlers": {
+            "autobuilderlog": {
+                "class": "logging.FileHandler",
+                "formatter": "logfileFormatter",
+                "level": "DEBUG",
+                "filename": "hashequiv.log",
+                "mode": "w"
+            }
+        },
+        "formatters": {
+                "logfileFormatter": {
+                    "format": "%(name)s: %(levelname)s: %(message)s"
+                }
+        },
+        "loggers": {
+            "BitBake.SigGen.HashEquiv": {
+                "level": "VERBOSE",
+                "handlers": ["autobuilderlog"]
+            },
+            "BitBake.RunQueue.HashEquiv": {
+                "level": "VERBOSE",
+                "handlers": ["autobuilderlog"]
+            }
+        }
+    }
+            </literallayout>
+        </para>
+    </section>
 </chapter>

bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.xml

@@ -539,6 +539,17 @@
             </glossdef>
         </glossentry>
 
+        <glossentry id='var-bb-BB_LOGCONFIG'><glossterm>BB_LOGCONFIG</glossterm>
+            <glossdef>
+                <para>
+                    Specifies the name of a config file that contains the user
+                    logging configuration. See
+                    <link linkend="logging">Logging</link> for additional
+                    information
+                </para>
+            </glossdef>
+        </glossentry>
+
         <glossentry id='var-bb-BB_LOGFMT'><glossterm>BB_LOGFMT</glossterm>
             <glossdef>
                 <para>

bitbake/lib/bb/__init__.py

@@ -43,12 +43,13 @@ class BBLogger(Logger):
         Logger.__init__(self, name)
 
     def bbdebug(self, level, msg, *args, **kwargs):
+        loglevel = logging.DEBUG - level + 1
         if not bb.event.worker_pid:
-            if self.name in bb.msg.loggerDefaultDomains and level > (bb.msg.loggerDefaultDomains[self.name]):
+            if self.name in bb.msg.loggerDefaultDomains and loglevel > (bb.msg.loggerDefaultDomains[self.name]):
                 return
-            if level > (bb.msg.loggerDefaultDebugLevel):
+            if loglevel > bb.msg.loggerDefaultLogLevel:
                 return
-        return self.log(logging.DEBUG - level + 1, msg, *args, **kwargs)
+        return self.log(loglevel, msg, *args, **kwargs)
 
     def plain(self, msg, *args, **kwargs):
         return self.log(logging.INFO + 1, msg, *args, **kwargs)

bitbake/lib/bb/build.py

@@ -65,6 +65,8 @@ class TaskBase(event.Event):
         self.taskname = self._task
         self.logfile = logfile
         self.time = time.time()
+        self.pn = d.getVar("PN")
+        self.pv = d.getVar("PV")
         event.Event.__init__(self)
         self._message = "recipe %s: task %s: %s" % (d.getVar("PF"), t, self.getDisplayName())
 

bitbake/lib/bb/command.py

@@ -414,7 +414,11 @@ class CommandsSync:
     getAllAppends.readonly = True
 
     def findProviders(self, command, params):
-        return command.cooker.findProviders()
+        try:
+            mc = params[0]
+        except IndexError:
+            mc = ''
+        return command.cooker.findProviders(mc)
     findProviders.readonly = True
 
     def findBestProvider(self, command, params):

bitbake/lib/bb/cooker.py

@@ -1023,16 +1023,16 @@ class BBCooker:
             bb.event.fire(bb.event.FilesMatchingFound(filepattern, matches), self.data)
 
     def findProviders(self, mc=''):
-        return bb.providers.findProviders(self.data, self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
+        return bb.providers.findProviders(self.databuilder.mcdata[mc], self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
 
     def findBestProvider(self, pn, mc=''):
         if pn in self.recipecaches[mc].providers:
             filenames = self.recipecaches[mc].providers[pn]
-            eligible, foundUnique = bb.providers.filterProviders(filenames, pn, self.data, self.recipecaches[mc])
+            eligible, foundUnique = bb.providers.filterProviders(filenames, pn, self.databuilder.mcdata[mc], self.recipecaches[mc])
             filename = eligible[0]
             return None, None, None, filename
         elif pn in self.recipecaches[mc].pkg_pn:
-            return bb.providers.findBestProvider(pn, self.data, self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
+            return bb.providers.findBestProvider(pn, self.databuilder.mcdata[mc], self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
         else:
             return None, None, None, None
 
@@ -1665,6 +1665,8 @@ class BBCooker:
         self.command.reset()
         self.databuilder.reset()
         self.data = self.databuilder.data
+        self.parsecache_valid = False
+        self.baseconfig_valid = False
 
 
 class CookerExit(bb.event.Event):

bitbake/lib/bb/fetch2/__init__.py

@@ -34,6 +34,7 @@ _checksum_cache = bb.checksum.FileChecksumCache()
 logger = logging.getLogger("BitBake.Fetcher")
 
 CHECKSUM_LIST = [ "md5", "sha256", "sha1", "sha384", "sha512" ]
+SHOWN_CHECKSUM_LIST = ["sha256"]
 
 class BBFetchException(Exception):
     """Class all fetch exceptions inherit from"""
@@ -580,7 +581,9 @@ def verify_checksum(ud, d, precomputed={}):
     checksum_dict = {ci["id"] : ci["data"] for ci in checksum_infos}
     checksum_event = {"%ssum" % ci["id"] : ci["data"] for ci in checksum_infos}
 
-    checksum_lines = ["SRC_URI[%s] = \"%s\"" % (ci["name"], ci["data"]) for ci in checksum_infos]
+    for ci in checksum_infos:
+        if ci["id"] in SHOWN_CHECKSUM_LIST:
+            checksum_lines = ["SRC_URI[%s] = \"%s\"" % (ci["name"], ci["data"])]
 
     # If no checksum has been provided
     if ud.method.recommends_checksum(ud) and all(ci["expected"] is None for ci in checksum_infos):

bitbake/lib/bb/fetch2/gitsm.py

@@ -20,6 +20,8 @@ NOTE: Switching a SRC_URI from "git://" to "gitsm://" requires a clean of your r
 import os
 import bb
 import copy
+import shutil
+import tempfile
 from   bb.fetch2.git import Git
 from   bb.fetch2 import runfetchcmd
 from   bb.fetch2 import logger
@@ -130,7 +132,7 @@ class GitSM(Git):
             ld.setVar('SRCPV', d.getVar('SRCPV'))
             ld.setVar('SRCREV_FORMAT', module)
 
-            function(ud, url, module, paths[module], ld)
+            function(ud, url, module, paths[module], workdir, ld)
 
         return submodules != []
 
@@ -152,7 +154,7 @@ class GitSM(Git):
         return False
 
     def download(self, ud, d):
-        def download_submodule(ud, url, module, modpath, d):
+        def download_submodule(ud, url, module, modpath, workdir, d):
             url += ";bareclone=1;nobranch=1"
 
             # Is the following still needed?
@@ -163,16 +165,25 @@ class GitSM(Git):
                 newfetch.download()
                 # Drop a nugget to add each of the srcrevs we've fetched (used by need_update)
                 runfetchcmd("%s config --add bitbake.srcrev %s" % \
-                            (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir)
+                            (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=workdir)
             except Exception as e:
                 logger.error('gitsm: submodule download failed: %s %s' % (type(e).__name__, str(e)))
                 raise
 
         Git.download(self, ud, d)
-        self.process_submodules(ud, ud.clonedir, download_submodule, d)
+
+        # If we're using a shallow mirror tarball it needs to be unpacked
+        # temporarily so that we can examine the .gitmodules file
+        if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d):
+            tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
+            runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir)
+            self.process_submodules(ud, tmpdir, download_submodule, d)
+            shutil.rmtree(tmpdir)
+        else:
+            self.process_submodules(ud, ud.clonedir, download_submodule, d)
 
     def unpack(self, ud, destdir, d):
-        def unpack_submodules(ud, url, module, modpath, d):
+        def unpack_submodules(ud, url, module, modpath, workdir, d):
             url += ";bareclone=1;nobranch=1"
 
             # Figure out where we clone over the bare submodules...

bitbake/lib/bb/msg.py

@@ -13,6 +13,7 @@ Message handling infrastructure for bitbake
 import sys
 import copy
 import logging
+import logging.config
 from itertools import groupby
 import bb
 import bb.event
@@ -98,6 +99,9 @@ class BBLogFormatter(logging.Formatter):
     def enable_color(self):
         self.color_enabled = True
 
+    def __repr__(self):
+        return "%s fmt='%s' color=%s" % (self.__class__.__name__, self._fmt, "True" if self.color_enabled else "False")
+
 class BBLogFilter(object):
     def __init__(self, handler, level, debug_domains):
         self.stdlevel = level
@@ -116,60 +120,59 @@ class BBLogFilter(object):
             return True
         return False
 
-class BBLogFilterStdErr(BBLogFilter):
-    def filter(self, record):
-        if not BBLogFilter.filter(self, record):
-            return False
-        if record.levelno >= logging.ERROR:
-            return True
-        return False
+class LogFilterGEQLevel(logging.Filter):
+    def __init__(self, level):
+        self.strlevel = str(level)
+        self.level = stringToLevel(level)
+
+    def __repr__(self):
+        return "%s level >= %s (%d)" % (self.__class__.__name__, self.strlevel, self.level)
 
-class BBLogFilterStdOut(BBLogFilter):
     def filter(self, record):
-        if not BBLogFilter.filter(self, record):
-            return False
-        if record.levelno < logging.ERROR:
-            return True
-        return False
+        return (record.levelno >= self.level)
+
+class LogFilterLTLevel(logging.Filter):
+    def __init__(self, level):
+        self.strlevel = str(level)
+        self.level = stringToLevel(level)
+
+    def __repr__(self):
+        return "%s level < %s (%d)" % (self.__class__.__name__, self.strlevel, self.level)
+
+    def filter(self, record):
+        return (record.levelno < self.level)
 
 # Message control functions
 #
 
-loggerDefaultDebugLevel = 0
+loggerDefaultLogLevel = BBLogFormatter.NOTE
 loggerDefaultVerbose = False
 loggerVerboseLogs = False
-loggerDefaultDomains = []
+loggerDefaultDomains = {}
 
 def init_msgconfig(verbose, debug, debug_domains=None):
     """
     Set default verbosity and debug levels config the logger
     """
-    bb.msg.loggerDefaultDebugLevel = debug
     bb.msg.loggerDefaultVerbose = verbose
     if verbose:
         bb.msg.loggerVerboseLogs = True
-    if debug_domains:
-        bb.msg.loggerDefaultDomains = debug_domains
-    else:
-        bb.msg.loggerDefaultDomains = []
-
-def constructLogOptions():
-    debug = loggerDefaultDebugLevel
-    verbose = loggerDefaultVerbose
-    domains = loggerDefaultDomains
 
     if debug:
-        level = BBLogFormatter.DEBUG - debug + 1
+        bb.msg.loggerDefaultLogLevel = BBLogFormatter.DEBUG - debug + 1
     elif verbose:
-        level = BBLogFormatter.VERBOSE
+        bb.msg.loggerDefaultLogLevel = BBLogFormatter.VERBOSE
     else:
-        level = BBLogFormatter.NOTE
+        bb.msg.loggerDefaultLogLevel = BBLogFormatter.NOTE
 
-    debug_domains = {}
-    for (domainarg, iterator) in groupby(domains):
-        dlevel = len(tuple(iterator))
-        debug_domains["BitBake.%s" % domainarg] = logging.DEBUG - dlevel + 1
-    return level, debug_domains
+    bb.msg.loggerDefaultDomains = {}
+    if debug_domains:
+        for (domainarg, iterator) in groupby(debug_domains):
+            dlevel = len(tuple(iterator))
+            bb.msg.loggerDefaultDomains["BitBake.%s" % domainarg] = logging.DEBUG - dlevel + 1
+
+def constructLogOptions():
+    return loggerDefaultLogLevel, loggerDefaultDomains
 
 def addDefaultlogFilter(handler, cls = BBLogFilter, forcelevel=None):
     level, debug_domains = constructLogOptions()
@@ -179,6 +182,19 @@ def addDefaultlogFilter(handler, cls = BBLogFilter, forcelevel=None):
 
     cls(handler, level, debug_domains)
 
+def stringToLevel(level):
+    try:
+        return int(level)
+    except ValueError:
+        pass
+
+    try:
+        return getattr(logging, level)
+    except AttributeError:
+        pass
+
+    return getattr(BBLogFormatter, level)
+
 #
 # Message handling functions
 #
@@ -212,3 +228,105 @@ def has_console_handler(logger):
             if handler.stream in [sys.stderr, sys.stdout]:
                 return True
     return False
+
+def mergeLoggingConfig(logconfig, userconfig):
+    logconfig = copy.deepcopy(logconfig)
+    userconfig = copy.deepcopy(userconfig)
+
+    # Merge config with the default config
+    if userconfig.get('version') != logconfig['version']:
+        raise BaseException("Bad user configuration version. Expected %r, got %r" % (logconfig['version'], userconfig.get('version')))
+
+    # Set some defaults to make merging easier
+    userconfig.setdefault("loggers", {})
+
+    # If a handler, formatter, or filter is defined in the user
+    # config, it will replace an existing one in the default config
+    for k in ("handlers", "formatters", "filters"):
+        logconfig.setdefault(k, {}).update(userconfig.get(k, {}))
+
+    seen_loggers = set()
+    for name, l in logconfig["loggers"].items():
+        # If the merge option is set, merge the handlers and
+        # filters. Otherwise, if it is False, this logger won't get
+        # add to the set of seen loggers and will replace the
+        # existing one
+        if l.get('bitbake_merge', True):
+            ulogger = userconfig["loggers"].setdefault(name, {})
+            ulogger.setdefault("handlers", [])
+            ulogger.setdefault("filters", [])
+
+            # Merge lists
+            l.setdefault("handlers", []).extend(ulogger["handlers"])
+            l.setdefault("filters", []).extend(ulogger["filters"])
+
+            # Replace other properties if present
+            if "level" in ulogger:
+                l["level"] = ulogger["level"]
+
+            if "propagate" in ulogger:
+                l["propagate"] = ulogger["propagate"]
+
+            seen_loggers.add(name)
+
+    # Add all loggers present in the user config, but not any that
+    # have already been processed
+    for name in set(userconfig["loggers"].keys()) - seen_loggers:
+        logconfig["loggers"][name] = userconfig["loggers"][name]
+
+    return logconfig
+
+def setLoggingConfig(defaultconfig, userconfigfile=None):
+    logconfig = copy.deepcopy(defaultconfig)
+
+    if userconfigfile:
+        with open(userconfigfile, 'r') as f:
+            if userconfigfile.endswith('.yml') or userconfigfile.endswith('.yaml'):
+                import yaml
+                userconfig = yaml.load(f)
+            elif userconfigfile.endswith('.json') or userconfigfile.endswith('.cfg'):
+                import json
+                userconfig = json.load(f)
+            else:
+                raise BaseException("Unrecognized file format: %s" % userconfigfile)
+
+            if userconfig.get('bitbake_merge', True):
+                logconfig = mergeLoggingConfig(logconfig, userconfig)
+            else:
+                # Replace the entire default config
+                logconfig = userconfig
+
+    # Convert all level parameters to integers in case users want to use the
+    # bitbake defined level names
+    for h in logconfig["handlers"].values():
+        if "level" in h:
+            h["level"] = bb.msg.stringToLevel(h["level"])
+
+    for l in logconfig["loggers"].values():
+        if "level" in l:
+            l["level"] = bb.msg.stringToLevel(l["level"])
+
+    conf = logging.config.dictConfigClass(logconfig)
+    conf.configure()
+
+    # The user may have specified logging domains they want at a higher debug
+    # level than the standard.
+    for name, l in logconfig["loggers"].items():
+        if not name.startswith("BitBake."):
+            continue
+
+        if not "level" in l:
+            continue
+
+        curlevel = bb.msg.loggerDefaultDomains.get(name)
+        # Note: level parameter should already be a int because of conversion
+        # above
+        newlevel = int(l["level"])
+        if curlevel is None or newlevel < curlevel:
+            bb.msg.loggerDefaultDomains[name] = newlevel
+
+        # TODO: I don't think that setting the global log level should be necessary
+        #if newlevel < bb.msg.loggerDefaultLogLevel:
+        #    bb.msg.loggerDefaultLogLevel = newlevel
+
+    return conf

bitbake/lib/bb/runqueue.py

@@ -27,6 +27,7 @@ import pprint
 
 bblogger = logging.getLogger("BitBake")
 logger = logging.getLogger("BitBake.RunQueue")
+hashequiv_logger = logging.getLogger("BitBake.RunQueue.HashEquiv")
 
 __find_sha256__ = re.compile( r'(?i)(?<![a-z0-9])[a-f0-9]{64}(?![a-z0-9])' )
 
@@ -920,9 +921,11 @@ class RunQueueData:
             runq_build = {}
 
             for task in self.cooker.configuration.runall:
+                if not task.startswith("do_"):
+                    task = "do_{0}".format(task)
                 runall_tids = set()
                 for tid in list(self.runtaskentries):
-                    wanttid = fn_from_tid(tid) + ":do_%s" % task
+                    wanttid = "{0}:{1}".format(fn_from_tid(tid), task)
                     if wanttid in delcount:
                         self.runtaskentries[wanttid] = delcount[wanttid]
                     if wanttid in self.runtaskentries:
@@ -949,7 +952,9 @@ class RunQueueData:
             runq_build = {}
 
             for task in self.cooker.configuration.runonly:
-                runonly_tids = { k: v for k, v in self.runtaskentries.items() if taskname_from_tid(k) == "do_%s" % task }
+                if not task.startswith("do_"):
+                    task = "do_{0}".format(task)
+                runonly_tids = { k: v for k, v in self.runtaskentries.items() if taskname_from_tid(k) == task }
 
                 for tid in list(runonly_tids):
                     mark_active(tid,1)
@@ -1250,7 +1255,7 @@ class RunQueue:
             "fakerootdirs" : self.rqdata.dataCaches[mc].fakerootdirs,
             "fakerootnoenv" : self.rqdata.dataCaches[mc].fakerootnoenv,
             "sigdata" : bb.parse.siggen.get_taskdata(),
-            "logdefaultdebug" : bb.msg.loggerDefaultDebugLevel,
+            "logdefaultlevel" : bb.msg.loggerDefaultLogLevel,
             "logdefaultverbose" : bb.msg.loggerDefaultVerbose,
             "logdefaultverboselogs" : bb.msg.loggerVerboseLogs,
             "logdefaultdomain" : bb.msg.loggerDefaultDomains,
@@ -2054,7 +2059,7 @@ class RunQueueExecute:
         self.update_holdofftasks()
 
         if not self.sq_live and not self.sqdone and not self.sq_deferred and not self.updated_taskhash_queue and not self.holdoff_tasks:
-            logger.info("Setscene tasks completed")
+            hashequiv_logger.verbose("Setscene tasks completed")
 
             err = self.summarise_scenequeue_errors()
             if err:
@@ -2259,7 +2264,7 @@ class RunQueueExecute:
             self.updated_taskhash_queue.remove((tid, unihash))
 
             if unihash != self.rqdata.runtaskentries[tid].unihash:
-                logger.info("Task %s unihash changed to %s" % (tid, unihash))
+                hashequiv_logger.verbose("Task %s unihash changed to %s" % (tid, unihash))
                 self.rqdata.runtaskentries[tid].unihash = unihash
                 bb.parse.siggen.set_unihash(tid, unihash)
                 toprocess.add(tid)
@@ -2304,7 +2309,7 @@ class RunQueueExecute:
                 elif tid in self.scenequeue_covered or tid in self.sq_live:
                     # Already ran this setscene task or it running. Report the new taskhash
                     bb.parse.siggen.report_unihash_equiv(tid, newhash, origuni, newuni, self.rqdata.dataCaches)
-                    logger.info("Already covered setscene for %s so ignoring rehash (remap)" % (tid))
+                    hashequiv_logger.verbose("Already covered setscene for %s so ignoring rehash (remap)" % (tid))
                     remapped = True
 
                 if not remapped:
@@ -2323,7 +2328,7 @@ class RunQueueExecute:
             for mc in self.rq.fakeworker:
                 self.rq.fakeworker[mc].process.stdin.write(b"<newtaskhashes>" + pickle.dumps(bb.parse.siggen.get_taskhashes()) + b"</newtaskhashes>")
 
-            logger.debug(1, pprint.pformat("Tasks changed:\n%s" % (changed)))
+            hashequiv_logger.debug(1, pprint.pformat("Tasks changed:\n%s" % (changed)))
 
         for tid in changed:
             if tid not in self.rqdata.runq_setscene_tids:
@@ -2342,7 +2347,7 @@ class RunQueueExecute:
             # Check no tasks this covers are running
             for dep in self.sqdata.sq_covered_tasks[tid]:
                 if dep in self.runq_running and dep not in self.runq_complete:
-                    logger.debug(2, "Task %s is running which blocks setscene for %s from running" % (dep, tid))
+                    hashequiv_logger.debug(2, "Task %s is running which blocks setscene for %s from running" % (dep, tid))
                     valid = False
                     break
             if not valid:
@@ -2405,7 +2410,7 @@ class RunQueueExecute:
 
         for (tid, harddepfail, origvalid) in update_tasks:
             if tid in self.sqdata.valid and not origvalid:
-                logger.info("Setscene task %s became valid" % tid)
+                hashequiv_logger.verbose("Setscene task %s became valid" % tid)
             if harddepfail:
                 self.sq_task_failoutright(tid)
 

bitbake/lib/bb/siggen.py

@@ -16,6 +16,7 @@ from bb import runqueue
 import hashserv
 
 logger = logging.getLogger('BitBake.SigGen')
+hashequiv_logger = logging.getLogger('BitBake.SigGen.HashEquiv')
 
 def init(d):
     siggens = [obj for obj in globals().values()
@@ -502,9 +503,9 @@ class SignatureGeneratorUniHashMixIn(object):
                 # A unique hash equal to the taskhash is not very interesting,
                 # so it is reported it at debug level 2. If they differ, that
                 # is much more interesting, so it is reported at debug level 1
-                bb.debug((1, 2)[unihash == taskhash], 'Found unihash %s in place of %s for %s from %s' % (unihash, taskhash, tid, self.server))
+                hashequiv_logger.debug((1, 2)[unihash == taskhash], 'Found unihash %s in place of %s for %s from %s' % (unihash, taskhash, tid, self.server))
             else:
-                bb.debug(2, 'No reported unihash for %s:%s from %s' % (tid, taskhash, self.server))
+                hashequiv_logger.debug(2, 'No reported unihash for %s:%s from %s' % (tid, taskhash, self.server))
         except hashserv.client.HashConnectionError as e:
             bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))
 
@@ -578,12 +579,12 @@ class SignatureGeneratorUniHashMixIn(object):
                 new_unihash = data['unihash']
 
                 if new_unihash != unihash:
-                    bb.debug(1, 'Task %s unihash changed %s -> %s by server %s' % (taskhash, unihash, new_unihash, self.server))
+                    hashequiv_logger.debug(1, 'Task %s unihash changed %s -> %s by server %s' % (taskhash, unihash, new_unihash, self.server))
                     bb.event.fire(bb.runqueue.taskUniHashUpdate(fn + ':do_' + task, new_unihash), d)
                     self.set_unihash(tid, new_unihash)
                     d.setVar('BB_UNIHASH', new_unihash)
                 else:
-                    bb.debug(1, 'Reported task %s as unihash %s to %s' % (taskhash, unihash, self.server))
+                    hashequiv_logger.debug(1, 'Reported task %s as unihash %s to %s' % (taskhash, unihash, self.server))
             except hashserv.client.HashConnectionError as e:
                 bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))
         finally:
@@ -606,7 +607,7 @@ class SignatureGeneratorUniHashMixIn(object):
                 method = method + self.extramethod[tid]
 
             data = self.client().report_unihash_equiv(taskhash, method, wanted_unihash, extra_data)
-            bb.note('Reported task %s as unihash %s to %s (%s)' % (tid, wanted_unihash, self.server, str(data)))
+            hashequiv_logger.verbose('Reported task %s as unihash %s to %s (%s)' % (tid, wanted_unihash, self.server, str(data)))
 
             if data is None:
                 bb.warn("Server unable to handle unihash report")
@@ -615,14 +616,14 @@ class SignatureGeneratorUniHashMixIn(object):
             finalunihash = data['unihash']
 
             if finalunihash == current_unihash:
-                bb.note('Task %s unihash %s unchanged by server' % (tid, finalunihash))
+                hashequiv_logger.verbose('Task %s unihash %s unchanged by server' % (tid, finalunihash))
             elif finalunihash == wanted_unihash:
-                bb.note('Task %s unihash changed %s -> %s as wanted' % (tid, current_unihash, finalunihash))
+                hashequiv_logger.verbose('Task %s unihash changed %s -> %s as wanted' % (tid, current_unihash, finalunihash))
                 self.set_unihash(tid, finalunihash)
                 return True
             else:
                 # TODO: What to do here?
-                bb.note('Task %s unihash reported as unwanted hash %s' % (tid, finalunihash))
+                hashequiv_logger.verbose('Task %s unihash reported as unwanted hash %s' % (tid, finalunihash))
 
         except hashserv.client.HashConnectionError as e:
             bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))

bitbake/lib/bb/tests/fetch.py

@@ -1691,6 +1691,47 @@ class GitShallowTest(FetcherTest):
         # Verify the submodule is also shallow
         self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule'))
 
+    def test_shallow_submodule_mirrors(self):
+        self.add_empty_file('a')
+        self.add_empty_file('b')
+
+        smdir = os.path.join(self.tempdir, 'gitsubmodule')
+        bb.utils.mkdirhier(smdir)
+        self.git('init', cwd=smdir)
+        # Make this look like it was cloned from a remote...
+        self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir)
+        self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir)
+        self.add_empty_file('asub', cwd=smdir)
+        self.add_empty_file('bsub', cwd=smdir)
+
+        self.git('submodule init', cwd=self.srcdir)
+        self.git('submodule add file://%s' % smdir, cwd=self.srcdir)
+        self.git('submodule update', cwd=self.srcdir)
+        self.git('commit -m submodule -a', cwd=self.srcdir)
+
+        uri = 'gitsm://%s;protocol=file;subdir=${S}' % self.srcdir
+
+        # Fetch once to generate the shallow tarball
+        fetcher, ud = self.fetch(uri)
+
+        # Set up the mirror
+        mirrordir = os.path.join(self.tempdir, 'mirror')
+        os.rename(self.dldir, mirrordir)
+        self.d.setVar('PREMIRRORS', 'gitsm://.*/.* file://%s/\n' % mirrordir)
+
+        # Fetch from the mirror
+        bb.utils.remove(self.dldir, recurse=True)
+        bb.utils.remove(self.gitdir, recurse=True)
+        self.fetch_and_unpack(uri)
+
+        # Verify the main repository is shallow
+        self.assertRevCount(1)
+
+        # Verify the gitsubmodule directory is present
+        assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule'))
+
+        # Verify the submodule is also shallow
+        self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule'))
 
     if any(os.path.exists(os.path.join(p, 'git-annex')) for p in os.environ.get('PATH').split(':')):
         def test_shallow_annex(self):

bitbake/lib/bb/tinfoil.py

@@ -127,12 +127,13 @@ class TinfoilCookerAdapter:
 
     class TinfoilRecipeCacheAdapter:
         """ cooker.recipecache adapter """
-        def __init__(self, tinfoil):
+        def __init__(self, tinfoil, mc=''):
             self.tinfoil = tinfoil
+            self.mc = mc
             self._cache = {}
 
         def get_pkg_pn_fn(self):
-            pkg_pn = defaultdict(list, self.tinfoil.run_command('getRecipes') or [])
+            pkg_pn = defaultdict(list, self.tinfoil.run_command('getRecipes', self.mc) or [])
             pkg_fn = {}
             for pn, fnlist in pkg_pn.items():
                 for fn in fnlist:
@@ -151,27 +152,27 @@ class TinfoilCookerAdapter:
                 self.get_pkg_pn_fn()
                 return self._cache[name]
             elif name == 'deps':
-                attrvalue = defaultdict(list, self.tinfoil.run_command('getRecipeDepends') or [])
+                attrvalue = defaultdict(list, self.tinfoil.run_command('getRecipeDepends', self.mc) or [])
             elif name == 'rundeps':
-                attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeDepends') or [])
+                attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeDepends', self.mc) or [])
             elif name == 'runrecs':
-                attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeRecommends') or [])
+                attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeRecommends', self.mc) or [])
             elif name == 'pkg_pepvpr':
-                attrvalue = self.tinfoil.run_command('getRecipeVersions') or {}
+                attrvalue = self.tinfoil.run_command('getRecipeVersions', self.mc) or {}
             elif name == 'inherits':
-                attrvalue = self.tinfoil.run_command('getRecipeInherits') or {}
+                attrvalue = self.tinfoil.run_command('getRecipeInherits', self.mc) or {}
             elif name == 'bbfile_priority':
-                attrvalue = self.tinfoil.run_command('getBbFilePriority') or {}
+                attrvalue = self.tinfoil.run_command('getBbFilePriority', self.mc) or {}
             elif name == 'pkg_dp':
-                attrvalue = self.tinfoil.run_command('getDefaultPreference') or {}
+                attrvalue = self.tinfoil.run_command('getDefaultPreference', self.mc) or {}
             elif name == 'fn_provides':
-                attrvalue = self.tinfoil.run_command('getRecipeProvides') or {}
+                attrvalue = self.tinfoil.run_command('getRecipeProvides', self.mc) or {}
             elif name == 'packages':
-                attrvalue = self.tinfoil.run_command('getRecipePackages') or {}
+                attrvalue = self.tinfoil.run_command('getRecipePackages', self.mc) or {}
             elif name == 'packages_dynamic':
-                attrvalue = self.tinfoil.run_command('getRecipePackagesDynamic') or {}
+                attrvalue = self.tinfoil.run_command('getRecipePackagesDynamic', self.mc) or {}
             elif name == 'rproviders':
-                attrvalue = self.tinfoil.run_command('getRProviders') or {}
+                attrvalue = self.tinfoil.run_command('getRProviders', self.mc) or {}
             else:
                 raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
 
@@ -182,8 +183,9 @@ class TinfoilCookerAdapter:
         self.tinfoil = tinfoil
         self.collection = self.TinfoilCookerCollectionAdapter(tinfoil)
         self.recipecaches = {}
-        # FIXME all machines
         self.recipecaches[''] = self.TinfoilRecipeCacheAdapter(tinfoil)
+        for mc in (tinfoil.config_data.getVar('BBMULTICONFIG') or '').split():
+            self.recipecaches[mc] = self.TinfoilRecipeCacheAdapter(tinfoil, mc)
         self._cache = {}
     def __getattr__(self, name):
         # Grab these only when they are requested since they aren't always used
@@ -501,11 +503,11 @@ class Tinfoil:
         """
         return OrderedDict(self.run_command('getSkippedRecipes'))
 
-    def get_all_providers(self):
-        return defaultdict(list, self.run_command('allProviders'))
+    def get_all_providers(self, mc=''):
+        return defaultdict(list, self.run_command('allProviders', mc))
 
-    def find_providers(self):
-        return self.run_command('findProviders')
+    def find_providers(self, mc=''):
+        return self.run_command('findProviders', mc)
 
     def find_best_provider(self, pn):
         return self.run_command('findBestProvider', pn)
@@ -723,21 +725,9 @@ class Tinfoil:
             # Borrowed from knotty, instead somewhat hackily we use the helper
             # as the object to store "shutdown" on
             helper = bb.ui.uihelper.BBUIHelper()
-            # We set up logging optionally in the constructor so now we need to
-            # grab the handlers to pass to TerminalFilter
-            console = None
-            errconsole = None
-            for handler in self.logger.handlers:
-                if isinstance(handler, logging.StreamHandler):
-                    if handler.stream == sys.stdout:
-                        console = handler
-                    elif handler.stream == sys.stderr:
-                        errconsole = handler
-            format_str = "%(levelname)s: %(message)s"
-            format = bb.msg.BBLogFormatter(format_str)
             helper.shutdown = 0
             parseprogress = None
-            termfilter = bb.ui.knotty.TerminalFilter(helper, helper, console, errconsole, format, quiet=self.quiet)
+            termfilter = bb.ui.knotty.TerminalFilter(helper, helper, self.logger.handlers, quiet=self.quiet)
             try:
                 while True:
                     try:

bitbake/lib/bb/ui/knotty.py

@@ -109,12 +109,11 @@ def pluralise(singular, plural, qty):
 
 
 class InteractConsoleLogFilter(logging.Filter):
-    def __init__(self, tf, format):
+    def __init__(self, tf):
         self.tf = tf
-        self.format = format
 
     def filter(self, record):
-        if record.levelno == self.format.NOTE and (record.msg.startswith("Running") or record.msg.startswith("recipe ")):
+        if record.levelno == bb.msg.BBLogFormatter.NOTE and (record.msg.startswith("Running") or record.msg.startswith("recipe ")):
             return False
         self.tf.clearFooter()
         return True
@@ -150,7 +149,7 @@ class TerminalFilter(object):
                 cr = (25, 80)
         return cr
 
-    def __init__(self, main, helper, console, errconsole, format, quiet):
+    def __init__(self, main, helper, handlers, quiet):
         self.main = main
         self.helper = helper
         self.cuu = None
@@ -180,7 +179,11 @@ class TerminalFilter(object):
             termios.tcsetattr(fd, termios.TCSADRAIN, new)
             curses.setupterm()
             if curses.tigetnum("colors") > 2:
-                format.enable_color()
+                for h in handlers:
+                    try:
+                        h.formatter.enable_color()
+                    except AttributeError:
+                        pass
             self.ed = curses.tigetstr("ed")
             if self.ed:
                 self.cuu = curses.tigetstr("cuu")
@@ -196,10 +199,9 @@ class TerminalFilter(object):
             self.interactive = False
             bb.note("Unable to use interactive mode for this terminal, using fallback")
             return
-        if console:
-            console.addFilter(InteractConsoleLogFilter(self, format))
-        if errconsole:
-            errconsole.addFilter(InteractConsoleLogFilter(self, format))
+
+        for h in handlers:
+            h.addFilter(InteractConsoleLogFilter(self))
 
         self.main_progress = None
 
@@ -362,7 +364,11 @@ def _log_settings_from_server(server, observe_only):
     if error:
         logger.error("Unable to get the value of BB_CONSOLELOG variable: %s" % error)
         raise BaseException(error)
-    return includelogs, loglines, consolelogfile
+    logconfigfile, error = server.runCommand([cmd, "BB_LOGCONFIG"])
+    if error:
+        logger.error("Unable to get the value of BB_LOGCONFIG variable: %s" % error)
+        raise BaseException(error)
+    return includelogs, loglines, consolelogfile, logconfigfile
 
 _evt_list = [ "bb.runqueue.runQueueExitWait", "bb.event.LogExecTTY", "logging.LogRecord",
               "bb.build.TaskFailed", "bb.build.TaskBase", "bb.event.ParseStarted",
@@ -379,7 +385,148 @@ def main(server, eventHandler, params, tf = TerminalFilter):
     if not params.observe_only:
         params.updateToServer(server, os.environ.copy())
 
-    includelogs, loglines, consolelogfile = _log_settings_from_server(server, params.observe_only)
+    includelogs, loglines, consolelogfile, logconfigfile = _log_settings_from_server(server, params.observe_only)
+
+    loglevel, _ = bb.msg.constructLogOptions()
+
+    if params.options.quiet == 0:
+        console_loglevel = loglevel
+    elif params.options.quiet > 2:
+        console_loglevel = bb.msg.BBLogFormatter.ERROR
+    else:
+        console_loglevel = bb.msg.BBLogFormatter.WARNING
+
+    logconfig = {
+        "version": 1,
+        "handlers": {
+            "BitBake.console": {
+                "class": "logging.StreamHandler",
+                "formatter": "BitBake.consoleFormatter",
+                "level": console_loglevel,
+                "stream": "ext://sys.stdout",
+                "filters": ["BitBake.stdoutFilter"],
+                ".": {
+                    "is_console": True,
+                },
+            },
+            "BitBake.errconsole": {
+                "class": "logging.StreamHandler",
+                "formatter": "BitBake.consoleFormatter",
+                "level": loglevel,
+                "stream": "ext://sys.stderr",
+                "filters": ["BitBake.stderrFilter"],
+                ".": {
+                    "is_console": True,
+                },
+            },
+            # This handler can be used if specific loggers should print on
+            # the console at a lower severity than the default. It will
+            # display any messages sent to it that are lower than then
+            # BitBake.console logging level (so as to prevent duplication of
+            # messages). Nothing is attached to this handler by default
+            "BitBake.verbconsole": {
+                "class": "logging.StreamHandler",
+                "formatter": "BitBake.consoleFormatter",
+                "level": 1,
+                "stream": "ext://sys.stdout",
+                "filters": ["BitBake.verbconsoleFilter"],
+                ".": {
+                    "is_console": True,
+                },
+            },
+        },
+        "formatters": {
+            # This format instance will get color output enabled by the
+            # terminal
+            "BitBake.consoleFormatter" : {
+                "()": "bb.msg.BBLogFormatter",
+                "format": "%(levelname)s: %(message)s"
+            },
+            # The file log requires a separate instance so that it doesn't get
+            # color enabled
+            "BitBake.logfileFormatter": {
+                "()": "bb.msg.BBLogFormatter",
+                "format": "%(levelname)s: %(message)s"
+            }
+        },
+        "filters": {
+            "BitBake.stdoutFilter": {
+                "()": "bb.msg.LogFilterLTLevel",
+                "level": "ERROR"
+            },
+            "BitBake.stderrFilter": {
+                "()": "bb.msg.LogFilterGEQLevel",
+                "level": "ERROR"
+            },
+            "BitBake.verbconsoleFilter": {
+                "()": "bb.msg.LogFilterLTLevel",
+                "level": console_loglevel
+            },
+        },
+        "loggers": {
+            "BitBake": {
+                "level": loglevel,
+                "handlers": ["BitBake.console", "BitBake.errconsole"],
+            }
+        },
+        "disable_existing_loggers": False
+    }
+
+    # Enable the console log file if enabled
+    if consolelogfile and not params.options.show_environment and not params.options.show_versions:
+        logconfig = bb.msg.mergeLoggingConfig(logconfig, {
+                "version": 1,
+                "handlers" : {
+                    "BitBake.consolelog": {
+                        "class": "logging.FileHandler",
+                        "formatter": "BitBake.logfileFormatter",
+                        "level": loglevel,
+                        "filename": consolelogfile,
+                    },
+                    # Just like verbconsole, anything sent here will go to the
+                    # log file, unless it would go to BitBake.consolelog
+                    "BitBake.verbconsolelog" : {
+                        "class": "logging.FileHandler",
+                        "formatter": "BitBake.logfileFormatter",
+                        "level": 1,
+                        "filename": consolelogfile,
+                        "filters": ["BitBake.verbconsolelogFilter"],
+                    },
+                },
+                "filters": {
+                    "BitBake.verbconsolelogFilter": {
+                        "()": "bb.msg.LogFilterLTLevel",
+                        "level": loglevel,
+                    },
+                },
+                "loggers": {
+                    "BitBake": {
+                        "handlers": ["BitBake.consolelog"],
+                    },
+
+                    # Other interesting things that we want to keep an eye on
+                    # in the log files in case someone has an issue, but not
+                    # necessarily show to the user on the console
+                    "BitBake.SigGen.HashEquiv": {
+                        "level": "VERBOSE",
+                        "handlers": ["BitBake.verbconsolelog"],
+                    },
+                    "BitBake.RunQueue.HashEquiv": {
+                        "level": "VERBOSE",
+                        "handlers": ["BitBake.verbconsolelog"],
+                    }
+                }
+            })
+
+        bb.utils.mkdirhier(os.path.dirname(consolelogfile))
+        loglink = os.path.join(os.path.dirname(consolelogfile), 'console-latest.log')
+        bb.utils.remove(loglink)
+        try:
+           os.symlink(os.path.basename(consolelogfile), loglink)
+        except OSError:
+           pass
+
+    conf = bb.msg.setLoggingConfig(logconfig, logconfigfile)
 
     if sys.stdin.isatty() and sys.stdout.isatty():
         log_exec_tty = True
@@ -388,23 +535,9 @@ def main(server, eventHandler, params, tf = TerminalFilter):
 
     helper = uihelper.BBUIHelper()
 
-    console = logging.StreamHandler(sys.stdout)
-    errconsole = logging.StreamHandler(sys.stderr)
-    format_str = "%(levelname)s: %(message)s"
-    format = bb.msg.BBLogFormatter(format_str)
-    if params.options.quiet == 0:
-        forcelevel = None
-    elif params.options.quiet > 2:
-        forcelevel = bb.msg.BBLogFormatter.ERROR
-    else:
-        forcelevel = bb.msg.BBLogFormatter.WARNING
-    bb.msg.addDefaultlogFilter(console, bb.msg.BBLogFilterStdOut, forcelevel)
-    bb.msg.addDefaultlogFilter(errconsole, bb.msg.BBLogFilterStdErr)
-    console.setFormatter(format)
-    errconsole.setFormatter(format)
-    if not bb.msg.has_console_handler(logger):
-        logger.addHandler(console)
-        logger.addHandler(errconsole)
+    # Look for the specially designated handlers which need to be passed to the
+    # terminal handler
+    console_handlers = [h for h in conf.config['handlers'].values() if getattr(h, 'is_console', False)]
 
     bb.utils.set_process_name("KnottyUI")
 
@@ -412,24 +545,14 @@ def main(server, eventHandler, params, tf = TerminalFilter):
         server.terminateServer()
         return
 
-    consolelog = None
-    if consolelogfile and not params.options.show_environment and not params.options.show_versions:
-        bb.utils.mkdirhier(os.path.dirname(consolelogfile))
-        conlogformat = bb.msg.BBLogFormatter(format_str)
-        consolelog = logging.FileHandler(consolelogfile)
-        bb.msg.addDefaultlogFilter(consolelog)
-        consolelog.setFormatter(conlogformat)
-        logger.addHandler(consolelog)
-        loglink = os.path.join(os.path.dirname(consolelogfile), 'console-latest.log')
-        bb.utils.remove(loglink)
-        try:
-           os.symlink(os.path.basename(consolelogfile), loglink)
-        except OSError:
-           pass
-
     llevel, debug_domains = bb.msg.constructLogOptions()
     server.runCommand(["setEventMask", server.getEventHandle(), llevel, debug_domains, _evt_list])
 
+    # The logging_tree module is *extremely* helpful in debugging logging
+    # domains. Uncomment here to dump the logging tree when bitbake starts
+    #import logging_tree
+    #logging_tree.printout()
+
     universe = False
     if not params.observe_only:
         params.updateFromServer(server)
@@ -464,7 +587,7 @@ def main(server, eventHandler, params, tf = TerminalFilter):
     printinterval = 5000
     lastprint = time.time()
 
-    termfilter = tf(main, helper, console, errconsole, format, params.options.quiet)
+    termfilter = tf(main, helper, console_handlers, params.options.quiet)
     atexit.register(termfilter.finish)
 
     while True:
@@ -503,26 +626,26 @@ def main(server, eventHandler, params, tf = TerminalFilter):
             if isinstance(event, logging.LogRecord):
                 lastprint = time.time()
                 printinterval = 5000
-                if event.levelno >= format.ERROR:
+                if event.levelno >= bb.msg.BBLogFormatter.ERROR:
                     errors = errors + 1
                     return_value = 1
-                elif event.levelno == format.WARNING:
+                elif event.levelno == bb.msg.BBLogFormatter.WARNING:
                     warnings = warnings + 1
 
                 if event.taskpid != 0:
                     # For "normal" logging conditions, don't show note logs from tasks
                     # but do show them if the user has changed the default log level to
                     # include verbose/debug messages
-                    if event.levelno <= format.NOTE and (event.levelno < llevel or (event.levelno == format.NOTE and llevel != format.VERBOSE)):
+                    if event.levelno <= bb.msg.BBLogFormatter.NOTE and (event.levelno < llevel or (event.levelno == bb.msg.BBLogFormatter.NOTE and llevel != bb.msg.BBLogFormatter.VERBOSE)):
                         continue
 
                     # Prefix task messages with recipe/task
-                    if event.taskpid in helper.pidmap and event.levelno != format.PLAIN:
+                    if event.taskpid in helper.pidmap and event.levelno != bb.msg.BBLogFormatter.PLAIN:
                         taskinfo = helper.running_tasks[helper.pidmap[event.taskpid]]
                         event.msg = taskinfo['title'] + ': ' + event.msg
                 if hasattr(event, 'fn'):
                     event.msg = event.fn + ': ' + event.msg
-                logger.handle(event)
+                logging.getLogger(event.name).handle(event)
                 continue
 
             if isinstance(event, bb.build.TaskFailedSilent):
@@ -748,8 +871,6 @@ def main(server, eventHandler, params, tf = TerminalFilter):
         if e.errno == errno.EPIPE:
             pass
 
-    if consolelog:
-        logger.removeHandler(consolelog)
-        consolelog.close()
+    logging.shutdown()
 
     return return_value

bitbake/lib/bb/ui/teamcity.py

@@ -0,0 +1,398 @@
+#
+# TeamCity UI Implementation
+#
+# Implements a TeamCity frontend for the BitBake utility, via service messages.
+# See https://www.jetbrains.com/help/teamcity/build-script-interaction-with-teamcity.html
+#
+# Based on ncurses.py and knotty.py, variously by Michael Lauer and Richard Purdie
+#
+# Copyright (C) 2006 Michael 'Mickey' Lauer
+# Copyright (C) 2006-2012 Richard Purdie
+# Copyright (C) 2018-2020 Agilent Technologies, Inc.
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Author: Chris Laplante <chris.laplante@agilent.com>
+
+from __future__ import division
+
+import datetime
+import logging
+import math
+import os
+import re
+import sys
+import xmlrpc.client
+from collections import deque
+
+import bb
+import bb.build
+import bb.command
+import bb.cooker
+import bb.event
+import bb.exceptions
+import bb.runqueue
+from bb.ui import uihelper
+
+logger = logging.getLogger("BitBake")
+
+
+class TeamCityUI:
+    def __init__(self):
+        self._block_stack = []
+        self._last_progress_state = None
+
+    @classmethod
+    def escape_service_value(cls, value):
+        """
+        Escape a value for inclusion in a service message. TeamCity uses the vertical pipe character for escaping.
+        See: https://confluence.jetbrains.com/display/TCD10/Build+Script+Interaction+with+TeamCity#BuildScriptInteractionwithTeamCity-Escapedvalues
+        """
+        return re.sub(r"(['|\[\]])", r"|\1", value).replace("\n", "|n").replace("\r", "|r")
+
+    @classmethod
+    def emit_service_message(cls, message_type, **kwargs):
+        print(cls.format_service_message(message_type, **kwargs), flush=True)
+
+    @classmethod
+    def format_service_message(cls, message_type, **kwargs):
+        payload = " ".join(["{0}='{1}'".format(k, cls.escape_service_value(v)) for k, v in kwargs.items()])
+        return "##teamcity[{0} {1}]".format(message_type, payload)
+
+    @classmethod
+    def emit_simple_service_message(cls, message_type, message):
+        print(cls.format_simple_service_message(message_type, message), flush=True)
+
+    @classmethod
+    def format_simple_service_message(cls, message_type, message):
+        return "##teamcity[{0} '{1}']".format(message_type, cls.escape_service_value(message))
+
+    @classmethod
+    def format_build_message(cls, text, status):
+        return cls.format_service_message("message", text=text, status=status)
+
+    def block_start(self, name):
+        self._block_stack.append(name)
+        self.emit_service_message("blockOpened", name=name)
+
+    def block_end(self):
+        if self._block_stack:
+            name = self._block_stack.pop()
+            self.emit_service_message("blockClosed", name=name)
+
+    def progress(self, message, percent, extra=None):
+        now = datetime.datetime.now()
+        percent = "{0: >3.0f}".format(percent)
+
+        report = False
+        if not self._last_progress_state \
+                or (self._last_progress_state[0] == message
+                    and self._last_progress_state[1] != percent
+                    and (now - self._last_progress_state[2]).microseconds >= 5000) \
+                or self._last_progress_state[0] != message:
+            report = True
+            self._last_progress_state = (message, percent, now)
+
+        if report or percent in [0, 100]:
+            self.emit_simple_service_message("progressMessage", "{0}: {1}%{2}".format(message, percent, extra or ""))
+
+
+class TeamcityLogFormatter(logging.Formatter):
+    def format(self, record):
+        details = ""
+        if hasattr(record, 'bb_exc_formatted'):
+            details = ''.join(record.bb_exc_formatted)
+        elif hasattr(record, 'bb_exc_info'):
+            etype, value, tb = record.bb_exc_info
+            formatted = bb.exceptions.format_exception(etype, value, tb, limit=5)
+            details = ''.join(formatted)
+
+        if record.levelno in [bb.msg.BBLogFormatter.ERROR, bb.msg.BBLogFormatter.CRITICAL]:
+            # ERROR gets a separate errorDetails field
+            msg = TeamCityUI.format_service_message("message", text=record.getMessage(), status="ERROR",
+                                                    errorDetails=details)
+        else:
+            payload = record.getMessage()
+            if details:
+                payload += "\n" + details
+            if record.levelno == bb.msg.BBLogFormatter.PLAIN:
+                msg = payload
+            elif record.levelno == bb.msg.BBLogFormatter.WARNING:
+                msg = TeamCityUI.format_service_message("message", text=payload, status="WARNING")
+            else:
+                msg = TeamCityUI.format_service_message("message", text=payload, status="NORMAL")
+
+        return msg
+
+
+_evt_list = ["bb.runqueue.runQueueExitWait", "bb.event.LogExecTTY", "logging.LogRecord",
+             "bb.build.TaskFailed", "bb.build.TaskBase", "bb.event.ParseStarted",
+             "bb.event.ParseProgress", "bb.event.ParseCompleted", "bb.event.CacheLoadStarted",
+             "bb.event.CacheLoadProgress", "bb.event.CacheLoadCompleted", "bb.command.CommandFailed",
+             "bb.command.CommandExit", "bb.command.CommandCompleted", "bb.cooker.CookerExit",
+             "bb.event.MultipleProviders", "bb.event.NoProvider", "bb.runqueue.sceneQueueTaskStarted",
+             "bb.runqueue.runQueueTaskStarted", "bb.runqueue.runQueueTaskFailed", "bb.runqueue.sceneQueueTaskFailed",
+             "bb.event.BuildBase", "bb.build.TaskStarted", "bb.build.TaskSucceeded", "bb.build.TaskFailedSilent",
+             "bb.build.TaskProgress", "bb.event.ProcessStarted", "bb.event.ProcessProgress", "bb.event.ProcessFinished"]
+
+
+def _log_settings_from_server(server):
+    # Get values of variables which control our output
+    includelogs, error = server.runCommand(["getVariable", "BBINCLUDELOGS"])
+    if error:
+        logger.error("Unable to get the value of BBINCLUDELOGS variable: %s" % error)
+        raise BaseException(error)
+    loglines, error = server.runCommand(["getVariable", "BBINCLUDELOGS_LINES"])
+    if error:
+        logger.error("Unable to get the value of BBINCLUDELOGS_LINES variable: %s" % error)
+        raise BaseException(error)
+    return includelogs, loglines
+
+
+def main(server, eventHandler, params):
+    params.updateToServer(server, os.environ.copy())
+
+    includelogs, loglines = _log_settings_from_server(server)
+
+    ui = TeamCityUI()
+
+    helper = uihelper.BBUIHelper()
+
+    console = logging.StreamHandler(sys.stdout)
+    errconsole = logging.StreamHandler(sys.stderr)
+    format = TeamcityLogFormatter()
+    if params.options.quiet == 0:
+        forcelevel = None
+    elif params.options.quiet > 2:
+        forcelevel = bb.msg.BBLogFormatter.ERROR
+    else:
+        forcelevel = bb.msg.BBLogFormatter.WARNING
+    bb.msg.addDefaultlogFilter(console, bb.msg.BBLogFilterStdOut, forcelevel)
+    bb.msg.addDefaultlogFilter(errconsole, bb.msg.BBLogFilterStdErr)
+    console.setFormatter(format)
+    errconsole.setFormatter(format)
+    if not bb.msg.has_console_handler(logger):
+        logger.addHandler(console)
+        logger.addHandler(errconsole)
+
+    if params.options.remote_server and params.options.kill_server:
+        server.terminateServer()
+        return
+
+    if params.observe_only:
+        logger.error("Observe-only mode not supported in this UI")
+        return 1
+
+    llevel, debug_domains = bb.msg.constructLogOptions()
+    server.runCommand(["setEventMask", server.getEventHandle(), llevel, debug_domains, _evt_list])
+
+    try:
+        params.updateFromServer(server)
+        cmdline = params.parseActions()
+        if not cmdline:
+            logger.error("No task given")
+            return 1
+        if 'msg' in cmdline and cmdline['msg']:
+            logger.error(cmdline['msg'])
+            return 1
+        cmdline = cmdline['action']
+        ret, error = server.runCommand(cmdline)
+        if error:
+            logger.error("{0}: {1}".format(cmdline, error))
+            return 1
+        elif not ret:
+            logger.error("Couldn't get default commandline: {0}".format(re))
+            return 1
+    except xmlrpc.client.Fault as x:
+        logger.error("XMLRPC Fault getting commandline: {0}".format(x))
+        return 1
+
+    active_process_total = None
+    is_tasks_running = False
+
+    while True:
+        try:
+            event = eventHandler.waitEvent(0.25)
+            if not event:
+                continue
+
+            helper.eventHandler(event)
+
+            if isinstance(event, bb.build.TaskBase):
+                logger.info(event._message)
+            if isinstance(event, logging.LogRecord):
+                # Don't report sstate failures as errors, since Yocto will just run the tasks for real
+                if event.msg == "No suitable staging package found" or (event.msg.startswith(
+                        "Fetcher failure: Unable to find file") and "downloadfilename" in event.msg and "sstate" in event.msg):
+                    event.levelno = bb.msg.BBLogFormatter.WARNING
+                if event.taskpid != 0:
+                    # For "normal" logging conditions, don't show note logs from tasks
+                    # but do show them if the user has changed the default log level to
+                    # include verbose/debug messages
+                    if event.levelno <= bb.msg.BBLogFormatter.NOTE and (event.levelno < llevel or (
+                            event.levelno == bb.msg.BBLogFormatter.NOTE and llevel != bb.msg.BBLogFormatter.VERBOSE)):
+                        continue
+
+                    # Prefix task messages with recipe/task
+                    if event.taskpid in helper.running_tasks and event.levelno != bb.msg.BBLogFormatter.PLAIN:
+                        taskinfo = helper.running_tasks[event.taskpid]
+                        event.msg = taskinfo['title'] + ': ' + event.msg
+                if hasattr(event, 'fn'):
+                    event.msg = event.fn + ': ' + event.msg
+                logger.handle(event)
+            if isinstance(event, bb.build.TaskFailedSilent):
+                logger.warning("Logfile for failed setscene task is %s" % event.logfile)
+                continue
+            if isinstance(event, bb.build.TaskFailed):
+                rt = "{0}-{1}:{2}".format(event.pn, event.pv.replace("AUTOINC", "0"), event.task)
+
+                logfile = event.logfile
+                if not logfile or not os.path.exists(logfile):
+                    TeamCityUI.emit_service_message("buildProblem", description="{0}\nUnknown failure (no log file available)".format(rt))
+                    if not event.task.endswith("_setscene"):
+                        server.runCommand(["stateForceShutdown"])
+                    continue
+
+                details = deque(maxlen=loglines)
+                error_lines = []
+                if includelogs and not event.errprinted:
+                    with open(logfile, "r") as f:
+                        while True:
+                            line = f.readline()
+                            if not line:
+                                break
+                            line = line.rstrip()
+                            details.append(' | %s' % line)
+                            # TODO: a less stupid check for errors
+                            if (event.task == "do_compile") and ("error:" in line):
+                                error_lines.append(line)
+
+                if error_lines:
+                    TeamCityUI.emit_service_message("compilationStarted", compiler=rt)
+                    for line in error_lines:
+                        TeamCityUI.emit_service_message("message", text=line, status="ERROR")
+                    TeamCityUI.emit_service_message("compilationFinished", compiler=rt)
+                else:
+                    TeamCityUI.emit_service_message("buildProblem", description=rt)
+
+                err = "Logfile of failure stored in: %s" % logfile
+                if details:
+                    ui.block_start("{0} task log".format(rt))
+                    # TeamCity seems to choke on service messages longer than about 63800 characters, so if error
+                    # details is longer than, say, 60000, batch it up into several messages.
+                    first_message = True
+                    while details:
+                        detail_len = 0
+                        batch = deque()
+                        while details and detail_len < 60000:
+                            # TODO: This code doesn't bother to handle lines that themselves are extremely long.
+                            line = details.popleft()
+                            batch.append(line)
+                            detail_len += len(line)
+
+                        if first_message:
+                            batch.appendleft("Log data follows:")
+                            first_message = False
+                            TeamCityUI.emit_service_message("message", text=err, status="ERROR",
+                                                            errorDetails="\n".join(batch))
+                        else:
+                            TeamCityUI.emit_service_message("message", text="[continued]", status="ERROR",
+                                                            errorDetails="\n".join(batch))
+                    ui.block_end()
+                else:
+                    TeamCityUI.emit_service_message("message", text=err, status="ERROR", errorDetails="")
+
+                if not event.task.endswith("_setscene"):
+                    server.runCommand(["stateForceShutdown"])
+
+            if isinstance(event, bb.event.ProcessStarted):
+                if event.processname in ["Initialising tasks", "Checking sstate mirror object availability"]:
+                    active_process_total = event.total
+                    ui.block_start(event.processname)
+            if isinstance(event, bb.event.ProcessFinished):
+                if event.processname in ["Initialising tasks", "Checking sstate mirror object availability"]:
+                    ui.progress(event.processname, 100)
+                    ui.block_end()
+            if isinstance(event, bb.event.ProcessProgress):
+                if event.processname in ["Initialising tasks",
+                                         "Checking sstate mirror object availability"] and active_process_total != 0:
+                    ui.progress(event.processname, event.progress * 100 / active_process_total)
+            if isinstance(event, bb.event.CacheLoadStarted):
+                ui.block_start("Loading cache")
+            if isinstance(event, bb.event.CacheLoadProgress):
+                if event.total != 0:
+                    ui.progress("Loading cache", math.floor(event.current * 100 / event.total))
+            if isinstance(event, bb.event.CacheLoadCompleted):
+                ui.progress("Loading cache", 100)
+                ui.block_end()
+            if isinstance(event, bb.event.ParseStarted):
+                ui.block_start("Parsing recipes and checking upstream revisions")
+            if isinstance(event, bb.event.ParseProgress):
+                if event.total != 0:
+                    ui.progress("Parsing recipes", math.floor(event.current * 100 / event.total))
+            if isinstance(event, bb.event.ParseCompleted):
+                ui.progress("Parsing recipes", 100)
+                ui.block_end()
+            if isinstance(event, bb.command.CommandCompleted):
+                return
+            if isinstance(event, bb.command.CommandFailed):
+                logger.error(str(event))
+                return 1
+            if isinstance(event, bb.event.MultipleProviders):
+                logger.warning(str(event))
+                continue
+            if isinstance(event, bb.event.NoProvider):
+                logger.error(str(event))
+                continue
+            if isinstance(event, bb.command.CommandExit):
+                return
+            if isinstance(event, bb.cooker.CookerExit):
+                return
+            if isinstance(event, bb.runqueue.sceneQueueTaskStarted):
+                if not is_tasks_running:
+                    is_tasks_running = True
+                    ui.block_start("Running tasks")
+                if event.stats.total != 0:
+                    ui.progress("Running setscene tasks", (
+                            event.stats.completed + event.stats.active + event.stats.failed + 1) * 100 / event.stats.total)
+            if isinstance(event, bb.runqueue.runQueueTaskStarted):
+                if not is_tasks_running:
+                    is_tasks_running = True
+                    ui.block_start("Running tasks")
+                if event.stats.total != 0:
+                    pseudo_total = event.stats.total - event.stats.skipped
+                    pseudo_complete = event.stats.completed + event.stats.active - event.stats.skipped + event.stats.failed + 1
+                    # TODO: sometimes this gives over 100%
+                    ui.progress("Running runqueue tasks", (pseudo_complete) * 100 / pseudo_total,
+                            " ({0}/{1})".format(pseudo_complete, pseudo_total))
+            if isinstance(event, bb.runqueue.sceneQueueTaskFailed):
+                logger.warning(str(event))
+                continue
+            if isinstance(event, bb.runqueue.runQueueTaskFailed):
+                logger.error(str(event))
+                return 1
+            if isinstance(event, bb.event.LogExecTTY):
+                pass
+        except EnvironmentError as ioerror:
+            # ignore interrupted io
+            if ioerror.args[0] == 4:
+                pass
+        except Exception as ex:
+            logger.error(str(ex))
+
+        # except KeyboardInterrupt:
+        #     if shutdown == 2:
+        #         mw.appendText("Third Keyboard Interrupt, exit.\n")
+        #         exitflag = True
+        #     if shutdown == 1:
+        #         mw.appendText("Second Keyboard Interrupt, stopping...\n")
+        #         _, error = server.runCommand(["stateForceShutdown"])
+        #         if error:
+        #             print("Unable to cleanly stop: %s" % error)
+        #     if shutdown == 0:
+        #         mw.appendText("Keyboard Interrupt, closing down...\n")
+        #         _, error = server.runCommand(["stateShutdown"])
+        #         if error:
+        #             print("Unable to cleanly shutdown: %s" % error)
+        #     shutdown = shutdown + 1
+        #     pass

bitbake/lib/bblayers/query.py

@@ -46,7 +46,7 @@ layer, with the preferred version first. Note that skipped recipes that
 are overlayed will also be listed, with a " (skipped)" suffix.
 """
 
-        items_listed = self.list_recipes('Overlayed recipes', None, True, args.same_version, args.filenames, False, True, None, False, None)
+        items_listed = self.list_recipes('Overlayed recipes', None, True, args.same_version, args.filenames, False, True, None, False, None, args.mc)
 
         # Check for overlayed .bbclass files
         classes = collections.defaultdict(list)
@@ -112,9 +112,9 @@ skipped recipes will also be listed, with a " (skipped)" suffix.
             title = 'Matching recipes:'
         else:
             title = 'Available recipes:'
-        self.list_recipes(title, args.pnspec, False, False, args.filenames, args.recipes_only, args.multiple, args.layer, args.bare, inheritlist)
+        self.list_recipes(title, args.pnspec, False, False, args.filenames, args.recipes_only, args.multiple, args.layer, args.bare, inheritlist, args.mc)
 
-    def list_recipes(self, title, pnspec, show_overlayed_only, show_same_ver_only, show_filenames, show_recipes_only, show_multi_provider_only, selected_layer, bare, inherits):
+    def list_recipes(self, title, pnspec, show_overlayed_only, show_same_ver_only, show_filenames, show_recipes_only, show_multi_provider_only, selected_layer, bare, inherits, mc):
         if inherits:
             bbpath = str(self.tinfoil.config_data.getVar('BBPATH'))
             for classname in inherits:
@@ -123,14 +123,18 @@ skipped recipes will also be listed, with a " (skipped)" suffix.
                     logger.error('No class named %s found in BBPATH', classfile)
                     sys.exit(1)
 
-        pkg_pn = self.tinfoil.cooker.recipecaches[''].pkg_pn
-        (latest_versions, preferred_versions) = self.tinfoil.find_providers()
-        allproviders = self.tinfoil.get_all_providers()
+        pkg_pn = self.tinfoil.cooker.recipecaches[mc].pkg_pn
+        (latest_versions, preferred_versions) = self.tinfoil.find_providers(mc)
+        allproviders = self.tinfoil.get_all_providers(mc)
 
         # Ensure we list skipped recipes
         # We are largely guessing about PN, PV and the preferred version here,
         # but we have no choice since skipped recipes are not fully parsed
         skiplist = list(self.tinfoil.cooker.skiplist.keys())
+        mcspec = 'mc:%s:' % mc
+        if mc:
+            skiplist = [s[len(mcspec):] for s in skiplist if s.startswith(mcspec)]
+
         for fn in skiplist:
             recipe_parts = os.path.splitext(os.path.basename(fn))[0].split('_')
             p = recipe_parts[0]
@@ -187,7 +191,7 @@ skipped recipes will also be listed, with a " (skipped)" suffix.
                 # We only display once per recipe, we should prefer non extended versions of the
                 # recipe if present (so e.g. in OpenEmbedded, openssl rather than nativesdk-openssl
                 # which would otherwise sort first).
-                if realfn[1] and realfn[0] in self.tinfoil.cooker.recipecaches[''].pkg_fn:
+                if realfn[1] and realfn[0] in self.tinfoil.cooker.recipecaches[mc].pkg_fn:
                     continue
 
                 if inherits:
@@ -496,6 +500,7 @@ NOTE: .bbappend files can impact the dependencies.
         parser_show_overlayed = self.add_command(sp, 'show-overlayed', self.do_show_overlayed)
         parser_show_overlayed.add_argument('-f', '--filenames', help='instead of the default formatting, list filenames of higher priority recipes with the ones they overlay indented underneath', action='store_true')
         parser_show_overlayed.add_argument('-s', '--same-version', help='only list overlayed recipes where the version is the same', action='store_true')
+        parser_show_overlayed.add_argument('--mc', help='use specified multiconfig', default='')
 
         parser_show_recipes = self.add_command(sp, 'show-recipes', self.do_show_recipes)
         parser_show_recipes.add_argument('-f', '--filenames', help='instead of the default formatting, list filenames of higher priority recipes with the ones they overlay indented underneath', action='store_true')
@@ -504,6 +509,7 @@ NOTE: .bbappend files can impact the dependencies.
         parser_show_recipes.add_argument('-i', '--inherits', help='only list recipes that inherit the named class(es) - separate multiple classes using , (without spaces)', metavar='CLASS', default='')
         parser_show_recipes.add_argument('-l', '--layer', help='only list recipes from the selected layer', default='')
         parser_show_recipes.add_argument('-b', '--bare', help='output just names without the "(skipped)" marker', action='store_true')
+        parser_show_recipes.add_argument('--mc', help='use specified multiconfig', default='')
         parser_show_recipes.add_argument('pnspec', nargs='*', help='optional recipe name specification (wildcards allowed, enclose in quotes to avoid shell expansion)')
 
         parser_show_appends = self.add_command(sp, 'show-appends', self.do_show_appends)

documentation/brief-yoctoprojectqs/brief-yoctoprojectqs.xml

@@ -266,7 +266,7 @@
          meta-toolchain
          meta-ide-support
 
-     You can also run generated qemu images with a command like 'runqemu qemux86'
+     You can also run generated qemu images with a command like 'runqemu qemux86-64'
                     </literallayout>
                     Among other things, the script creates the
                     <ulink url='&YOCTO_DOCS_REF_URL;#build-directory'>Build Directory</ulink>,
@@ -332,7 +332,7 @@
                     QEMU, which is a Quick EMUlator that ships with
                     the Yocto Project:
                     <literallayout class='monospaced'>
-     $ runqemu qemux86
+     $ runqemu qemux86-64
                     </literallayout>
                     If you want to learn more about running QEMU, see the
                     "<ulink url="&YOCTO_DOCS_DEV_URL;#dev-manual-qemu">Using the Quick EMUlator (QEMU)</ulink>"

documentation/bsp-guide/bsp-guide.xml

@@ -174,13 +174,15 @@
                        "ACTIVE RELEASES DOCUMENTATION" or "DOCUMENTS ARCHIVE"
                        pull-down menus.
                        </para></listitem>
-                   <listitem><para>
+                   <listitem>
+                       <para>
                        To report any inaccuracies or problems with this
-                       manual, send an email to the Yocto Project
-                       discussion group at
-                       <filename>yocto@yoctoproject.com</filename> or log into
-                       the freenode <filename>#yocto</filename> channel.
-                       </para></listitem>
+                       (or any other Yocto Project) manual, send an email to
+                       the Yocto Project documentation mailing list at
+                       <filename>docs@lists.yoctoproject.org</filename> or
+                       log into the freenode <filename>#yocto</filename> channel.
+                       </para>
+                   </listitem>
                </itemizedlist>
            </note>
     </legalnotice>

documentation/dev-manual/dev-manual-common-tasks.xml

@@ -1397,7 +1397,7 @@
                 package specified in the <filename>PACKAGES</filename>
                 statement.
                 <note>
-                    The <filename>inherit packages</filename> should be
+                    The <filename>inherit packagegroup</filename> line should be
                     located near the top of the recipe, certainly before
                     the <filename>PACKAGES</filename> statement.
                 </note>
@@ -10822,7 +10822,7 @@
                 Use the following BitBake command form to fetch all the
                 necessary sources without starting the build:
                 <literallayout class='monospaced'>
-     $ bitbake -c <replaceable>target</replaceable> runall="fetch"
+     $ bitbake <replaceable>target</replaceable> --runall=fetch
                 </literallayout>
                 This variation of the BitBake command guarantees that you
                 have all the sources for that BitBake target should you

documentation/dev-manual/dev-manual-qemu.xml

@@ -151,13 +151,13 @@
                     <itemizedlist>
                         <listitem><para>
                             This example starts QEMU with
-                            <replaceable>MACHINE</replaceable> set to "qemux86".
+                            <replaceable>MACHINE</replaceable> set to "qemux86-64".
                             Assuming a standard
                             <ulink url='&YOCTO_DOCS_REF_URL;#build-directory'>Build Directory</ulink>,
                             <filename>runqemu</filename> automatically finds the
-                            <filename>bzImage-qemux86.bin</filename> image file and
+                            <filename>bzImage-qemux86-64.bin</filename> image file and
                             the
-                            <filename>core-image-minimal-qemux86-20140707074611.rootfs.ext3</filename>
+                            <filename>core-image-minimal-qemux86-64-20200218002850.rootfs.ext4</filename>
                             (assuming the current build created a
                             <filename>core-image-minimal</filename> image).
                             <note>
@@ -166,7 +166,7 @@
                             timestamp.
                             </note>
                             <literallayout class='monospaced'>
-     $ runqemu qemux86
+     $ runqemu qemux86-64
                             </literallayout>
                             </para></listitem>
                         <listitem><para>
@@ -175,7 +175,7 @@
                             This command, however, specifically provides the image
                             and root filesystem type.
                             <literallayout class='monospaced'>
-     $ runqemu qemux86 core-image-minimal ext3
+     $ runqemu qemux86-64 core-image-minimal ext4
                             </literallayout>
                             </para></listitem>
                         <listitem><para>
@@ -188,7 +188,7 @@
                             be installed (see the previous description for the
                             <filename>audio</filename> option for more information).
                             <literallayout class='monospaced'>
-     $ runqemu qemux86 ramfs audio
+     $ runqemu qemux86-64 ramfs audio
                             </literallayout>
                             </para></listitem>
                         <listitem><para>
@@ -200,7 +200,7 @@
                             <replaceable>KERNEL</replaceable>, or
                             <replaceable>VM</replaceable> option.
                             <literallayout class='monospaced'>
-     $ runqemu ext3
+     $ runqemu ext4
                             </literallayout>
                             </para></listitem>
                         <listitem><para>
@@ -209,9 +209,9 @@
                             From the <filename>.wic.vmdk</filename>,
                             <filename>runqemu</filename> determines the QEMU
                             architecture (<replaceable>MACHINE</replaceable>) to be
-                            "qemux86" and the root filesystem type to be "vmdk".
+                            "qemux86-64" and the root filesystem type to be "vmdk".
                             <literallayout class='monospaced'>
-     $ runqemu /home/scott-lenovo/vm/core-image-minimal-qemux86.wic.vmdk
+     $ runqemu /home/scott-lenovo/vm/core-image-minimal-qemux86-64.wic.vmdk
                             </literallayout>
                             </para></listitem>
                     </itemizedlist>
@@ -296,7 +296,7 @@
                     extracts it to a directory named
                     <filename>test-nfs</filename>:
                     <literallayout class='monospaced'>
-     runqemu-extract-sdk ./tmp/deploy/images/qemux86/core-image-sato-qemux86.tar.bz2 test-nfs
+     runqemu-extract-sdk ./tmp/deploy/images/qemux86-64/core-image-sato-qemux86-64.tar.bz2 test-nfs
                     </literallayout>
                     </para></listitem>
                 <listitem><para>
@@ -310,7 +310,7 @@
                     Here is an example using the <filename>qemux86</filename>
                     image:
                     <literallayout class='monospaced'>
-     runqemu qemux86 ./test-nfs
+     runqemu qemux86-64 ./test-nfs
                     </literallayout>
                     </para></listitem>
             </orderedlist>

documentation/dev-manual/dev-manual.xml

@@ -166,13 +166,15 @@
                        "ACTIVE RELEASES DOCUMENTATION" or "DOCUMENTS ARCHIVE"
                        pull-down menus.
                        </para></listitem>
-                   <listitem><para>
+                   <listitem>
+                       <para>
                        To report any inaccuracies or problems with this
-                       manual, send an email to the Yocto Project
-                       discussion group at
-                       <filename>yocto@yoctoproject.com</filename> or log into
-                       the freenode <filename>#yocto</filename> channel.
-                       </para></listitem>
+                       (or any other Yocto Project) manual, send an email to
+                       the Yocto Project documentation mailing list at
+                       <filename>docs@lists.yoctoproject.org</filename> or
+                       log into the freenode <filename>#yocto</filename> channel.
+                       </para>
+                   </listitem>
                </itemizedlist>
            </note>
     </legalnotice>

documentation/kernel-dev/kernel-dev-common.xml

@@ -89,8 +89,8 @@
                         <emphasis>Prepare Your <filename>local.conf</filename> File:</emphasis>
                         By default, the
                         <ulink url='&YOCTO_DOCS_REF_URL;#var-MACHINE'><filename>MACHINE</filename></ulink>
-                        variable is set to "qemux86", which is fine if you are
-                        building for the QEMU emulator in 32-bit mode.
+                        variable is set to "qemux86-64", which is fine if you are
+                        building for the QEMU emulator in 64-bit mode.
                         However, if you are not, you need to set the
                         <filename>MACHINE</filename> variable appropriately in
                         your <filename>conf/local.conf</filename> file found in
@@ -104,10 +104,12 @@
                         <ulink url='&YOCTO_DOCS_REF_URL;#var-MACHINE_ESSENTIAL_EXTRA_RRECOMMENDS'><filename>MACHINE_ESSENTIAL_EXTRA_RRECOMMENDS</filename></ulink>
                         variable to include kernel modules.</para>
 
-                        <para>This example uses the default "qemux86" for the
-                        <filename>MACHINE</filename> variable but needs to
-                        add the "kernel-modules":
+                        <para>In this example we wish to build for qemux86 so
+                        we must set the <filename>MACHINE</filename> variable
+                        to "qemux86" and also add the "kernel-modules". As described
+                        we do this by appending to <filename>conf/local.conf</filename>:
                         <literallayout class='monospaced'>
+     MACHINE = "qemux86"
      MACHINE_ESSENTIAL_EXTRA_RRECOMMENDS += "kernel-modules"
                         </literallayout>
                         </para></listitem>
@@ -314,8 +316,8 @@
                         File:</emphasis>
                         By default, the
                         <ulink url='&YOCTO_DOCS_REF_URL;#var-MACHINE'><filename>MACHINE</filename></ulink>
-                        variable is set to "qemux86", which is fine if you are
-                        building for the QEMU emulator in 32-bit mode.
+                        variable is set to "qemux86-64", which is fine if you are
+                        building for the QEMU emulator in 64-bit mode.
                         However, if you are not, you need to set the
                         <filename>MACHINE</filename> variable appropriately in
                         your <filename>conf/local.conf</filename> file found
@@ -329,10 +331,12 @@
                         <ulink url='&YOCTO_DOCS_REF_URL;#var-MACHINE_ESSENTIAL_EXTRA_RRECOMMENDS'><filename>MACHINE_ESSENTIAL_EXTRA_RRECOMMENDS</filename></ulink>
                         variable to include kernel modules.</para>
 
-                        <para>This example uses the default "qemux86" for the
-                        <filename>MACHINE</filename> variable but needs to
-                        add the "kernel-modules":
+                        <para>In this example we wish to build for qemux86 so
+                        we must set the <filename>MACHINE</filename> variable
+                        to "qemux86" and also add the "kernel-modules". As described
+                        we do this by appending to <filename>conf/local.conf</filename>:
                         <literallayout class='monospaced'>
+     MACHINE = "qemux86"
      MACHINE_ESSENTIAL_EXTRA_RRECOMMENDS += "kernel-modules"
                         </literallayout>
                         </para></listitem>

documentation/kernel-dev/kernel-dev.xml

@@ -149,13 +149,15 @@
                        "ACTIVE RELEASES DOCUMENTATION" or "DOCUMENTS ARCHIVE"
                        pull-down menus.
                        </para></listitem>
-                   <listitem><para>
+                   <listitem>
+                       <para>
                        To report any inaccuracies or problems with this
-                       manual, send an email to the Yocto Project
-                       discussion group at
-                       <filename>yocto@yoctoproject.com</filename> or log into
-                       the freenode <filename>#yocto</filename> channel.
-                       </para></listitem>
+                       (or any other Yocto Project) manual, send an email to
+                       the Yocto Project documentation mailing list at
+                       <filename>docs@lists.yoctoproject.org</filename> or
+                       log into the freenode <filename>#yocto</filename> channel.
+                       </para>
+                   </listitem>
                </itemizedlist>
            </note>
     </legalnotice>

documentation/mega-manual/mega-manual.xml

@@ -140,13 +140,15 @@
                        "ACTIVE RELEASES DOCUMENTATION" or "DOCUMENTS ARCHIVE"
                        pull-down menus.
                        </para></listitem>
-                   <listitem><para>
+                   <listitem>
+                       <para>
                        To report any inaccuracies or problems with this
-                       manual, send an email to the Yocto Project
-                       discussion group at
-                       <filename>yocto@yoctoproject.com</filename> or log into
-                       the freenode <filename>#yocto</filename> channel.
-                       </para></listitem>
+                       (or any other Yocto Project) manual, send an email to
+                       the Yocto Project documentation mailing list at
+                       <filename>docs@lists.yoctoproject.org</filename> or
+                       log into the freenode <filename>#yocto</filename> channel.
+                       </para>
+                   </listitem>
                </itemizedlist>
            </note>
 

documentation/overview-manual/overview-manual.xml

@@ -101,13 +101,15 @@
                        "ACTIVE RELEASES DOCUMENTATION" or "DOCUMENTS ARCHIVE"
                        pull-down menus.
                        </para></listitem>
-                   <listitem><para>
+                   <listitem>
+                       <para>
                        To report any inaccuracies or problems with this
-                       manual, send an email to the Yocto Project
-                       discussion group at
-                       <filename>yocto@yoctoproject.com</filename> or log into
-                       the freenode <filename>#yocto</filename> channel.
-                       </para></listitem>
+                       (or any other Yocto Project) manual, send an email to
+                       the Yocto Project documentation mailing list at
+                       <filename>docs@lists.yoctoproject.org</filename> or
+                       log into the freenode <filename>#yocto</filename> channel.
+                       </para>
+                   </listitem>
                </itemizedlist>
            </note>
     </legalnotice>

documentation/poky.ent

@@ -58,7 +58,7 @@
 <!ENTITY OE_INIT_PATH "&YOCTO_POKY;/oe-init-build-env">
 <!ENTITY OE_INIT_FILE "oe-init-build-env">
 <!ENTITY UBUNTU_HOST_PACKAGES_ESSENTIAL "gawk wget git-core diffstat unzip texinfo gcc-multilib \
-     build-essential chrpath socat cpio python python3 python3-pip python3-pexpect \
+     build-essential chrpath socat cpio python3 python3-pip python3-pexpect \
      xz-utils debianutils iputils-ping python3-git python3-jinja2 libegl1-mesa libsdl1.2-dev \
      pylint3 xterm">
 <!ENTITY FEDORA_HOST_PACKAGES_ESSENTIAL "gawk make wget tar bzip2 gzip python3 unzip perl patch \

documentation/profile-manual/profile-manual-usage.xml

@@ -2182,7 +2182,7 @@
               meta-toolchain
               meta-ide-support
 
-     You can also run generated qemu images with a command like 'runqemu qemux86'
+     You can also run generated qemu images with a command like 'runqemu qemux86-64'
 
             </literallayout>
             Once you've done that, you can cd to whatever directory

documentation/profile-manual/profile-manual.xml

@@ -151,13 +151,15 @@
                        "ACTIVE RELEASES DOCUMENTATION" or "DOCUMENTS ARCHIVE"
                        pull-down menus.
                        </para></listitem>
-                   <listitem><para>
+                   <listitem>
+                       <para>
                        To report any inaccuracies or problems with this
-                       manual, send an email to the Yocto Project
-                       discussion group at
-                       <filename>yocto@yoctoproject.com</filename> or log into
-                       the freenode <filename>#yocto</filename> channel.
-                       </para></listitem>
+                       (or any other Yocto Project) manual, send an email to
+                       the Yocto Project documentation mailing list at
+                       <filename>docs@lists.yoctoproject.org</filename> or
+                       log into the freenode <filename>#yocto</filename> channel.
+                       </para>
+                   </listitem>
                </itemizedlist>
            </note>
     </legalnotice>

documentation/ref-manual/migration.xml

@@ -2110,7 +2110,7 @@
             such as the following:
             <literallayout class='monospaced'>
      inherit bluetooth
-     PACKAGECONFIG ??= "${@bb.utils.contains('DISTRO_FEATURES', 'bluetooth', '${BLUEZ}', '', d)}
+     PACKAGECONFIG ??= "${@bb.utils.contains('DISTRO_FEATURES', 'bluetooth', '${BLUEZ}', '', d)}"
      PACKAGECONFIG[bluez4] = "--enable-bluetooth,--disable-bluetooth,bluez4"
      PACKAGECONFIG[bluez5] = "--enable-bluez5,--disable-bluez5,bluez5"
             </literallayout>
@@ -3624,7 +3624,7 @@ $ runqemu qemux86-64 tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.
             image types, this part of the kernel image base name as been
             removed leaving only the following:
             <literallayout class='monospaced'>
-     KERNEL_IMAGE_BASE_NAME ?= "${PKGE}-${PKGV}-${PKGR}-${MACHINE}-${DATETIME}
+     KERNEL_IMAGE_BASE_NAME ?= "${PKGE}-${PKGV}-${PKGR}-${MACHINE}-${DATETIME}"
             </literallayout>
             If you have recipes or classes that use
             <filename>KERNEL_IMAGE_BASE_NAME</filename> directly, you might

documentation/ref-manual/ref-manual.xml

@@ -175,13 +175,15 @@
                        "ACTIVE RELEASES DOCUMENTATION" or "DOCUMENTS ARCHIVE"
                        pull-down menus.
                        </para></listitem>
-                   <listitem><para>
+                   <listitem>
+                       <para>
                        To report any inaccuracies or problems with this
-                       manual, send an email to the Yocto Project
-                       discussion group at
-                       <filename>yocto@yoctoproject.com</filename> or log into
-                       the freenode <filename>#yocto</filename> channel.
-                       </para></listitem>
+                       (or any other Yocto Project) manual, send an email to
+                       the Yocto Project documentation mailing list at
+                       <filename>docs@lists.yoctoproject.org</filename> or
+                       log into the freenode <filename>#yocto</filename> channel.
+                       </para>
+                   </listitem>
                </itemizedlist>
            </note>
     </legalnotice>

documentation/ref-manual/ref-structure.xml

@@ -217,7 +217,7 @@
          meta-toolchain
          meta-ide-support
 
-     You can also run generated qemu images with a command like 'runqemu qemux86'
+     You can also run generated qemu images with a command like 'runqemu qemux86-64'
             </literallayout>
             The script gets its default list of common targets from the
             <filename>conf-notes.txt</filename> file, which is found in the
@@ -424,7 +424,7 @@
     </section>
 
     <section id='structure-build-conf-sanity_info'>
-        <title><filename>build/conf/sanity_info</filename></title>
+        <title><filename>build/cache/sanity_info</filename></title>
 
         <para>
             This file indicates the state of the sanity checks and is created

documentation/ref-manual/ref-variables.xml

@@ -367,7 +367,7 @@
 
         <glossentry id='var-ASSUME_SHLIBS'><glossterm>ASSUME_SHLIBS</glossterm>
             <info>
-                ASSUME_SHLIBS[doc] = Provides additional shlibs provider mapping information, which adds to or overwrites the information provided automatically by the system."
+                ASSUME_SHLIBS[doc] = "Provides additional shlibs provider mapping information, which adds to or overwrites the information provided automatically by the system."
             </info>
             <glossdef>
                 <para role="glossdeffirst">
@@ -495,6 +495,30 @@
             </glossdef>
         </glossentry>
 
+        <glossentry id='var-AVAILABLE_LICENSES'><glossterm>AVAILABLE_LICENSES</glossterm>
+            <info>
+                AVAILABLE_LICENSES[doc] = "List of licenses found in the directories specified by COMMON_LICENSE_DIR and LICENSE_PATH."
+            </info>
+            <glossdef>
+                <para role="glossdeffirst">
+<!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
+
+                    List of licenses found in the directories specified
+                    by <link linkend='var-COMMON_LICENSE_DIR'><filename>COMMON_LICENSE_DIR</filename></link>
+                    and <link linkend='var-LICENSE_PATH'><filename>LICENSE_PATH</filename></link>.
+
+                    <note>
+                        It is assumed that all changes
+                        to <filename>COMMON_LICENSE_DIR</filename>
+                        and <filename>LICENSE_PATH</filename> have been done
+                        before <filename>AVAILABLE_LICENSES</filename> is
+                        defined
+                        (in <link linkend='ref-classes-license'>license.bbclass</link>).
+                    </note>
+                </para>
+            </glossdef>
+        </glossentry>
+
         <glossentry id='var-AVAILTUNES'><glossterm>AVAILTUNES</glossterm>
             <info>
                 AVAILTUNES[doc] = "The list of defined CPU and Application Binary Interface (ABI) tunings (i.e.  "tunes") available for use by the OpenEmbedded build system."
@@ -1423,7 +1447,7 @@
                     Use the following format to export the variable to the
                     BitBake environment:
                     <literallayout class='monospaced'>
-     export BBSERVER=localhost:$port"
+     export BBSERVER=localhost:$port
                     </literallayout>
                 </para>
 
@@ -3786,7 +3810,7 @@
                     It is not intended to be user-configurable.
                     It is best to just reference the variable to see which distro features are
                     being backfilled for all distro configurations.
-                    See the <link linkend='ref-features-backfill'>Feature Backfilling</link> section for
+                    See the "<link linkend='ref-features-backfill'>Feature Backfilling</link>" section for
                     more information.
                 </para>
             </glossdef>
@@ -6750,6 +6774,25 @@
                     components that are required to produce a functional system
                     image.
                 </note>
+
+                <note><title>Tips</title>
+                    It is possible to define a list of licenses that are allowed
+                    to be used instead of the licenses that are excluded. To do
+                    this, define a
+                    variable <filename>COMPATIBLE_LICENSES</filename> with the
+                    names of the licences that are allowed. Then
+                    define <filename>INCOMPATIBLE_LICENSE</filename> as:
+                    <literallayout class='monospaced'>
+     INCOMPATIBLE_LICENSE = "${@' '.join(sorted(set(d.getVar('AVAILABLE_LICENSES').split()) - set(d.getVar('COMPATIBLE_LICENSES').split())))}"
+                    </literallayout>
+                    This will result
+                    in <filename>INCOMPATIBLE_LICENSE</filename> containing the
+                    names of all licences
+                    from <link linkend='var-AVAILABLE_LICENSES'><filename>AVAILABLE_LICENSES</filename></link>
+                    except the ones specified
+                    in <filename>COMPATIBLE_LICENSES</filename>, thus only
+                    allowing the latter licences to be used.
+                </note>
             </glossdef>
         </glossentry>
 
@@ -11054,7 +11097,7 @@
 
         <glossentry id='var-PN'><glossterm>PN</glossterm>
             <info>
-                PN[doc] = "PN refers to a recipe name in the context of a file used by the OpenEmbedded build system as input to create a package.
+                PN[doc] = "PN refers to a recipe name in the context of a file used by the OpenEmbedded build system as input to create a package."
             </info>
             <glossdef>
                 <para role="glossdeffirst">
@@ -13383,8 +13426,7 @@
 
         <glossentry id='var-SKIP_FILEDEPS'><glossterm>SKIP_FILEDEPS</glossterm>
             <info>
-                SKIP_FILEDEPS[doc] = "Enables you to remove all files from
-                the "Provides" section of an RPM package."
+                SKIP_FILEDEPS[doc] = "Enables you to remove all files from the 'Provides' section of an RPM package."
             </info>
             <glossdef>
                 <para role="glossdeffirst">
@@ -15331,7 +15373,7 @@
 
         <glossentry id='var-TCLIBC'><glossterm>TCLIBC</glossterm>
             <info>
-                TCLIBC[doc] = "Specifies GNU standard C library (libc) variant to use during the build process. You can select 'glibc', 'musl' or "newlib."
+                TCLIBC[doc] = "Specifies GNU standard C library (libc) variant to use during the build process. You can select 'glibc', 'musl' or 'newlib'."
             </info>
             <glossdef>
                 <para role="glossdeffirst">
@@ -16473,7 +16515,7 @@
                     Appends a string to the name of the local version of the
                     U-Boot image.
                     For example, assuming the version of the U-Boot image
-                    built was "2013.10, the full version string reported by
+                    built was "2013.10", the full version string reported by
                     U-Boot would be "2013.10-yocto" given the following
                     statement:
                     <literallayout class='monospaced'>
@@ -16771,18 +16813,21 @@
 
         <glossentry id='var-USERADD_ERROR_DYNAMIC'><glossterm>USERADD_ERROR_DYNAMIC</glossterm>
             <info>
-               USERADD_ERROR_DYNAMIC[doc] = "If set to 'error', forces the OpenEmbedded build system to produce an error if the user identification (uid) and group identification (gid) values are not defined in files/passwd and files/group files. If set to 'warn', a warning will be issued instead."
+               USERADD_ERROR_DYNAMIC[doc] = "If set to 'error', forces the OpenEmbedded build system to produce an error if the user identification (uid) and group identification (gid) values are not defined in any of the files listed in USERADD_UID_TABLES and USERADD_GID_TABLES. If set to 'warn', a warning will be issued instead."
             </info>
             <glossdef>
                 <para role="glossdeffirst">
 <!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
-                    If set to "error", forces the OpenEmbedded build system to
-                    produce an error if the user identification
-                    (<filename>uid</filename>) and group identification
-                    (<filename>gid</filename>) values are not defined
-                    in <filename>files/passwd</filename>
-                    and <filename>files/group</filename> files.
-                    If set to "warn", a warning will be issued instead.
+
+                    If set to <filename>error</filename>, forces the
+                    OpenEmbedded build system to produce an error if the user
+                    identification (<filename>uid</filename>) and group
+                    identification (<filename>gid</filename>) values are not
+                    defined in any of the files listed
+                    in <link linkend='var-USERADD_UID_TABLES'><filename>USERADD_UID_TABLES</filename></link>
+                    and <link linkend='var-USERADD_GID_TABLES'><filename>USERADD_GID_TABLES</filename></link>. If
+                    set to <filename>warn</filename>, a warning will be issued
+                    instead.
                 </para>
 
                 <para>
@@ -16809,6 +16854,20 @@
                     <link linkend='var-USERADD_GID_TABLES'><filename>USERADD_GID_TABLES</filename></link>
                     variables.
                 </para>
+
+                <note>
+                    There is a difference in behavior between
+                    setting <filename>USERADD_ERROR_DYNAMIC</filename>
+                    to <filename>error</filename> and setting it
+                    to <filename>warn</filename>. When it is set
+                    to <filename>warn</filename>, the build system will report a
+                    warning for every undefined <filename>uid</filename> and
+                    <filename>gid</filename> in any recipe. But when it is set
+                    to <filename>error</filename>, it will only report errors
+                    for recipes that are actually built. This saves you from
+                    having to add static IDs for recipes that you know will
+                    never be built.
+                </note>
             </glossdef>
         </glossentry>
 
@@ -17108,7 +17167,7 @@
                     "<ulink url='&YOCTO_DOCS_DEV_URL;#creating-partitioned-images-using-wic'>Creating Partitioned Images Using Wic</ulink>"
                     section in the Yocto Project Development Tasks Manual.
                     For details on the kickstart file format, see the
-                    "<link linkend='ref-kickstart'>OpenEmbedded Kickstart (<filename>.wks</filename>) Reference</link>
+                    "<link linkend='ref-kickstart'>OpenEmbedded Kickstart (<filename>.wks</filename>) Reference</link>"
                     Chapter.
                 </para>
             </glossdef>
@@ -17178,8 +17237,7 @@
 
         <glossentry id='var-XSERVER'><glossterm>XSERVER</glossterm>
             <info>
-               XSERVER[doc] = "Specifies the packages that should be installed
-               to provide an X server and drivers for the current machine."
+               XSERVER[doc] = "Specifies the packages that should be installed to provide an X server and drivers for the current machine."
             </info>
             <glossdef>
                 <para role="glossdeffirst">

documentation/sdk-manual/sdk-manual.xml

@@ -119,13 +119,15 @@
                        "ACTIVE RELEASES DOCUMENTATION" or "DOCUMENTS ARCHIVE"
                        pull-down menus.
                        </para></listitem>
-                   <listitem><para>
+                   <listitem>
+                       <para>
                        To report any inaccuracies or problems with this
-                       manual, send an email to the Yocto Project
-                       discussion group at
-                       <filename>yocto@yoctoproject.com</filename> or log into
-                       the freenode <filename>#yocto</filename> channel.
-                       </para></listitem>
+                       (or any other Yocto Project) manual, send an email to
+                       the Yocto Project documentation mailing list at
+                       <filename>docs@lists.yoctoproject.org</filename> or
+                       log into the freenode <filename>#yocto</filename> channel.
+                       </para>
+                   </listitem>
                </itemizedlist>
            </note>
     </legalnotice>

documentation/toaster-manual/toaster-manual.xml

@@ -129,13 +129,15 @@
                        "ACTIVE RELEASES DOCUMENTATION" or "DOCUMENTS ARCHIVE"
                        pull-down menus.
                        </para></listitem>
-                   <listitem><para>
+                   <listitem>
+                       <para>
                        To report any inaccuracies or problems with this
-                       manual, send an email to the Yocto Project
-                       discussion group at
-                       <filename>yocto@yoctoproject.com</filename> or log into
-                       the freenode <filename>#yocto</filename> channel.
-                       </para></listitem>
+                       (or any other Yocto Project) manual, send an email to
+                       the Yocto Project documentation mailing list at
+                       <filename>docs@lists.yoctoproject.org</filename> or
+                       log into the freenode <filename>#yocto</filename> channel.
+                       </para>
+                   </listitem>
                </itemizedlist>
            </note>
 

meta-poky/conf/layer.conf

@@ -9,7 +9,7 @@ BBFILE_COLLECTIONS += "yocto"
 BBFILE_PATTERN_yocto = "^${LAYERDIR}/"
 BBFILE_PRIORITY_yocto = "5"
 
-LAYERSERIES_COMPAT_yocto = "zeus"
+LAYERSERIES_COMPAT_yocto = "dunfell"
 
 # This should only be incremented on significant changes that will
 # cause compatibility issues with other layers

meta-selftest/conf/layer.conf

@@ -9,4 +9,4 @@ BBFILE_COLLECTIONS += "selftest"
 BBFILE_PATTERN_selftest = "^${LAYERDIR}/"
 BBFILE_PRIORITY_selftest = "5"
 
-LAYERSERIES_COMPAT_selftest = "zeus"
+LAYERSERIES_COMPAT_selftest = "dunfell"

meta-skeleton/conf/layer.conf

@@ -14,4 +14,4 @@ LAYERVERSION_skeleton = "1"
 
 LAYERDEPENDS_skeleton = "core"
 
-LAYERSERIES_COMPAT_skeleton = "zeus"
+LAYERSERIES_COMPAT_skeleton = "dunfell"

meta-yocto-bsp/conf/layer.conf

@@ -9,4 +9,4 @@ BBFILE_COLLECTIONS += "yoctobsp"
 BBFILE_PATTERN_yoctobsp = "^${LAYERDIR}/"
 BBFILE_PRIORITY_yoctobsp = "5"
 LAYERVERSION_yoctobsp = "4"
-LAYERSERIES_COMPAT_yoctobsp = "zeus"
+LAYERSERIES_COMPAT_yoctobsp = "dunfell"

meta-yocto-bsp/recipes-kernel/linux/linux-yocto_5.2.bbappend

@@ -1,23 +0,0 @@
-KBRANCH_genericx86  = "v5.2/standard/base"
-KBRANCH_genericx86-64  = "v5.2/standard/base"
-KBRANCH_edgerouter = "v5.2/standard/edgerouter"
-KBRANCH_beaglebone-yocto = "v5.2/standard/beaglebone"
-
-KMACHINE_genericx86 ?= "common-pc"
-KMACHINE_genericx86-64 ?= "common-pc-64"
-KMACHINE_beaglebone-yocto ?= "beaglebone"
-
-SRCREV_machine_genericx86    ?= "25b14cdf9659c68ab32d66616dfdbbe2ece3fd94"
-SRCREV_machine_genericx86-64 ?= "25b14cdf9659c68ab32d66616dfdbbe2ece3fd94"
-SRCREV_machine_edgerouter ?= "25b14cdf9659c68ab32d66616dfdbbe2ece3fd94"
-SRCREV_machine_beaglebone-yocto ?= "25b14cdf9659c68ab32d66616dfdbbe2ece3fd94"
-
-COMPATIBLE_MACHINE_genericx86 = "genericx86"
-COMPATIBLE_MACHINE_genericx86-64 = "genericx86-64"
-COMPATIBLE_MACHINE_edgerouter = "edgerouter"
-COMPATIBLE_MACHINE_beaglebone-yocto = "beaglebone-yocto"
-
-LINUX_VERSION_genericx86 = "5.2.17"
-LINUX_VERSION_genericx86-64 = "5.2.17"
-LINUX_VERSION_edgerouter = "5.2.17"
-LINUX_VERSION_beaglebone-yocto = "5.2.17"

meta/classes/base.bbclass

@@ -393,7 +393,7 @@ python () {
     # These take the form:
     #
     # PACKAGECONFIG ??= "<default options>"
-    # PACKAGECONFIG[foo] = "--enable-foo,--disable-foo,foo_depends,foo_runtime_depends,foo_runtime_recommends"
+    # PACKAGECONFIG[foo] = "--enable-foo,--disable-foo,foo_depends,foo_runtime_depends,foo_runtime_recommends,foo_conflict_packageconfig"
     pkgconfigflags = d.getVarFlags("PACKAGECONFIG") or {}
     if pkgconfigflags:
         pkgconfig = (d.getVar('PACKAGECONFIG') or "").split()
@@ -440,8 +440,8 @@ python () {
         for flag, flagval in sorted(pkgconfigflags.items()):
             items = flagval.split(",")
             num = len(items)
-            if num > 5:
-                bb.error("%s: PACKAGECONFIG[%s] Only enable,disable,depend,rdepend,rrecommend can be specified!"
+            if num > 6:
+                bb.error("%s: PACKAGECONFIG[%s] Only enable,disable,depend,rdepend,rrecommend,conflict_packageconfig can be specified!"
                     % (d.getVar('PN'), flag))
 
             if flag in pkgconfig:
@@ -455,6 +455,20 @@ python () {
                     extraconf.append(items[0])
             elif num >= 2 and items[1]:
                     extraconf.append(items[1])
+
+            if num >= 6 and items[5]:
+                conflicts = set(items[5].split())
+                invalid = conflicts.difference(set(pkgconfigflags.keys()))
+                if invalid:
+                    bb.error("%s: PACKAGECONFIG[%s] Invalid conflict package config%s '%s' specified."
+                        % (d.getVar('PN'), flag, 's' if len(invalid) > 1 else '', ' '.join(invalid)))
+
+                if flag in pkgconfig:
+                    intersec = conflicts.intersection(set(pkgconfig))
+                    if intersec:
+                        bb.fatal("%s: PACKAGECONFIG[%s] Conflict package config%s '%s' set in PACKAGECONFIG."
+                            % (d.getVar('PN'), flag, 's' if len(intersec) > 1 else '', ' '.join(intersec)))
+
         appendVar('DEPENDS', extradeps)
         appendVar('RDEPENDS_${PN}', extrardeps)
         appendVar('RRECOMMENDS_${PN}', extrarrecs)

meta/classes/cve-check.bbclass

@@ -56,10 +56,10 @@ python do_cve_check () {
             patched_cves = get_patches_cves(d)
         except FileNotFoundError:
             bb.fatal("Failure in searching patches")
-        patched, unpatched = check_cves(d, patched_cves)
+        whitelisted, patched, unpatched = check_cves(d, patched_cves)
         if patched or unpatched:
             cve_data = get_cve_info(d, patched + unpatched)
-            cve_write_data(d, patched, unpatched, cve_data)
+            cve_write_data(d, patched, unpatched, whitelisted, cve_data)
     else:
         bb.note("No CVE database found, skipping CVE check")
 
@@ -179,13 +179,13 @@ def check_cves(d, patched_cves):
     products = d.getVar("CVE_PRODUCT").split()
     # If this has been unset then we're not scanning for CVEs here (for example, image recipes)
     if not products:
-        return ([], [])
+        return ([], [], [])
     pv = d.getVar("CVE_VERSION").split("+git")[0]
 
     # If the recipe has been whitlisted we return empty lists
     if d.getVar("PN") in d.getVar("CVE_CHECK_PN_WHITELIST").split():
         bb.note("Recipe has been whitelisted, skipping check")
-        return ([], [])
+        return ([], [], [])
 
     old_cve_whitelist =  d.getVar("CVE_CHECK_CVE_WHITELIST")
     if old_cve_whitelist:
@@ -263,7 +263,7 @@ def check_cves(d, patched_cves):
 
     conn.close()
 
-    return (list(patched_cves), cves_unpatched)
+    return (list(cve_whitelist), list(patched_cves), cves_unpatched)
 
 def get_cve_info(d, cves):
     """
@@ -287,7 +287,7 @@ def get_cve_info(d, cves):
     conn.close()
     return cve_data
 
-def cve_write_data(d, patched, unpatched, cve_data):
+def cve_write_data(d, patched, unpatched, whitelisted, cve_data):
     """
     Write CVE information in WORKDIR; and to CVE_CHECK_DIR, and
     CVE manifest if enabled.
@@ -303,7 +303,9 @@ def cve_write_data(d, patched, unpatched, cve_data):
         write_string += "PACKAGE NAME: %s\n" % d.getVar("PN")
         write_string += "PACKAGE VERSION: %s\n" % d.getVar("PV")
         write_string += "CVE: %s\n" % cve
-        if cve in patched:
+        if cve in whitelisted:
+            write_string += "CVE STATUS: Whitelisted\n"
+        elif cve in patched:
             write_string += "CVE STATUS: Patched\n"
         else:
             unpatched_cves.append(cve)

meta/classes/grub-efi-cfg.bbclass

@@ -88,6 +88,12 @@ python build_efi_cfg() {
     for label in labels.split():
         localdata = d.createCopy()
 
+        overrides = localdata.getVar('OVERRIDES')
+        if not overrides:
+            bb.fatal('OVERRIDES not defined')
+
+        localdata.setVar('OVERRIDES', 'grub_' + label + ':' + overrides)
+
         for btype in btypes:
             cfgfile.write('\nmenuentry \'%s%s\'{\n' % (label, btype[0]))
             lb = label

meta/classes/image-prelink.bbclass

@@ -17,6 +17,16 @@ prelink_image () {
 	pre_prelink_size=`du -ks ${IMAGE_ROOTFS} | awk '{size = $1 ; print size }'`
 	echo "Size before prelinking $pre_prelink_size."
 
+	# The filesystem may not contain sysconfdir so establish what is present
+	# to enable cleanup after temporary creation of sysconfdir if needed
+	presentdir="${IMAGE_ROOTFS}${sysconfdir}"
+	while [ "${IMAGE_ROOTFS}" != "${presentdir}" ] ; do
+		[ ! -d "${presentdir}" ] || break
+		presentdir=`dirname "${presentdir}"`
+	done
+
+	mkdir -p "${IMAGE_ROOTFS}${sysconfdir}"
+
 	# We need a prelink conf on the filesystem, add one if it's missing
 	if [ ! -e ${IMAGE_ROOTFS}${sysconfdir}/prelink.conf ]; then
 		cp ${STAGING_ETCDIR_NATIVE}/prelink.conf \
@@ -59,6 +69,13 @@ prelink_image () {
 		rm $ldsoconf
 	fi
 
+	# Remove any directories temporarily created for sysconfdir
+	cleanupdir="${IMAGE_ROOTFS}${sysconfdir}"
+	while [ "${presentdir}" != "${cleanupdir}" ] ; do
+		rmdir "${cleanupdir}"
+		cleanupdir=`dirname ${cleanupdir}`
+	done
+
 	pre_prelink_size=`du -ks ${IMAGE_ROOTFS} | awk '{size = $1 ; print size }'`
 	echo "Size after prelinking $pre_prelink_size."
 }

meta/classes/kernel-yocto.bbclass

@@ -1,5 +1,5 @@
 # remove tasks that modify the source tree in case externalsrc is inherited
-SRCTREECOVEREDTASKS += "do_kernel_configme do_validate_branches do_kernel_configcheck do_kernel_checkout do_fetch do_unpack do_patch"
+SRCTREECOVEREDTASKS += "do_validate_branches do_kernel_configcheck do_kernel_checkout do_fetch do_unpack do_patch"
 PATCH_GIT_USER_EMAIL ?= "kernel-yocto@oe"
 PATCH_GIT_USER_NAME ?= "OpenEmbedded"
 
@@ -317,7 +317,7 @@ do_kernel_checkout() {
 }
 do_kernel_checkout[dirs] = "${S}"
 
-addtask kernel_checkout before do_kernel_metadata after do_unpack
+addtask kernel_checkout before do_kernel_metadata after do_symlink_kernsrc
 addtask kernel_metadata after do_validate_branches do_unpack before do_patch
 do_kernel_metadata[depends] = "kern-tools-native:do_populate_sysroot"
 do_validate_branches[depends] = "kern-tools-native:do_populate_sysroot"
@@ -325,6 +325,7 @@ do_validate_branches[depends] = "kern-tools-native:do_populate_sysroot"
 do_kernel_configme[depends] += "virtual/${TARGET_PREFIX}binutils:do_populate_sysroot"
 do_kernel_configme[depends] += "virtual/${TARGET_PREFIX}gcc:do_populate_sysroot"
 do_kernel_configme[depends] += "bc-native:do_populate_sysroot bison-native:do_populate_sysroot"
+do_kernel_configme[depends] += "kern-tools-native:do_populate_sysroot"
 do_kernel_configme[dirs] += "${S} ${B}"
 do_kernel_configme() {
 	# translate the kconfig_mode into something that merge_config.sh

meta/classes/kernel.bbclass

@@ -294,14 +294,10 @@ kernel_do_compile() {
 		# kernel sources do not use do_unpack, so SOURCE_DATE_EPOCH may not
 		# be set....
 		if [ "${SOURCE_DATE_EPOCH}" = "" -o "${SOURCE_DATE_EPOCH}" = "0" ]; then
-			olddir=`pwd`
-			cd ${S}
-			SOURCE_DATE_EPOCH=`git log  -1 --pretty=%ct`
-			# git repo not guaranteed, so fall back to REPRODUCIBLE_TIMESTAMP_ROOTFS
-			if [ $? -ne 0 ]; then
-				SOURCE_DATE_EPOCH=${REPRODUCIBLE_TIMESTAMP_ROOTFS}
-			fi
-			cd $olddir
+			# The source directory is not necessarily a git repository, so we
+			# specify the git-dir to ensure that git does not query a
+			# repository in any parent directory.
+			SOURCE_DATE_EPOCH=`git --git-dir="${S}/.git" log -1 --pretty=%ct 2>/dev/null || echo "${REPRODUCIBLE_TIMESTAMP_ROOTFS}"`
 		fi
 
 		ts=`LC_ALL=C date -d @$SOURCE_DATE_EPOCH`
@@ -570,9 +566,9 @@ RDEPENDS_${KERNEL_PACKAGE_NAME} = "${KERNEL_PACKAGE_NAME}-base"
 # Allow machines to override this dependency if kernel image files are
 # not wanted in images as standard
 RDEPENDS_${KERNEL_PACKAGE_NAME}-base ?= "${KERNEL_PACKAGE_NAME}-image"
-PKG_${KERNEL_PACKAGE_NAME}-image = "${KERNEL_PACKAGE_NAME}-image-${@legitimize_package_name('${KERNEL_VERSION}')}"
+PKG_${KERNEL_PACKAGE_NAME}-image = "${KERNEL_PACKAGE_NAME}-image-${@legitimize_package_name(d.getVar('KERNEL_VERSION'))}"
 RDEPENDS_${KERNEL_PACKAGE_NAME}-image += "${@oe.utils.conditional('KERNEL_IMAGETYPE', 'vmlinux', '${KERNEL_PACKAGE_NAME}-vmlinux', '', d)}"
-PKG_${KERNEL_PACKAGE_NAME}-base = "${KERNEL_PACKAGE_NAME}-${@legitimize_package_name('${KERNEL_VERSION}')}"
+PKG_${KERNEL_PACKAGE_NAME}-base = "${KERNEL_PACKAGE_NAME}-${@legitimize_package_name(d.getVar('KERNEL_VERSION'))}"
 RPROVIDES_${KERNEL_PACKAGE_NAME}-base += "${KERNEL_PACKAGE_NAME}-${KERNEL_VERSION}"
 ALLOW_EMPTY_${KERNEL_PACKAGE_NAME} = "1"
 ALLOW_EMPTY_${KERNEL_PACKAGE_NAME}-base = "1"

meta/classes/kernelsrc.bbclass

@@ -1,7 +1,7 @@
 S = "${STAGING_KERNEL_DIR}"
 deltask do_fetch
 deltask do_unpack
-do_patch[depends] += "virtual/kernel:do_patch"
+do_patch[depends] += "virtual/kernel:do_shared_workdir"
 do_patch[noexec] = "1"
 do_package[depends] += "virtual/kernel:do_populate_sysroot"
 KERNEL_VERSION = "${@get_kernelversion_file("${STAGING_KERNEL_BUILDDIR}")}"

meta/classes/mime-xdg.bbclass

@@ -8,7 +8,7 @@ PACKAGE_WRITE_DEPS += "desktop-file-utils-native"
 DESKTOPDIR = "${datadir}/applications"
 
 # There are recipes out there installing their .desktop files as absolute
-# symlinks. For us these are dangling and cannot be introspected for "MymeType"
+# symlinks. For us these are dangling and cannot be introspected for "MimeType"
 # easily. By addding package-names to MIME_XDG_PACKAGES, packager can force
 # proper update-desktop-database handling. Note that all introspection is
 # skipped for MIME_XDG_PACKAGES not empty

meta/classes/populate_sdk_ext.bbclass

@@ -388,9 +388,13 @@ python copy_buildsystem () {
         bb.utils.mkdirhier(os.path.join(baseoutpath, 'cache'))
         shutil.copyfile(builddir + '/cache/bb_unihashes.dat', baseoutpath + '/cache/bb_unihashes.dat')
 
-    # Write a templateconf.cfg
-    with open(baseoutpath + '/conf/templateconf.cfg', 'w') as f:
-        f.write('meta/conf\n')
+    # Use templateconf.cfg file from builddir if exists
+    if os.path.exists(builddir + '/conf/templateconf.cfg'):
+        shutil.copyfile(builddir + '/conf/templateconf.cfg', baseoutpath + '/conf/templateconf.cfg')
+    else:
+        # Write a templateconf.cfg
+        with open(baseoutpath + '/conf/templateconf.cfg', 'w') as f:
+            f.write('meta/conf\n')
 
     # Ensure any variables set from the external environment (by way of
     # BB_ENV_EXTRAWHITE) are set in the SDK's configuration

meta/classes/reproducible_build.bbclass

@@ -150,11 +150,12 @@ def fixed_source_date_epoch():
     bb.debug(1, "No tarball or git repo found to determine SOURCE_DATE_EPOCH")
     return 0
 
-python do_create_source_date_epoch_stamp() {
+python create_source_date_epoch_stamp() {
     epochfile = d.getVar('SDE_FILE')
+    # If it exists we need to regenerate as the sources may have changed
     if os.path.isfile(epochfile):
-        bb.debug(1, "Reusing SOURCE_DATE_EPOCH from: %s" % epochfile)
-        return
+        bb.debug(1, "Deleting existing SOURCE_DATE_EPOCH from: %s" % epochfile)
+        os.remove(epochfile)
 
     sourcedir = d.getVar('S')
     source_date_epoch = (
@@ -197,5 +198,5 @@ BB_HASHBASE_WHITELIST += "SOURCE_DATE_EPOCH"
 
 python () {
     if d.getVar('BUILD_REPRODUCIBLE_BINARIES') == '1':
-        d.appendVarFlag("do_unpack", "postfuncs", " do_create_source_date_epoch_stamp")
+        d.appendVarFlag("do_unpack", "postfuncs", " create_source_date_epoch_stamp")
 }

meta/classes/sstate.bbclass

@@ -966,10 +966,6 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
             if len(tasklist) >= min_tasks:
                 bb.event.fire(bb.event.ProcessFinished(msg), d)
 
-    # Likely checking an individual task hash again for multiconfig sharing of sstate tasks so skip reporting
-    if len(sq_data['hash']) == 1:
-        return found
-
     inheritlist = d.getVar("INHERIT")
     if "toaster" in inheritlist:
         evdata = {'missed': [], 'found': []};

meta/conf/bitbake.conf

@@ -795,7 +795,10 @@ BB_NUMBER_THREADS ?= "${@oe.utils.cpu_count()}"
 PARALLEL_MAKE ?= "-j ${@oe.utils.cpu_count()}"
 
 # Default parallelism and resource usage for xz
-XZ_DEFAULTS ?= "--memlimit=50% --threads=${@oe.utils.cpu_count()}"
+XZ_MEMLIMIT ?= "50%"
+XZ_THREADS ?= "${@oe.utils.cpu_count(at_least=2)}"
+XZ_DEFAULTS ?= "--memlimit=${XZ_MEMLIMIT} --threads=${XZ_THREADS}"
+XZ_DEFAULTS[vardepsexclude] += "XZ_MEMLIMIT XZ_THREADS"
 
 ##################################################################
 # Magic Cookie for SANITY CHECK

meta/conf/distro/include/distro_alias.inc

@@ -15,6 +15,7 @@ DISTRO_PN_ALIAS_pn-alsa-utils-scripts = "OE-Core"
 DISTRO_PN_ALIAS_pn-atk = "Fedora=atk OpenSuSE=atk"
 DISTRO_PN_ALIAS_pn-avahi-ui = "Ubuntu=avahi-discover Debian=avahi-discover"
 DISTRO_PN_ALIAS_pn-babeltrace = "OSPDT"
+DISTRO_PN_ALIAS_pn-babeltrace2 = "OSPDT"
 DISTRO_PN_ALIAS_pn-bjam = "OpenSuSE=boost-jam Debian=bjam"
 DISTRO_PN_ALIAS_pn-blktool = "Debian=blktool Mandriva=blktool"
 DISTRO_PN_ALIAS_pn-bluez5 = "Fedora=bluez  Opensuse=bluez"

meta/conf/distro/include/maintainers.inc

@@ -59,6 +59,7 @@ RECIPE_MAINTAINER_pn-automake = "Robert Yang <liezhi.yang@windriver.com>"
 RECIPE_MAINTAINER_pn-avahi = "Yi Zhao <yi.zhao@windriver.com>"
 RECIPE_MAINTAINER_pn-avahi-ui = "Yi Zhao <yi.zhao@windriver.com>"
 RECIPE_MAINTAINER_pn-babeltrace = "Alexander Kanavin <alex.kanavin@gmail.com>"
+RECIPE_MAINTAINER_pn-babeltrace2 = "Alexander Kanavin <alex.kanavin@gmail.com>"
 RECIPE_MAINTAINER_pn-base-files = "Anuj Mittal <anuj.mittal@intel.com>"
 RECIPE_MAINTAINER_pn-base-passwd = "Anuj Mittal <anuj.mittal@intel.com>"
 RECIPE_MAINTAINER_pn-bash = "Hongxu Jia <hongxu.jia@windriver.com>"
@@ -193,7 +194,7 @@ RECIPE_MAINTAINER_pn-gcc-cross-canadian-${TRANSLATED_TARGET_ARCH} = "Khem Raj <r
 RECIPE_MAINTAINER_pn-gcc-crosssdk-${SDK_SYS} = "Khem Raj <raj.khem@gmail.com>"
 RECIPE_MAINTAINER_pn-gcc-runtime = "Khem Raj <raj.khem@gmail.com>"
 RECIPE_MAINTAINER_pn-gcc-sanitizers = "Khem Raj <raj.khem@gmail.com>"
-RECIPE_MAINTAINER_pn-gcc-source-9.2.0 = "Khem Raj <raj.khem@gmail.com>"
+RECIPE_MAINTAINER_pn-gcc-source-9.3.0 = "Khem Raj <raj.khem@gmail.com>"
 RECIPE_MAINTAINER_pn-gconf = "Ross Burton <ross.burton@intel.com>"
 RECIPE_MAINTAINER_pn-gcr = "Alexander Kanavin <alex.kanavin@gmail.com>"
 RECIPE_MAINTAINER_pn-gdb = "Khem Raj <raj.khem@gmail.com>"
@@ -525,8 +526,6 @@ RECIPE_MAINTAINER_pn-nfs-export-root = "Robert Yang <liezhi.yang@windriver.com>"
 RECIPE_MAINTAINER_pn-nfs-utils = "Robert Yang <liezhi.yang@windriver.com>"
 RECIPE_MAINTAINER_pn-ninja = "Khem Raj <raj.khem@gmail.com>"
 RECIPE_MAINTAINER_pn-npth = "Alexander Kanavin <alex.kanavin@gmail.com>"
-RECIPE_MAINTAINER_pn-nspr = "Armin Kuster <akuster808@gmail.com>"
-RECIPE_MAINTAINER_pn-nss = "Armin Kuster <akuster808@gmail.com>"
 RECIPE_MAINTAINER_pn-nss-myhostname = "Anuj Mittal <anuj.mittal@intel.com>"
 RECIPE_MAINTAINER_pn-ofono = "Ross Burton <ross.burton@intel.com>"
 RECIPE_MAINTAINER_pn-opensbi = "Alistair Francis <alistair.francis@wdc.com>"

meta/conf/distro/include/ptest-packagelists.inc

@@ -64,7 +64,9 @@ PTESTS_FAST = "\
 
 PTESTS_SLOW = "\
     babeltrace-ptest \
+    babeltrace2-ptest \
     busybox-ptest \
+    coreutils-ptest \
     dbus-test-ptest \
     e2fsprogs-ptest \
     glib-2.0-ptest \

meta/conf/distro/include/tcmode-default.inc

@@ -20,7 +20,7 @@ GCCVERSION ?= "9.%"
 SDKGCCVERSION ?= "${GCCVERSION}"
 BINUVERSION ?= "2.34%"
 GDBVERSION ?= "9.%"
-GLIBCVERSION ?= "2.31"
+GLIBCVERSION ?= "2.31%"
 LINUXLIBCVERSION ?= "5.4%"
 QEMUVERSION ?= "4.2%"
 GOVERSION ?= "1.14%"

meta/conf/distro/include/yocto-uninative.inc

@@ -6,9 +6,9 @@
 # to the distro running on the build machine.
 #
 
-UNINATIVE_MAXGLIBCVERSION = "2.30"
+UNINATIVE_MAXGLIBCVERSION = "2.31"
 
-UNINATIVE_URL ?= "http://downloads.yoctoproject.org/releases/uninative/2.7/"
-UNINATIVE_CHECKSUM[aarch64] ?= "e76a45886ee8a0b3904b761c17ac8ff91edf9811ee455f1832d10763ba794dfc"
-UNINATIVE_CHECKSUM[i686] ?= "810d027dfb1c7675226afbcec07808770516c969ee7378f6d8240281083f8924"
-UNINATIVE_CHECKSUM[x86_64] ?= "9498d8bba047499999a7310ac2576d0796461184965351a56f6d32c888a1f216"
+UNINATIVE_URL ?= "http://downloads.yoctoproject.org/releases/uninative/2.8/"
+UNINATIVE_CHECKSUM[aarch64] ?= "989187344bf9539b464fb7ed9c223e51f4bdb4c7a677d2c314e6fed393176efe"
+UNINATIVE_CHECKSUM[i686] ?= "cc3e45bc8594488b407363e3fa9af5a099279dab2703c64342098719bd674990"
+UNINATIVE_CHECKSUM[x86_64] ?= "a09922172c3a439105e0ae6b943daad2d83505b17da0aba97961ff433b8c21ab"

meta/conf/layer.conf

@@ -7,12 +7,12 @@ BBFILE_COLLECTIONS += "core"
 BBFILE_PATTERN_core = "^${LAYERDIR}/"
 BBFILE_PRIORITY_core = "5"
 
-LAYERSERIES_CORENAMES = "zeus"
+LAYERSERIES_CORENAMES = "dunfell zeus"
 
 # This should only be incremented on significant changes that will
 # cause compatibility issues with other layers
 LAYERVERSION_core = "11"
-LAYERSERIES_COMPAT_core = "warrior zeus"
+LAYERSERIES_COMPAT_core = "zeus dunfell"
 
 BBLAYERS_LAYERINDEX_NAME_core = "openembedded-core"
 

meta/conf/machine/qemuarm64.conf

@@ -13,7 +13,7 @@ SERIAL_CONSOLES ?= "115200;ttyAMA0 115200;hvc0"
 QB_SYSTEM_NAME = "qemu-system-aarch64"
 QB_MACHINE = "-machine virt"
 QB_CPU = "-cpu cortex-a57"
-QB_CPU_KVM = "-cpu host"
+QB_CPU_KVM = "-cpu host -machine gic-version=3"
 # Standard Serial console
 QB_KERNEL_CMDLINE_APPEND = "console=ttyAMA0"
 # For graphics to work we need to define the VGA device as well as the necessary USB devices

meta/files/toolchain-shar-extract.sh

@@ -1,13 +1,8 @@
 #!/bin/sh
 
-[ -z "$ENVCLEANED" ] && exec /usr/bin/env -i ENVCLEANED=1 HOME="$HOME" \
-	LC_ALL=en_US.UTF-8 \
-	TERM=$TERM \
-	ICECC_PATH="$ICECC_PATH" \
-	http_proxy="$http_proxy" https_proxy="$https_proxy" ftp_proxy="$ftp_proxy" \
-	no_proxy="$no_proxy" all_proxy="$all_proxy" GIT_PROXY_COMMAND="$GIT_PROXY_COMMAND" "$0" "$@"
-[ -f /etc/environment ] && . /etc/environment
-export PATH=`echo "$PATH" | sed -e 's/:\.//' -e 's/::/:/'`
+export LC_ALL=en_US.UTF-8
+# Remove invalid PATH elements first (maybe from a previously setup toolchain now deleted
+PATH=`python3 -c 'import os; print(":".join(e for e in os.environ["PATH"].split(":") if os.path.exists(e)))'`
 
 tweakpath () {
     case ":${PATH}:" in

meta/lib/oe/utils.py

@@ -248,9 +248,10 @@ def trim_version(version, num_parts=2):
     trimmed = ".".join(parts[:num_parts])
     return trimmed
 
-def cpu_count():
+def cpu_count(at_least=1):
     import multiprocessing
-    return multiprocessing.cpu_count()
+    cpus = multiprocessing.cpu_count()
+    return max(cpus, at_least)
 
 def execute_pre_post_process(d, cmds):
     if cmds is None:

meta/lib/oeqa/core/context.py

@@ -116,6 +116,9 @@ class OETestContextExecutor(object):
                 default=self.default_output_log,
                 help="results output log, default: %s" % self.default_output_log)
 
+        self.parser.add_argument('--json-result-dir', action='store',
+                help="json result output dir, create testresults.json here if set")
+
         group = self.parser.add_mutually_exclusive_group()
         group.add_argument('--run-tests', action='store', nargs='+',
                 default=self.default_tests,
@@ -178,6 +181,22 @@ class OETestContextExecutor(object):
 
         self.module_paths = args.CASES_PATHS
 
+    def _get_json_result_dir(self, args):
+        return args.json_result_dir
+
+    def _get_configuration(self):
+        td = self.tc_kwargs['init']['td']
+        configuration = {'TEST_TYPE': self.name,
+                        'MACHINE': td.get("MACHINE"),
+                        'DISTRO': td.get("DISTRO"),
+                        'IMAGE_BASENAME': td.get("IMAGE_BASENAME"),
+                        'DATETIME': td.get("DATETIME")}
+        return configuration
+
+    def _get_result_id(self, configuration):
+        return '%s_%s_%s_%s' % (configuration['TEST_TYPE'], configuration['IMAGE_BASENAME'],
+                                configuration['MACHINE'], configuration['DATETIME'])
+
     def _pre_run(self):
         pass
 
@@ -196,7 +215,16 @@ class OETestContextExecutor(object):
         else:
             self._pre_run()
             rc = self.tc.runTests(**self.tc_kwargs['run'])
-            rc.logDetails()
+
+            json_result_dir = self._get_json_result_dir(args)
+            if json_result_dir:
+                configuration = self._get_configuration()
+                rc.logDetails(json_result_dir,
+                              configuration,
+                              self._get_result_id(configuration))
+            else:
+                rc.logDetails()
+
             rc.logSummary(self.name)
 
         output_link = os.path.join(os.path.dirname(args.output_log),

meta/lib/oeqa/core/runner.py

@@ -319,10 +319,17 @@ class OETestResultJSONHelper(object):
             the_file.write(file_content)
 
     def dump_testresult_file(self, write_dir, configuration, result_id, test_result):
-        bb.utils.mkdirhier(write_dir)
-        lf = bb.utils.lockfile(os.path.join(write_dir, 'jsontestresult.lock'))
+        try:
+            import bb
+            has_bb = True
+            bb.utils.mkdirhier(write_dir)
+            lf = bb.utils.lockfile(os.path.join(write_dir, 'jsontestresult.lock'))
+        except ImportError:
+            has_bb = False
+            os.makedirs(write_dir, exist_ok=True)
         test_results = self._get_existing_testresults_if_available(write_dir)
         test_results[result_id] = {'configuration': configuration, 'result': test_result}
         json_testresults = json.dumps(test_results, sort_keys=True, indent=4)
         self._write_file(write_dir, self.testresult_filename, json_testresults)
-        bb.utils.unlockfile(lf)
+        if has_bb:
+            bb.utils.unlockfile(lf)

meta/lib/oeqa/runtime/cases/ltp.py

@@ -59,7 +59,7 @@ class LtpTest(LtpTestBase):
 
     ltp_groups = ["math", "syscalls", "dio", "io", "mm", "ipc", "sched", "nptl", "pty", "containers", "controllers", "filecaps", "cap_bounds", "fcntl-locktests", "connectors", "commands", "net.ipv6_lib", "input","fs_perms_simple"]
 
-    ltp_fs = ["fs", "fsx", "fs_bind", "fs_ext4"]
+    ltp_fs = ["fs", "fsx", "fs_bind"]
     # skip kernel cpuhotplug
     ltp_kernel = ["power_management_tests", "hyperthreading ", "kernel_misc", "hugetlb"]
     ltp_groups += ltp_fs

meta/lib/oeqa/runtime/cases/parselogs.py

@@ -56,7 +56,8 @@ common_errors = [
     "error retry time-out =",
     "logind: cannot setup systemd-logind helper (-61), using legacy fallback",
     "Error changing net interface name 'eth0' to ",
-    "Cannot find a map file"
+    "Cannot find a map file",
+    "[rdrand]: Initialization Failed"
     ]
 
 video_related = [
@@ -130,6 +131,7 @@ ignore_errors = {
         '(EE) Server terminated with error (1). Closing log file.',
         'dmi: Firmware registration failed.',
         'irq: type mismatch, failed to map hwirq-27 for /intc',
+        'logind: failed to get session seat',
         ] + common_errors,
     'intel-core2-32' : [
         'ACPI: No _BQC method, cannot determine initial brightness',

meta/lib/oeqa/runtime/cases/ptest.py

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: MIT
 #
 
+import os
 import unittest
 import pprint
 import datetime

meta/lib/oeqa/runtime/context.py

@@ -77,7 +77,7 @@ class OERuntimeTestContextExecutor(OETestContextExecutor):
 
         runtime_group.add_argument('--packages-manifest', action='store',
                 default=self.default_manifest,
-                help="Package manifest of the image under testi, default: %s" \
+                help="Package manifest of the image under test, default: %s" \
                 % self.default_manifest)
 
         runtime_group.add_argument('--extract-dir', action='store',
@@ -184,7 +184,7 @@ class OERuntimeTestContextExecutor(OETestContextExecutor):
         except:
             obj = None
         return obj
-        
+
     @staticmethod
     def readPackagesManifest(manifest):
         if not manifest or not os.path.exists(manifest):

meta/lib/oeqa/sdkext/testsdk.py

@@ -25,11 +25,8 @@ class TestSDKExt(TestSDKBase):
 
         subprocesstweak.errors_have_output()
 
-        # extensible sdk can be contaminated if native programs are
-        # in PATH, i.e. use perl-native instead of eSDK one.
-        paths_to_avoid = [d.getVar('STAGING_DIR'),
-                        d.getVar('BASE_WORKDIR')]
-        os.environ['PATH'] = avoid_paths_in_environ(paths_to_avoid)
+        # We need the original PATH for testing the eSDK, not with our manipulations
+        os.environ['PATH'] = d.getVar("BB_ORIGENV", False).getVar("PATH")
 
         tcname = d.expand("${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.sh")
         if not os.path.exists(tcname):

meta/lib/oeqa/selftest/context.py

@@ -46,7 +46,7 @@ class OESelftestTestContext(OETestContext):
         oe.path.copytree(selftestdir, newselftestdir)
 
         for e in os.environ:
-            if builddir in os.environ[e]:
+            if builddir + "/" in os.environ[e] or os.environ[e].endswith(builddir):
                 os.environ[e] = os.environ[e].replace(builddir, newbuilddir)
 
         subprocess.check_output("git init; git add *; git commit -a -m 'initial'", cwd=newselftestdir, shell=True)

meta/recipes-connectivity/bluez5/bluez5.inc

@@ -52,6 +52,8 @@ SRC_URI = "${KERNELORG_MIRROR}/linux/bluetooth/bluez-${PV}.tar.xz \
            ${@bb.utils.contains('DISTRO_FEATURES', 'systemd', '', 'file://0001-Allow-using-obexd-without-systemd-in-the-user-sessio.patch', d)} \
            file://0001-tests-add-a-target-for-building-tests-without-runnin.patch \
            file://0001-test-gatt-Fix-hung-issue.patch \
+           file://CVE-2020-0556-1.patch \
+           file://CVE-2020-0556-2.patch \
            "
 S = "${WORKDIR}/bluez-${PV}"
 

meta/recipes-connectivity/bluez5/bluez5/CVE-2020-0556-1.patch

@@ -0,0 +1,35 @@
+From 8cdbd3b09f29da29374e2f83369df24228da0ad1 Mon Sep 17 00:00:00 2001
+From: Alain Michaud <alainm@chromium.org>
+Date: Tue, 10 Mar 2020 02:35:16 +0000
+Subject: [PATCH 1/2] HOGP must only accept data from bonded devices.
+
+HOGP 1.0 Section 6.1 establishes that the HOGP must require bonding.
+
+Reference:
+https://www.intel.com/content/www/us/en/security-center/advisory/intel-sa-00352.htm
+
+Upstream-Status: Backport [https://git.kernel.org/pub/scm/bluetooth/bluez.git/commit/?id=8cdbd3b09f29da29374e2f83369df24228da0ad1]
+Signed-off-by: Anuj Mittal <anuj.mittal@intel.com>
+CVE: CVE-2020-0556
+---
+ profiles/input/hog.c | 4 ++++
+ 1 file changed, 4 insertions(+)
+
+diff --git a/profiles/input/hog.c b/profiles/input/hog.c
+index 83c017dcb..dfac68921 100644
+--- a/profiles/input/hog.c
++++ b/profiles/input/hog.c
+@@ -186,6 +186,10 @@ static int hog_accept(struct btd_service *service)
+ 			return -EINVAL;
+ 	}
+ 
++	/* HOGP 1.0 Section 6.1 requires bonding */
++	if (!device_is_bonded(device, btd_device_get_bdaddr_type(device)))
++		return -ECONNREFUSED;
++
+ 	/* TODO: Replace GAttrib with bt_gatt_client */
+ 	bt_hog_attach(dev->hog, attrib);
+ 
+-- 
+2.24.1
+

meta/recipes-connectivity/bluez5/bluez5/CVE-2020-0556-2.patch

@@ -0,0 +1,143 @@
+From 3cccdbab2324086588df4ccf5f892fb3ce1f1787 Mon Sep 17 00:00:00 2001
+From: Alain Michaud <alainm@chromium.org>
+Date: Tue, 10 Mar 2020 02:35:18 +0000
+Subject: [PATCH 2/2] HID accepts bonded device connections only.
+
+This change adds a configuration for platforms to choose a more secure
+posture for the HID profile.  While some older mice are known to not
+support pairing or encryption, some platform may choose a more secure
+posture by requiring the device to be bonded  and require the
+connection to be encrypted when bonding is required.
+
+Reference:
+https://www.intel.com/content/www/us/en/security-center/advisory/intel-sa-00352.html
+
+Upstream-Status: Backport [https://git.kernel.org/pub/scm/bluetooth/bluez.git/commit/?id=3cccdbab2324086588df4ccf5f892fb3ce1f1787]
+Signed-off-by: Anuj Mittal <anuj.mittal@intel.com>
+CVE: CVE-2020-0556
+
+---
+ profiles/input/device.c   | 23 ++++++++++++++++++++++-
+ profiles/input/device.h   |  1 +
+ profiles/input/input.conf |  8 ++++++++
+ profiles/input/manager.c  | 13 ++++++++++++-
+ 4 files changed, 43 insertions(+), 2 deletions(-)
+
+diff --git a/profiles/input/device.c b/profiles/input/device.c
+index 2cb3811c8..d89da2d7c 100644
+--- a/profiles/input/device.c
++++ b/profiles/input/device.c
+@@ -92,6 +92,7 @@ struct input_device {
+ 
+ static int idle_timeout = 0;
+ static bool uhid_enabled = false;
++static bool classic_bonded_only = false;
+ 
+ void input_set_idle_timeout(int timeout)
+ {
+@@ -103,6 +104,11 @@ void input_enable_userspace_hid(bool state)
+ 	uhid_enabled = state;
+ }
+ 
++void input_set_classic_bonded_only(bool state)
++{
++	classic_bonded_only = state;
++}
++
+ static void input_device_enter_reconnect_mode(struct input_device *idev);
+ static int connection_disconnect(struct input_device *idev, uint32_t flags);
+ 
+@@ -970,8 +976,18 @@ static int hidp_add_connection(struct input_device *idev)
+ 	if (device_name_known(idev->device))
+ 		device_get_name(idev->device, req->name, sizeof(req->name));
+ 
++	/* Make sure the device is bonded if required */
++	if (classic_bonded_only && !device_is_bonded(idev->device,
++				btd_device_get_bdaddr_type(idev->device))) {
++		error("Rejected connection from !bonded device %s", dst_addr);
++		goto cleanup;
++	}
++
+ 	/* Encryption is mandatory for keyboards */
+-	if (req->subclass & 0x40) {
++	/* Some platforms may choose to require encryption for all devices */
++	/* Note that this only matters for pre 2.1 devices as otherwise the */
++	/* device is encrypted by default by the lower layers */
++	if (classic_bonded_only || req->subclass & 0x40) {
+ 		if (!bt_io_set(idev->intr_io, &gerr,
+ 					BT_IO_OPT_SEC_LEVEL, BT_IO_SEC_MEDIUM,
+ 					BT_IO_OPT_INVALID)) {
+@@ -1203,6 +1219,11 @@ static void input_device_enter_reconnect_mode(struct input_device *idev)
+ 	DBG("path=%s reconnect_mode=%s", idev->path,
+ 				reconnect_mode_to_string(idev->reconnect_mode));
+ 
++	/* Make sure the device is bonded if required */
++	if (classic_bonded_only && !device_is_bonded(idev->device,
++				btd_device_get_bdaddr_type(idev->device)))
++		return;
++
+ 	/* Only attempt an auto-reconnect when the device is required to
+ 	 * accept reconnections from the host.
+ 	 */
+diff --git a/profiles/input/device.h b/profiles/input/device.h
+index 51a9aee18..3044db673 100644
+--- a/profiles/input/device.h
++++ b/profiles/input/device.h
+@@ -29,6 +29,7 @@ struct input_conn;
+ 
+ void input_set_idle_timeout(int timeout);
+ void input_enable_userspace_hid(bool state);
++void input_set_classic_bonded_only(bool state);
+ 
+ int input_device_register(struct btd_service *service);
+ void input_device_unregister(struct btd_service *service);
+diff --git a/profiles/input/input.conf b/profiles/input/input.conf
+index 3e1d65aae..166aff4a4 100644
+--- a/profiles/input/input.conf
++++ b/profiles/input/input.conf
+@@ -11,3 +11,11 @@
+ # Enable HID protocol handling in userspace input profile
+ # Defaults to false (HIDP handled in HIDP kernel module)
+ #UserspaceHID=true
++
++# Limit HID connections to bonded devices
++# The HID Profile does not specify that devices must be bonded, however some
++# platforms may want to make sure that input connections only come from bonded
++# device connections. Several older mice have been known for not supporting
++# pairing/encryption.
++# Defaults to false to maximize device compatibility.
++#ClassicBondedOnly=true
+diff --git a/profiles/input/manager.c b/profiles/input/manager.c
+index 1d31b0652..5cd27b839 100644
+--- a/profiles/input/manager.c
++++ b/profiles/input/manager.c
+@@ -96,7 +96,7 @@ static int input_init(void)
+ 	config = load_config_file(CONFIGDIR "/input.conf");
+ 	if (config) {
+ 		int idle_timeout;
+-		gboolean uhid_enabled;
++		gboolean uhid_enabled, classic_bonded_only;
+ 
+ 		idle_timeout = g_key_file_get_integer(config, "General",
+ 							"IdleTimeout", &err);
+@@ -114,6 +114,17 @@ static int input_init(void)
+ 			input_enable_userspace_hid(uhid_enabled);
+ 		} else
+ 			g_clear_error(&err);
++
++		classic_bonded_only = g_key_file_get_boolean(config, "General",
++						"ClassicBondedOnly", &err);
++
++		if (!err) {
++			DBG("input.conf: ClassicBondedOnly=%s",
++					classic_bonded_only ? "true" : "false");
++			input_set_classic_bonded_only(classic_bonded_only);
++		} else
++			g_clear_error(&err);
++
+ 	}
+ 
+ 	btd_profile_register(&input_profile);
+-- 
+2.24.1
+

meta/recipes-connectivity/openssl/openssl_1.1.1d.bb

@@ -34,7 +34,7 @@ PACKAGECONFIG ?= ""
 PACKAGECONFIG_class-native = ""
 PACKAGECONFIG_class-nativesdk = ""
 
-PACKAGECONFIG[cryptodev-linux] = "enable-devcryptoeng,disable-devcryptoeng,cryptodev-linux"
+PACKAGECONFIG[cryptodev-linux] = "enable-devcryptoeng,disable-devcryptoeng,cryptodev-linux,,cryptodev-module"
 
 B = "${WORKDIR}/build"
 do_configure[cleandirs] = "${B}"
@@ -123,7 +123,7 @@ do_configure () {
 	fi
 	# WARNING: do not set compiler/linker flags (-I/-D etc.) in EXTRA_OECONF, as they will fully replace the
 	# environment variables set by bitbake. Adjust the environment variables instead.
-	PERL5LIB="${S}/external/perl/Text-Template-1.46/lib/" \
+	HASHBANGPERL="/usr/bin/env perl" PERL=perl PERL5LIB="${S}/external/perl/Text-Template-1.46/lib/" \
 	perl ${S}/Configure ${EXTRA_OECONF} ${PACKAGECONFIG_CONFARGS} --prefix=$useprefix --openssldir=${libdir}/ssl-1.1 --libdir=${libdir} $target
 	perl ${B}/configdata.pm --dump
 }

meta/recipes-core/coreutils/coreutils/run-ptest

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+# remove any stale lock files so that the calls to groupadd/useradd don't stop
+# the ptest if re-using the same image
+rm -rf /etc/passwd.lock /etc/group.lock /etc/gshadow.lock
+
+COREUTILSLIB=@libdir@/coreutils
+LOG="${COREUTILSLIB}/ptest/coreutils_ptest_$(date +%Y%m%d-%H%M%S).log"
+USERNAME="tester"
+groupadd ugroup1
+groupadd ugroup2
+useradd -G ugroup1,ugroup2 $USERNAME || echo "user $USERNAME already exists"
+
+su tester -c "cd ${COREUTILSLIB}/ptest && make check-TESTS top_srcdir=. srcdir=." 2>&1 | tee -a ${LOG}
+userdel $USERNAME 
+groupdel ugroup1
+groupdel ugroup2

meta/recipes-core/coreutils/coreutils_8.31.bb

@@ -18,6 +18,7 @@ SRC_URI = "${GNU_MIRROR}/coreutils/${BP}.tar.xz \
            file://0001-uname-report-processor-and-hardware-correctly.patch \
            file://disable-ls-output-quoting.patch \
            file://0001-local.mk-fix-cross-compiling-problem.patch \
+           file://run-ptest \
           "
 
 SRC_URI_append_libc-musl = "file://strtod_fix_clash_with_strtold.patch"
@@ -143,3 +144,47 @@ python __anonymous() {
 }
 
 BBCLASSEXTEND = "native nativesdk"
+
+inherit ptest
+
+RDEPENDS_${PN}-ptest += "bash findutils gawk liberror-perl libmodule-build-perl make perl perl-module-file-stat python3-core sed shadow"
+
+# -dev automatic dependencies fails as we don't want libmodule-build-perl-dev, its too heavy
+# may need tweaking if DEPENDS changes
+RRECOMMENDS_coreutils-dev[nodeprrecs] = "1"
+RRECOMMENDS_coreutils-dev = "acl-dev attr-dev gmp-dev libcap-dev bash-dev findutils-dev gawk-dev shadow-dev"
+
+do_install_ptest () {
+    install -d ${D}${PTEST_PATH}/tests
+    cp -r ${S}/tests/* ${D}${PTEST_PATH}/tests
+    sed -i 's/ginstall/install/g'  `grep -R ginstall ${D}${PTEST_PATH}/tests | awk -F: '{print $1}' | uniq`
+    install -d ${D}${PTEST_PATH}/build-aux
+    install ${S}/build-aux/test-driver ${D}${PTEST_PATH}/build-aux/
+    cp ${B}/Makefile ${D}${PTEST_PATH}/
+    cp ${S}/init.cfg ${D}${PTEST_PATH}/
+    cp -r ${B}/src ${D}${PTEST_PATH}/
+    cp -r ${S}/src/*.c ${D}${PTEST_PATH}/src
+    sed -i '/^VPATH/s/= .*$/= ./g' ${D}${PTEST_PATH}/Makefile
+    sed -i '/^PROGRAMS/s/^/#/g' ${D}${PTEST_PATH}/Makefile
+    sed -i '/^Makefile: /s/^.*$/Makefile:/g' ${D}${PTEST_PATH}/Makefile
+    sed -i '/^abs_srcdir/s/= .*$/= \$\{PWD\}/g' ${D}${PTEST_PATH}/Makefile
+    sed -i '/^abs_top_builddir/s/= .*$/= \$\{PWD\}/g' ${D}${PTEST_PATH}/Makefile
+    sed -i '/^abs_top_srcdir/s/= .*$/= \$\{PWD\}/g' ${D}${PTEST_PATH}/Makefile
+    sed -i '/^built_programs/s/ginstall/install/g' ${D}${PTEST_PATH}/Makefile
+    chmod -R 777 ${D}${PTEST_PATH}
+
+    # Disable subcase stty-pairs.sh, it will cause test framework hang
+    sed -i '/stty-pairs.sh/d' ${D}${PTEST_PATH}/Makefile
+
+    # Disable subcase tail-2/assert.sh as it has issues on 32-bit systems
+    sed -i '/assert.sh/d' ${D}${PTEST_PATH}/Makefile
+
+    # Tweak test d_type-check to use python3 instead of python
+    sed -i "1s@.*@#!/usr/bin/python3@" ${D}${PTEST_PATH}/tests/d_type-check
+    install ${B}/src/getlimits ${D}/${bindir}
+    
+    # handle multilib
+    sed -i s:@libdir@:${libdir}:g ${D}${PTEST_PATH}/run-ptest
+}
+
+FILES_${PN}-ptest += "${bindir}/getlimits"

meta/recipes-core/gettext/gettext-0.20.1/0001-init-env.in-do-not-add-C-CXX-parameters.patch

@@ -0,0 +1,29 @@
+From 9b912a47f790a7b282ec0c2295a188c5d8fb6a7c Mon Sep 17 00:00:00 2001
+From: Alexander Kanavin <alex.kanavin@gmail.com>
+Date: Fri, 6 Mar 2020 21:04:05 +0000
+Subject: [PATCH] init-env.in: do not add C/CXX parameters
+
+These are taken from the cross environment and include
+sysroot paths, so are not reproducible.
+
+Upstream-Status: Inappropriate [oe-core specific]
+Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
+---
+ gettext-tools/tests/init-env.in | 4 ----
+ 1 file changed, 4 deletions(-)
+
+diff --git a/gettext-tools/tests/init-env.in b/gettext-tools/tests/init-env.in
+index cc84ffd..b69c990 100644
+--- a/gettext-tools/tests/init-env.in
++++ b/gettext-tools/tests/init-env.in
+@@ -3,10 +3,6 @@ top_builddir=../..
+ 
+ OBJEXT="@OBJEXT@"
+ EXEEXT="@EXEEXT@"
+-CC="@CC@"
+-CFLAGS="@CFLAGS@"
+-CXX="@CXX@"
+-CXXFLAGS="@CXXFLAGS@"
+ CPPFLAGS="@CPPFLAGS@"
+ LDFLAGS="@LDFLAGS@"
+ LTLIBINTL="@LTLIBINTL@"

meta/recipes-core/gettext/gettext_0.20.1.bb

@@ -25,6 +25,7 @@ SRC_URI = "${GNU_MIRROR}/gettext/gettext-${PV}.tar.gz \
            file://serial-tests-config.patch \
            file://0001-msgmerge-Fix-behaviour-of-for-msgfmt-on-PO-files-wit.patch \
            file://0001-tests-autopoint-3-unset-MAKEFLAGS.patch \
+           file://0001-init-env.in-do-not-add-C-CXX-parameters.patch \
            "
 SRC_URI[md5sum] = "bb5b0c0caa028105f3ca1905ddc306e2"
 SRC_URI[sha256sum] = "66415634c6e8c3fa8b71362879ec7575e27da43da562c798a8a2f223e6e47f5c"

meta/recipes-core/glib-2.0/glib.inc

@@ -128,6 +128,9 @@ do_install_append_class-target () {
 			rm ${D}${datadir}/installed-tests/glib/gdbus-serialization.test
 		fi
 	fi
+        if test "x${MLPREFIX}" != "x"; then
+                mv ${D}${datadir}/installed-tests/glib/static-link.test ${D}${datadir}/installed-tests/glib/${MLPREFIX}static-link.test
+        fi
 }
 
 # As we do not build python3 for windows, makes no sense to ship the script that's using it

meta/recipes-core/glibc/glibc-version.inc

@@ -1,6 +1,6 @@
 SRCBRANCH ?= "release/2.31/master"
-PV = "2.31"
-SRCREV_glibc ?= "9ea3686266dca3f004ba874745a4087a89682617"
+PV = "2.31+git${SRCPV}"
+SRCREV_glibc ?= "71f2b249a28e17eac0e47c53af44d5c5b65101aa"
 SRCREV_localedef ?= "cd9f958c4c94a638fa7b2b4e21627364f1a1a655"
 
 GLIBC_GIT_URI ?= "git://sourceware.org/git/glibc.git"

meta/recipes-core/glibc/glibc.inc

@@ -9,8 +9,11 @@ inherit autotools texinfo features_check systemd
 
 LEAD_SONAME = "libc.so"
 
+# msgfmt could come from gettext-native but we don't depend on that and
+# disable for reproducibility
 CACHED_CONFIGUREVARS += " \
   ac_cv_path_BASH_SHELL=${base_bindir}/bash \
+  ac_cv_prog_MSGFMT= \
   libc_cv_slibdir=${base_libdir} \
   libc_cv_rootsbindir=${base_sbindir} \
   libc_cv_localedir=${localedir} \

meta/recipes-core/glibc/glibc/0007-nativesdk-glibc-Make-relocatable-install-for-locales.patch

@@ -17,11 +17,11 @@ Signed-off-by: Khem Raj <raj.khem@gmail.com>
  locale/localeinfo.h  | 2 +-
  3 files changed, 4 insertions(+), 4 deletions(-)
 
-diff --git a/locale/findlocale.c b/locale/findlocale.c
-index 9cd3b71a6d..84272310e0 100644
---- a/locale/findlocale.c
-+++ b/locale/findlocale.c
-@@ -56,7 +56,7 @@ struct __locale_data *const _nl_C[] attribute_hidden =
+Index: git/locale/findlocale.c
+===================================================================
+--- git.orig/locale/findlocale.c
++++ git/locale/findlocale.c
+@@ -56,7 +56,7 @@ struct __locale_data *const _nl_C[] attr
     which are somehow addressed.  */
  struct loaded_l10nfile *_nl_locale_file_list[__LC_LAST];
  
@@ -30,7 +30,7 @@ index 9cd3b71a6d..84272310e0 100644
  
  /* Checks if the name is actually present, that is, not NULL and not
     empty.  */
-@@ -166,7 +166,7 @@ _nl_find_locale (const char *locale_path, size_t locale_path_len,
+@@ -166,7 +166,7 @@ _nl_find_locale (const char *locale_path
  
        /* Nothing in the archive.  Set the default path to search below.  */
        locale_path = _nl_default_locale_path;
@@ -39,10 +39,10 @@ index 9cd3b71a6d..84272310e0 100644
      }
    else
      /* We really have to load some data.  First see whether the name is
-diff --git a/locale/loadarchive.c b/locale/loadarchive.c
-index ba0fe45648..9737fd4cda 100644
---- a/locale/loadarchive.c
-+++ b/locale/loadarchive.c
+Index: git/locale/loadarchive.c
+===================================================================
+--- git.orig/locale/loadarchive.c
++++ git/locale/loadarchive.c
 @@ -42,7 +42,7 @@
  
  
@@ -52,11 +52,11 @@ index ba0fe45648..9737fd4cda 100644
  
  /* Size of initial mapping window, optimal if large enough to
     cover the header plus the initial locale.  */
-diff --git a/locale/localeinfo.h b/locale/localeinfo.h
-index 1bfe22aa7f..fdc283c69a 100644
---- a/locale/localeinfo.h
-+++ b/locale/localeinfo.h
-@@ -331,7 +331,7 @@ _nl_lookup_word (locale_t l, int category, int item)
+Index: git/locale/localeinfo.h
+===================================================================
+--- git.orig/locale/localeinfo.h
++++ git/locale/localeinfo.h
+@@ -331,7 +331,7 @@ _nl_lookup_word (locale_t l, int categor
  }
  
  /* Default search path if no LOCPATH environment variable.  */
@@ -65,3 +65,35 @@ index 1bfe22aa7f..fdc283c69a 100644
  
  /* Load the locale data for CATEGORY from the file specified by *NAME.
     If *NAME is "", use environment variables as specified by POSIX, and
+Index: git/locale/programs/locale.c
+===================================================================
+--- git.orig/locale/programs/locale.c
++++ git/locale/programs/locale.c
+@@ -632,6 +632,7 @@ nameentcmp (const void *a, const void *b
+ 		  ((const struct nameent *) b)->name);
+ }
+ 
++static char _write_archive_locales_path[4096] attribute_hidden __attribute__ ((section (".gccrelocprefix"))) = ARCHIVE_NAME;
+ 
+ static int
+ write_archive_locales (void **all_datap, char *linebuf)
+@@ -645,7 +646,7 @@ write_archive_locales (void **all_datap,
+   int fd, ret = 0;
+   uint32_t cnt;
+ 
+-  fd = open64 (ARCHIVE_NAME, O_RDONLY);
++  fd = open64 (_write_archive_locales_path, O_RDONLY);
+   if (fd < 0)
+     return 0;
+ 
+@@ -700,8 +701,8 @@ write_archive_locales (void **all_datap,
+ 	  if (cnt)
+ 	    putchar_unlocked ('\n');
+ 
+-	  printf ("locale: %-15.15s archive: " ARCHIVE_NAME "\n%s\n",
+-		  names[cnt].name, linebuf);
++	  printf ("locale: %-15.15s archive: %s\n%s\n",
++		  names[cnt].name, _write_archive_locales_path, linebuf);
+ 
+ 	  locrec = (struct locrecent *) (addr + names[cnt].locrec_offset);
+ 

meta/recipes-core/glibc/glibc/CVE-2020-10029.patch

@@ -0,0 +1,128 @@
+From ce265ec5bc25ec35fba53807abac1b0c8469895e Mon Sep 17 00:00:00 2001
+From: Joseph Myers <joseph@codesourcery.com>
+Date: Wed, 12 Feb 2020 23:31:56 +0000
+Subject: [PATCH] Avoid ldbl-96 stack corruption from range reduction of
+
+ pseudo-zero (bug 25487).
+
+Bug 25487 reports stack corruption in ldbl-96 sinl on a pseudo-zero
+argument (an representation where all the significand bits, including
+the explicit high bit, are zero, but the exponent is not zero, which
+is not a valid representation for the long double type).
+
+Although this is not a valid long double representation, existing
+practice in this area (see bug 4586, originally marked invalid but
+subsequently fixed) is that we still seek to avoid invalid memory
+accesses as a result, in case of programs that treat arbitrary binary
+data as long double representations, although the invalid
+representations of the ldbl-96 format do not need to be consistently
+handled the same as any particular valid representation.
+
+This patch makes the range reduction detect pseudo-zero and unnormal
+representations that would otherwise go to __kernel_rem_pio2, and
+returns a NaN for them instead of continuing with the range reduction
+process.  (Pseudo-zero and unnormal representations whose unbiased
+exponent is less than -1 have already been safely returned from the
+function before this point without going through the rest of range
+reduction.)  Pseudo-zero representations would previously result in
+the value passed to __kernel_rem_pio2 being all-zero, which is
+definitely unsafe; unnormal representations would previously result in
+a value passed whose high bit is zero, which might well be unsafe
+since that is not a form of input expected by __kernel_rem_pio2.
+
+Tested for x86_64.
+
+CVE: CVE-2020-10029
+Upstream-Status: Backport [https://sourceware.org/git/gitweb.cgi?p=glibc.git;
+a=patch;h=9333498794cde1d5cca518badf79533a24114b6f]
+Signed-off-by: Zhixiong Chi <zhixiong.chi@windriver.com>
+
+---
+ sysdeps/ieee754/ldbl-96/Makefile           |  3 ++-
+ sysdeps/ieee754/ldbl-96/e_rem_pio2l.c      | 12 +++++++++
+ sysdeps/ieee754/ldbl-96/test-sinl-pseudo.c | 41 ++++++++++++++++++++++++++++++
+ 3 files changed, 55 insertions(+), 1 deletion(-)
+ create mode 100644 sysdeps/ieee754/ldbl-96/test-sinl-pseudo.c
+
+diff --git a/sysdeps/ieee754/ldbl-96/Makefile b/sysdeps/ieee754/ldbl-96/Makefile
+index b103254..052c1c7 100644
+--- a/sysdeps/ieee754/ldbl-96/Makefile
++++ b/sysdeps/ieee754/ldbl-96/Makefile
+@@ -17,5 +17,6 @@
+ # <https://www.gnu.org/licenses/>.
+ 
+ ifeq ($(subdir),math)
+-tests += test-canonical-ldbl-96 test-totalorderl-ldbl-96
++tests += test-canonical-ldbl-96 test-totalorderl-ldbl-96 test-sinl-pseudo
++CFLAGS-test-sinl-pseudo.c += -fstack-protector-all
+ endif
+diff --git a/sysdeps/ieee754/ldbl-96/e_rem_pio2l.c b/sysdeps/ieee754/ldbl-96/e_rem_pio2l.c
+index 805de22..1aeccb4 100644
+--- a/sysdeps/ieee754/ldbl-96/e_rem_pio2l.c
++++ b/sysdeps/ieee754/ldbl-96/e_rem_pio2l.c
+@@ -210,6 +210,18 @@ __ieee754_rem_pio2l (long double x, long double *y)
+       return 0;
+     }
+ 
++  if ((i0 & 0x80000000) == 0)
++    {
++      /* Pseudo-zero and unnormal representations are not valid
++	 representations of long double.  We need to avoid stack
++	 corruption in __kernel_rem_pio2, which expects input in a
++	 particular normal form, but those representations do not need
++	 to be consistently handled like any particular floating-point
++	 value.  */
++      y[1] = y[0] = __builtin_nanl ("");
++      return 0;
++    }
++
+   /* Split the 64 bits of the mantissa into three 24-bit integers
+      stored in a double array.  */
+   exp = j0 - 23;
+diff --git a/sysdeps/ieee754/ldbl-96/test-sinl-pseudo.c b/sysdeps/ieee754/ldbl-96/test-sinl-pseudo.c
+new file mode 100644
+index 0000000..f59b977
+--- /dev/null
++++ b/sysdeps/ieee754/ldbl-96/test-sinl-pseudo.c
+@@ -0,0 +1,41 @@
++/* Test sinl for pseudo-zeros and unnormals for ldbl-96 (bug 25487).
++   Copyright (C) 2020 Free Software Foundation, Inc.
++   This file is part of the GNU C Library.
++
++   The GNU C Library is free software; you can redistribute it and/or
++   modify it under the terms of the GNU Lesser General Public
++   License as published by the Free Software Foundation; either
++   version 2.1 of the License, or (at your option) any later version.
++
++   The GNU C Library is distributed in the hope that it will be useful,
++   but WITHOUT ANY WARRANTY; without even the implied warranty of
++   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
++   Lesser General Public License for more details.
++
++   You should have received a copy of the GNU Lesser General Public
++   License along with the GNU C Library; if not, see
++   <https://www.gnu.org/licenses/>.  */
++
++#include <math.h>
++#include <math_ldbl.h>
++#include <stdint.h>
++
++static int
++do_test (void)
++{
++  for (int i = 0; i < 64; i++)
++    {
++      uint64_t sig = i == 63 ? 0 : 1ULL << i;
++      long double ld;
++      SET_LDOUBLE_WORDS (ld, 0x4141,
++			 sig >> 32, sig & 0xffffffffULL);
++      /* The requirement is that no stack overflow occurs when the
++	 pseudo-zero or unnormal goes through range reduction.  */
++      volatile long double ldr;
++      ldr = sinl (ld);
++      (void) ldr;
++    }
++  return 0;
++}
++
++#include <support/test-driver.c>

meta/recipes-core/glibc/glibc_2.31.bb

@@ -40,6 +40,7 @@ SRC_URI =  "${GLIBC_GIT_URI};branch=${SRCBRANCH};name=glibc \
            file://0027-intl-Emit-no-lines-in-bison-generated-files.patch \
            file://0028-inject-file-assembly-directives.patch \
            file://0029-locale-prevent-maybe-uninitialized-errors-with-Os-BZ.patch \
+           file://CVE-2020-10029.patch \
            "
 S = "${WORKDIR}/git"
 B = "${WORKDIR}/build-${TARGET_SYS}"

meta/recipes-core/images/build-appliance-image_15.0.0.bb

@@ -24,7 +24,7 @@ IMAGE_FSTYPES = "wic.vmdk"
 
 inherit core-image module-base setuptools3
 
-SRCREV ?= "39d45ef825694705daf6d7ab4ab3066b1b8e3aa2"
+SRCREV ?= "1adc7478a0548e1ebd75d96d2b938c5852c78365"
 SRC_URI = "git://git.yoctoproject.org/poky \
            file://Yocto_Build_Appliance.vmx \
            file://Yocto_Build_Appliance.vmxf \

meta/recipes-core/initrdscripts/initramfs-framework/udev

@@ -41,6 +41,9 @@ udev_run() {
 	mkdir -p /run
 	mkdir -p /var/run
 
+	# Workaround if console=null, systemd-udevd needs valid stdin, stdout and stderr to work
+	sh -c "exec 4< /dev/console" || { exec 0> /dev/null; exec 1> /dev/null; exec 2> /dev/null; }
+
 	$_UDEV_DAEMON --daemon
 	udevadm trigger --action=add
 	udevadm settle

meta/recipes-core/libxcrypt/libxcrypt.inc

@@ -32,4 +32,4 @@ CFLAGS += "-fcommon"
 API = "--disable-obsolete-api"
 EXTRA_OECONF += "${API}"
 
-BBCLASSEXTEND = "nativesdk"
+BBCLASSEXTEND = "native nativesdk"

meta/recipes-core/meta/buildtools-extended-tarball.bb

@@ -25,6 +25,8 @@ TOOLCHAIN_HOST_TASK += "\
     nativesdk-libstdc++-dev \
     nativesdk-libtool \
     nativesdk-pkgconfig \
+    nativesdk-glibc-utils \
+    nativesdk-libxcrypt-dev \
     "
 
 TOOLCHAIN_OUTPUTNAME = "${SDK_ARCH}-buildtools-extended-nativesdk-standalone-${DISTRO_VERSION}"

meta/recipes-core/musl/musl_git.bb

@@ -66,27 +66,20 @@ do_install() {
 	rm -f ${D}${bindir}/ldd ${D}${GLIBC_LDSO}
 	lnr ${D}${libdir}/libc.so ${D}${bindir}/ldd
 	lnr ${D}${libdir}/libc.so ${D}${GLIBC_LDSO}
-	for l in crypt dl m pthread resolv rt util xnet
-	do
-		ln -sf libc.so ${D}${libdir}/lib$l.so
-	done
-	for i in libc.so.6 libcrypt.so.1 libdl.so.2 libm.so.6 libpthread.so.0 libresolv.so.2 librt.so.1 libutil.so.1; do
-		ln -sf libc.so ${D}${libdir}/$i
-	done
 }
 
 PACKAGES =+ "${PN}-glibc-compat"
 
-FILES_${PN}-glibc-compat += "\
-                ${libdir}/libc.so.6 ${libdir}/libcrypt.so.1 \
-                ${libdir}/libdl.so.2 ${libdir}/libm.so.6 \
-                ${libdir}/libpthread.so.0 ${libdir}/libresolv.so.2 \
-                ${libdir}/librt.so.1 ${libdir}/libutil.so.1 \
-                ${GLIBC_LDSO} \
-                "
+FILES_${PN}-glibc-compat += "${GLIBC_LDSO}"
+FILES_${PN}-staticdev = "${libdir}/libc.a"
+FILES_${PN}-dev =+ "${libdir}/libcrypt.a ${libdir}/libdl.a ${libdir}/libm.a \
+                    ${libdir}/libpthread.a ${libdir}/libresolv.a \
+                    ${libdir}/librt.a ${libdir}/libutil.a ${libdir}/libxnet.a \
+                   "
 
 RDEPENDS_${PN}-dev += "linux-libc-headers-dev bsd-headers-dev libssp-nonshared-staticdev"
 RPROVIDES_${PN}-dev += "libc-dev virtual-libc-dev"
 RPROVIDES_${PN} += "ldd libsegfault rtld(GNU_HASH)"
 
 LEAD_SONAME = "libc.so"
+INSANE_SKIP_${PN}-dev = "staticdev"

meta/recipes-core/ncurses/ncurses_6.2.bb

@@ -10,4 +10,5 @@ S = "${WORKDIR}/git"
 EXTRA_OECONF += "--with-abi-version=5 --cache-file=${B}/config.cache"
 UPSTREAM_CHECK_GITTAGREGEX = "(?P<pver>\d+(\.\d+)+(\+\d+)*)"
 
-CVE_VERSION = "6.1.${@d.getVar("PV").split('+')[1]}"
+# This is needed when using patchlevel versions like 6.1+20181013
+#CVE_VERSION = "${@d.getVar("PV").split('+')[0]}.${@d.getVar("PV").split('+')[1]}"

meta/recipes-core/netbase/netbase_6.1.bb

@@ -8,8 +8,8 @@ PE = "1"
 
 SRC_URI = "${DEBIAN_MIRROR}/main/n/${BPN}/${BPN}_${PV}.tar.xz"
 
-SRC_URI[md5sum] = "3417b0487161f1a2b070a3308cd7f957"
-SRC_URI[sha256sum] = "692baeb7b76eba5580c7edbc97ce1784a06b5aa4b367c5ed0b39e0ce7a97d594"
+SRC_URI[md5sum] = "e5871a3a5c8390557b8033cf19316a55"
+SRC_URI[sha256sum] = "084d743bd84d4d9380bac4c71c51e57406dce44f5a69289bb823c903e9b035d8"
 
 UPSTREAM_CHECK_URI = "${DEBIAN_MIRROR}/main/n/netbase/"
 do_install () {

meta/recipes-core/packagegroups/packagegroup-core-tools-profile.bb

@@ -46,6 +46,7 @@ LTTNGMODULES = "lttng-modules"
 LTTNGMODULES_arc = ""
 
 BABELTRACE = "babeltrace"
+BABELTRACE2 = "babeltrace2"
 
 # valgrind does not work on the following configurations/architectures
 
@@ -69,6 +70,7 @@ RDEPENDS_${PN} = "\
     ${LTTNGTOOLS} \
     ${LTTNGMODULES} \
     ${BABELTRACE} \
+    ${BABELTRACE2} \
     ${SYSTEMTAP} \
     ${VALGRIND} \
     "

meta/recipes-core/psplash/files/psplash-start.service

@@ -4,6 +4,7 @@ DefaultDependencies=no
 RequiresMountsFor=/run
 
 [Service]
+Type=notify
 ExecStart=/usr/bin/psplash
 
 [Install]

meta/recipes-core/psplash/files/psplash-systemd.service

@@ -2,6 +2,7 @@
 Description=Start psplash-systemd progress communication helper
 DefaultDependencies=no
 After=systemd-start.service
+After=psplash-start.service
 Requires=psplash-start.service
 RequiresMountsFor=/run
 

meta/recipes-core/psplash/psplash_git.bb

@@ -4,8 +4,9 @@ HOMEPAGE = "http://git.yoctoproject.org/cgit/cgit.cgi/psplash"
 SECTION = "base"
 LICENSE = "GPLv2+"
 LIC_FILES_CHKSUM = "file://psplash.h;beginline=1;endline=8;md5=8f232c1e95929eacab37f00900580224"
+DEPENDS = "gdk-pixbuf-native"
 
-SRCREV = "773a3977d255e8f59a741ad6ce37c4d40f1feaa1"
+SRCREV = "0a902f7cd875ccf018456451be369f05fa55f962"
 PV = "0.1+git${SRCPV}"
 PR = "r15"
 
@@ -23,7 +24,6 @@ python __anonymous() {
     splashfiles = d.getVar('SPLASH_IMAGES').split()
     pkgs = []
     localpaths = []
-    haspng = False
     for uri in splashfiles:
         fetcher = bb.fetch2.Fetch([uri], d)
         flocal = os.path.basename(fetcher.localpath(uri))
@@ -41,17 +41,12 @@ python __anonymous() {
             bb.fatal("The output name '%s' derived from the URI %s is not valid, please specify the outsuffix parameter" % (outname, uri))
         else:
             pkgs.append(outname)
-        if flocal.endswith(".png"):
-            haspng = True
         localpaths.append(flocal)
 
     # Set these so that we have less work to do in do_compile and do_install_append
     d.setVar("SPLASH_INSTALL", " ".join(pkgs))
     d.setVar("SPLASH_LOCALPATHS", " ".join(localpaths))
 
-    if haspng:
-        d.appendVar("DEPENDS", " gdk-pixbuf-native")
-
     d.prependVar("PACKAGES", "%s " % (" ".join(pkgs)))
     mlprefix = d.getVar('MLPREFIX') or ''
     pn = d.getVar('PN') or ''
@@ -84,7 +79,7 @@ python do_compile () {
     # Build a separate executable for each splash image
     workdir = d.getVar('WORKDIR')
     convertscript = "%s/make-image-header.sh" % d.getVar('S')
-    destfile = "%s/psplash-poky-img.h" % d.getVar('S')
+    destfile = "%s/psplash-poky-img.h" % d.getVar('B')
     localfiles = d.getVar('SPLASH_LOCALPATHS').split()
     outputfiles = d.getVar('SPLASH_INSTALL').split()
     for localfile, outputfile in zip(localfiles, outputfiles):

meta/recipes-core/systemd/systemd_244.3.bb

@@ -102,7 +102,7 @@ PACKAGECONFIG_remove_libc-musl = " \
     utmp \
 "
 
-CFLAGS_append_libc-musl = "-D__UAPI_DEF_ETHHDR=0 "
+CFLAGS_append_libc-musl = " -D__UAPI_DEF_ETHHDR=0 "
 
 # Use the upstream systemd serial-getty@.service and rely on
 # systemd-getty-generator instead of using the OE-core specific

meta/recipes-core/util-linux/util-linux.inc

@@ -8,7 +8,7 @@ SECTION = "base"
 
 LICENSE = "GPLv2+ & LGPLv2.1+ & BSD-3-Clause & BSD-4-Clause"
 
-LIC_FILES_CHKSUM = "file://README.licensing;md5=972a134f1e14b2b060e365df2fab0099 \
+LIC_FILES_CHKSUM = "file://README.licensing;md5=0fd5c050c6187d2bf0a4492b7f4e33da \
                     file://COPYING;md5=b234ee4d69f5fce4486a80fdaf4a4263 \
                     file://Documentation/licenses/COPYING.GPL-2.0-or-later;md5=b234ee4d69f5fce4486a80fdaf4a4263 \
                     file://Documentation/licenses/COPYING.LGPL-2.1-or-later;md5=4fbd65380cdd255951079008b364516c \
@@ -105,6 +105,7 @@ EXTRA_OECONF = "\
 EXTRA_OECONF_append_class-target = " --enable-setpriv"
 EXTRA_OECONF_append_class-native = " --without-cap-ng --disable-setpriv"
 EXTRA_OECONF_append_class-nativesdk = " --without-cap-ng --disable-setpriv"
+EXTRA_OECONF_append = " --disable-hwclock-gplv3"
 
 # enable pcre2 for native/nativesdk to match host distros
 # this helps to keep same expectations when using the SDK or

meta/recipes-core/util-linux/util-linux/0001-libfdisk-script-accept-sector-size-ignore-unknown-he.patch

@@ -0,0 +1,137 @@
+From 00e53f17c8462cb34ece08cc10db60a7da29a305 Mon Sep 17 00:00:00 2001
+From: Karel Zak <kzak@redhat.com>
+Date: Tue, 4 Feb 2020 15:11:19 +0100
+Subject: [PATCH] libfdisk: (script) accept sector-size, ignore unknown headers
+
+- add sector-size between supported headers (already in --dump output)
+
+- report unknown headers by -ENOTSUP
+
+- ignore ENOTSUP in sfdisk (but print warning) and in fdisk_script_read_file()
+
+Upstream-Status: Backport [https://github.com/karelzak/util-linux/commit/00e53f17c8462cb34ece08cc10db60a7da29a305]
+
+Addresses: https://github.com/karelzak/util-linux/issues/949
+Signed-off-by: Karel Zak <kzak@redhat.com>
+Signed-off-by: Pierre-Jean Texier <pjtexier@koncepto.io>
+---
+ disk-utils/sfdisk.c   |  6 +++++-
+ libfdisk/src/script.c | 49 ++++++++++++++++++++++++++-----------------------
+ 2 files changed, 31 insertions(+), 24 deletions(-)
+
+diff --git a/disk-utils/sfdisk.c b/disk-utils/sfdisk.c
+index bb6e1c6..c0bea70 100644
+--- a/disk-utils/sfdisk.c
++++ b/disk-utils/sfdisk.c
+@@ -1782,7 +1782,11 @@ static int command_fdisk(struct sfdisk *sf, int argc, char **argv)
+ 		}
+ 
+ 		rc = fdisk_script_read_line(dp, stdin, buf, sizeof(buf));
+-		if (rc < 0) {
++		if (rc == -ENOTSUP) {
++			buf[sizeof(buf) - 1] = '\0';
++			fdisk_warnx(sf->cxt, _("Unknown script header '%s' -- ignore."), buf);
++			continue;
++		} else if (rc < 0) {
+ 			DBG(PARSE, ul_debug("script parsing failed, trying sfdisk specific commands"));
+ 			buf[sizeof(buf) - 1] = '\0';
+ 			rc = loop_control_commands(sf, dp, buf);
+diff --git a/libfdisk/src/script.c b/libfdisk/src/script.c
+index a21771b..d3e67fa 100644
+--- a/libfdisk/src/script.c
++++ b/libfdisk/src/script.c
+@@ -805,8 +805,12 @@ static inline int is_header_line(const char *s)
+ /* parses "<name>: value", note modifies @s*/
+ static int parse_line_header(struct fdisk_script *dp, char *s)
+ {
+-	int rc = -EINVAL;
++	size_t i;
+ 	char *name, *value;
++	static const char *supported[] = {
++		"label", "unit", "label-id", "device", "grain",
++		"first-lba", "last-lba", "table-length", "sector-size"
++	};
+ 
+ 	DBG(SCRIPT, ul_debugobj(dp, "   parse header '%s'", s));
+ 
+@@ -816,7 +820,7 @@ static int parse_line_header(struct fdisk_script *dp, char *s)
+ 	name = s;
+ 	value = strchr(s, ':');
+ 	if (!value)
+-		goto done;
++		return -EINVAL;
+ 	*value = '\0';
+ 	value++;
+ 
+@@ -825,32 +829,30 @@ static int parse_line_header(struct fdisk_script *dp, char *s)
+ 	ltrim_whitespace((unsigned char *) value);
+ 	rtrim_whitespace((unsigned char *) value);
+ 
++	if (!*name || !*value)
++		return -EINVAL;
++
++	/* check header name */
++	for (i = 0; i < ARRAY_SIZE(supported); i++) {
++		if (strcmp(name, supported[i]) == 0)
++			break;
++	}
++	if (i == ARRAY_SIZE(supported))
++		return -ENOTSUP;
++
++	/* header specific actions */
+ 	if (strcmp(name, "label") == 0) {
+ 		if (dp->cxt && !fdisk_get_label(dp->cxt, value))
+-			goto done;			/* unknown label name */
++			return -EINVAL;			/* unknown label name */
+ 		dp->force_label = 1;
++
+ 	} else if (strcmp(name, "unit") == 0) {
+ 		if (strcmp(value, "sectors") != 0)
+-			goto done;			/* only "sectors" supported */
+-	} else if (strcmp(name, "label-id") == 0
+-		   || strcmp(name, "device") == 0
+-		   || strcmp(name, "grain") == 0
+-		   || strcmp(name, "first-lba") == 0
+-		   || strcmp(name, "last-lba") == 0
+-		   || strcmp(name, "table-length") == 0) {
+-		;					/* whatever is possible */
+-	} else
+-		goto done;				/* unknown header */
++			return -EINVAL;			/* only "sectors" supported */
+ 
+-	if (*name && *value)
+-		rc = fdisk_script_set_header(dp, name, value);
+-done:
+-	if (rc)
+-		DBG(SCRIPT, ul_debugobj(dp, "header parse error: "
+-				"[rc=%d, name='%s', value='%s']",
+-				rc, name, value));
+-	return rc;
++	}
+ 
++	return fdisk_script_set_header(dp, name, value);
+ }
+ 
+ /* returns zero terminated string with next token and @str is updated */
+@@ -1363,7 +1365,8 @@ int fdisk_script_set_fgets(struct fdisk_script *dp,
+  *
+  * Reads next line into dump.
+  *
+- * Returns: 0 on success, <0 on error, 1 when nothing to read.
++ * Returns: 0 on success, <0 on error, 1 when nothing to read. For unknown headers
++ *          returns -ENOTSUP, it's usually safe to ignore this error.
+  */
+ int fdisk_script_read_line(struct fdisk_script *dp, FILE *f, char *buf, size_t bufsz)
+ {
+@@ -1428,7 +1431,7 @@ int fdisk_script_read_file(struct fdisk_script *dp, FILE *f)
+ 
+ 	while (!feof(f)) {
+ 		rc = fdisk_script_read_line(dp, f, buf, sizeof(buf));
+-		if (rc)
++		if (rc && rc != -ENOTSUP)
+ 			break;
+ 	}
+ 
+-- 
+2.7.4
+

meta/recipes-core/util-linux/util-linux/0001-lsblk-force-to-print-PKNAME-for-partition.patch

@@ -1,36 +0,0 @@
-From e3bb9bfb76c17b1d05814436ced62c05c4011f48 Mon Sep 17 00:00:00 2001
-From: Karel Zak <kzak@redhat.com>
-Date: Thu, 27 Jun 2019 09:22:18 +0200
-Subject: [PATCH] lsblk: force to print PKNAME for partition
-
-PKNAME (parent kernel device name) is based on printed tree according
-to parent -> child relationship. The tree is optional and not printed
-if partition specified (.e.g "lsblk -o+PKNAME /dev/sda1"), but old
-versions print the PKNAME also in this case.
-
-Upstream-Status: Backport [https://github.com/karelzak/util-linux/commit/e3bb9bfb76c17b1d05814436ced62c05c4011f48]
-
-Addresses: https://github.com/karelzak/util-linux/issues/813
-Signed-off-by: Karel Zak <kzak@redhat.com>
-Signed-off-by: Liwei Song <liwei.song@windriver.com>
----
- misc-utils/lsblk.c | 3 +++
- 1 file changed, 3 insertions(+)
-
-diff --git a/misc-utils/lsblk.c b/misc-utils/lsblk.c
-index e95af7af0256..3ce6da730264 100644
---- a/misc-utils/lsblk.c
-+++ b/misc-utils/lsblk.c
-@@ -1019,6 +1019,9 @@ static void device_to_scols(
- 	DBG(DEV, ul_debugobj(dev, "add '%s' to scols", dev->name));
- 	ON_DBG(DEV, if (ul_path_isopen_dirfd(dev->sysfs)) ul_debugobj(dev, " %s ---> is open!", dev->name));
- 
-+	if (!parent && dev->wholedisk)
-+		parent = dev->wholedisk;
-+
- 	/* Do not print device more than one in --list mode */
- 	if (!(lsblk->flags & LSBLK_TREE) && dev->is_printed)
- 		return;
--- 
-2.17.1
-

meta/recipes-core/util-linux/util-linux_2.35.1.bb

@@ -7,8 +7,8 @@ SRC_URI += "file://configure-sbindir.patch \
             file://run-ptest \
             file://display_testname_for_subtest.patch \
             file://avoid_parallel_tests.patch \
-            file://0001-lsblk-force-to-print-PKNAME-for-partition.patch \
             file://0001-hwclock-fix-for-glibc-2.31-settimeofday.patch \
+            file://0001-libfdisk-script-accept-sector-size-ignore-unknown-he.patch \
 "
-SRC_URI[md5sum] = "a78cbeaed9c39094b96a48ba8f891d50"
-SRC_URI[sha256sum] = "743f9d0c7252b6db246b659c1e1ce0bd45d8d4508b4dfa427bbb4a3e9b9f62b5"
+SRC_URI[md5sum] = "7f64882f631225f0295ca05080cee1bf"
+SRC_URI[sha256sum] = "d9de3edd287366cd908e77677514b9387b22bc7b88f45b83e1922c3597f1d7f9"