bitbake: build: make shell traps less chatty when 'bitbake -v' is used

This shuts up both the DEBUG and EXIT handlers.

Also, remove an unneeded "set -o errtrace" (i.e. set -E),
since we no longer use an ERR trap.

(Bitbake rev: 89e851fa0403d1e98aeed69990101e3f84f0b283)

Signed-off-by: Chris Laplante <chris.laplante@agilent.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

bitbake/bin/bitbake-selftest

@@ -18,6 +18,7 @@ except RuntimeError as exc:
     sys.exit(str(exc))
 
 tests = ["bb.tests.codeparser",
+         "bb.tests.color",
          "bb.tests.cooker",
          "bb.tests.cow",
          "bb.tests.data",

bitbake/contrib/vim/LICENSE.txt

@@ -0,0 +1,18 @@
+The MIT License (MIT)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

bitbake/contrib/vim/ftdetect/bitbake.vim

@@ -6,12 +6,12 @@
 "
 " This sets up the syntax highlighting for BitBake files, like .bb, .bbclass and .inc
 
-if &compatible || version < 600
+if &compatible || version < 600 || exists("b:loaded_bitbake_plugin")
     finish
 endif
 
 " .bb, .bbappend and .bbclass
-au BufNewFile,BufRead *.{bb,bbappend,bbclass}	set filetype=bitbake
+au BufNewFile,BufRead *.{bb,bbappend,bbclass}  set filetype=bitbake
 
 " .inc
 au BufNewFile,BufRead *.inc		set filetype=bitbake

bitbake/contrib/vim/ftplugin/bitbake.vim

@@ -1,2 +1,13 @@
-set sts=4 sw=4 et
-set cms=#%s
+" Only do this when not done yet for this buffer
+if exists("b:did_ftplugin")
+  finish
+endif
+
+" Don't load another plugin for this buffer
+let b:did_ftplugin = 1
+
+let b:undo_ftplugin = "setl cms< sts< sw< et< sua<"
+
+setlocal commentstring=#\ %s
+setlocal softtabstop=4 shiftwidth=4 expandtab
+setlocal suffixesadd+=.bb,.bbclass

bitbake/contrib/vim/plugin/newbb.vim

@@ -10,7 +10,7 @@
 "
 " Will try to use git to find the user name and email
 
-if &compatible || v:version < 600
+if &compatible || v:version < 600 || exists("b:loaded_bitbake_plugin")
     finish
 endif
 
@@ -25,7 +25,7 @@ endfun
 fun! <SID>GetUserEmail()
     let l:user_email = system("git config --get user.email")
     if v:shell_error
-        return "unknow@user.org"
+        return "unknown@user.org"
     else
         return substitute(l:user_email, "\n", "", "")
 endfun
@@ -41,6 +41,10 @@ fun! BBHeader()
 endfun
 
 fun! NewBBTemplate()
+    if line2byte(line('$') + 1) != -1
+        return
+    endif
+
     let l:paste = &paste
     set nopaste
     
@@ -48,7 +52,7 @@ fun! NewBBTemplate()
     call BBHeader()
 
     " New the bb template
-    put ='DESCRIPTION = \"\"'
+    put ='SUMMARY = \"\"'
     put ='HOMEPAGE = \"\"'
     put ='LICENSE = \"\"' 
     put ='SECTION = \"\"'
@@ -58,7 +62,7 @@ fun! NewBBTemplate()
 
     " Go to the first place to edit
     0
-    /^DESCRIPTION =/
+    /^SUMMARY =/
     exec "normal 2f\""
 
     if paste == 1
@@ -76,7 +80,7 @@ if v:progname =~ "vimdiff"
 endif
 
 augroup NewBB
-    au BufNewFile *.bb
+    au BufNewFile,BufReadPost *.bb
                 \ if g:bb_create_on_empty |
                 \    call NewBBTemplate() |
                 \ endif

bitbake/contrib/vim/plugin/newbbappend.vim

@@ -0,0 +1,46 @@
+" Vim plugin file
+" Purpose:	Create a template for new bbappend file
+" Author:	Joshua Watt <JPEWhacker@gmail.com>
+" Copyright:	Copyright (C) 2017 Joshua Watt <JPEWhacker@gmail.com>
+"
+" This file is licensed under the MIT license, see COPYING.MIT in
+" this source distribution for the terms.
+"
+
+if &compatible || v:version < 600 || exists("b:loaded_bitbake_plugin")
+    finish
+endif
+
+fun! NewBBAppendTemplate()
+    if line2byte(line('$') + 1) != -1
+        return
+    endif
+
+    let l:paste = &paste
+    set nopaste
+
+    " New bbappend template
+    0 put ='FILESEXTRAPATHS_prepend := \"${THISDIR}/${PN}:\"'
+    2
+
+    if paste == 1
+        set paste
+    endif
+endfun
+
+if !exists("g:bb_create_on_empty")
+    let g:bb_create_on_empty = 1
+endif
+
+" disable in case of vimdiff
+if v:progname =~ "vimdiff"
+    let g:bb_create_on_empty = 0
+endif
+
+augroup NewBBAppend
+    au BufNewFile,BufReadPost *.bbappend
+                \ if g:bb_create_on_empty |
+                \    call NewBBAppendTemplate() |
+                \ endif
+augroup END
+

bitbake/contrib/vim/syntax/bitbake.vim

@@ -12,7 +12,7 @@
 "
 " It's an entirely new type, just has specific syntax in shell and python code
 
-if &compatible || v:version < 600
+if &compatible || v:version < 600 || exists("b:loaded_bitbake_plugin")
     finish
 endif
 if exists("b:current_syntax")
@@ -58,8 +58,8 @@ syn match bbVarValue            ".*$" contained contains=bbString,bbVarDeref,bbV
 syn region bbVarPyValue         start=+${@+ skip=+\\$+ end=+}+ contained contains=@python
 
 " Vars metadata flags
-syn match bbVarFlagDef          "^\([a-zA-Z0-9\-_\.]\+\)\(\[[a-zA-Z0-9\-_\.]\+\]\)\@=" contains=bbIdentifier nextgroup=bbVarFlagFlag
-syn region bbVarFlagFlag        matchgroup=bbArrayBrackets start="\[" end="\]\s*\(=\|+=\|=+\|?=\)\@=" contained contains=bbIdentifier nextgroup=bbVarEq
+syn match bbVarFlagDef          "^\([a-zA-Z0-9\-_\.]\+\)\(\[[a-zA-Z0-9\-_\.+]\+\]\)\@=" contains=bbIdentifier nextgroup=bbVarFlagFlag
+syn region bbVarFlagFlag        matchgroup=bbArrayBrackets start="\[" end="\]\s*\(:=\|=\|.=\|=.|+=\|=+\|?=\)\@=" contained contains=bbIdentifier nextgroup=bbVarEq
 
 " Includes and requires
 syn keyword bbInclude           inherit include require contained 
@@ -67,15 +67,15 @@ syn match bbIncludeRest         ".*$" contained contains=bbString,bbVarDeref
 syn match bbIncludeLine         "^\(inherit\|include\|require\)\s\+" contains=bbInclude nextgroup=bbIncludeRest
 
 " Add taks and similar
-syn keyword bbStatement         addtask addhandler after before EXPORT_FUNCTIONS contained
+syn keyword bbStatement         addtask deltask addhandler after before EXPORT_FUNCTIONS contained
 syn match bbStatementRest       ".*$" skipwhite contained contains=bbStatement
-syn match bbStatementLine       "^\(addtask\|addhandler\|after\|before\|EXPORT_FUNCTIONS\)\s\+" contains=bbStatement nextgroup=bbStatementRest
+syn match bbStatementLine       "^\(addtask\|deltask\|addhandler\|after\|before\|EXPORT_FUNCTIONS\)\s\+" contains=bbStatement nextgroup=bbStatementRest
 
 " OE Important Functions
 syn keyword bbOEFunctions       do_fetch do_unpack do_patch do_configure do_compile do_stage do_install do_package contained
 
 " Generic Functions
-syn match bbFunction            "\h[0-9A-Za-z_-]*" display contained contains=bbOEFunctions
+syn match bbFunction            "\h[0-9A-Za-z_\-\.]*" display contained contains=bbOEFunctions
 
 " BitBake shell metadata
 syn include @shell syntax/sh.vim
@@ -83,7 +83,7 @@ if exists("b:current_syntax")
   unlet b:current_syntax
 endif
 syn keyword bbShFakeRootFlag    fakeroot contained
-syn match bbShFuncDef           "^\(fakeroot\s*\)\?\([0-9A-Za-z_${}-]\+\)\(python\)\@<!\(\s*()\s*\)\({\)\@=" contains=bbShFakeRootFlag,bbFunction,bbVarDeref,bbDelimiter nextgroup=bbShFuncRegion skipwhite
+syn match bbShFuncDef           "^\(fakeroot\s*\)\?\([\.0-9A-Za-z_${}\-\.]\+\)\(python\)\@<!\(\s*()\s*\)\({\)\@=" contains=bbShFakeRootFlag,bbFunction,bbVarDeref,bbDelimiter nextgroup=bbShFuncRegion skipwhite
 syn region bbShFuncRegion       matchgroup=bbDelimiter start="{\s*$" end="^}\s*$" contained contains=@shell
 
 " Python value inside shell functions
@@ -91,7 +91,7 @@ syn region shDeref         start=+${@+ skip=+\\$+ excludenl end=+}+ contained co
 
 " BitBake python metadata
 syn keyword bbPyFlag            python contained
-syn match bbPyFuncDef           "^\(python\s\+\)\([0-9A-Za-z_${}-]\+\)\?\(\s*()\s*\)\({\)\@=" contains=bbPyFlag,bbFunction,bbVarDeref,bbDelimiter nextgroup=bbPyFuncRegion skipwhite
+syn match bbPyFuncDef           "^\(fakeroot\s*\)\?\(python\)\(\s\+[0-9A-Za-z_${}\-\.]\+\)\?\(\s*()\s*\)\({\)\@=" contains=bbShFakeRootFlag,bbPyFlag,bbFunction,bbVarDeref,bbDelimiter nextgroup=bbPyFuncRegion skipwhite
 syn region bbPyFuncRegion       matchgroup=bbDelimiter start="{\s*$" end="^}\s*$" contained contains=@python
 
 " BitBake 'def'd python functions

bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml

@@ -736,6 +736,10 @@
                 The fetcher's temporary working directory is set by
                 <link linkend='var-bb-P4DIR'><filename>P4DIR</filename></link>,
                 which defaults to "DL_DIR/p4".
+                The fetcher does not make use of a perforce client, instead it
+                relies on <filename>p4 files</filename> to retrieve a list of
+                files and <filename>p4 print</filename> to transfer the content
+                of those files locally.
             </para>
 
             <para>
@@ -783,6 +787,62 @@
                     to <filename>"${WORKDIR}/p4"</filename> in your recipe.
                 </note>
             </para>
+
+            <para>
+                By default, the fetcher strips the depot location from the
+                local file paths. In the above example, the content of
+                <filename>example-depot/main/source/</filename>
+                will be placed in <filename>${WORKDIR}/p4</filename>.
+                For situations where preserving parts of the remote depot paths
+                locally is desirable, the fetcher supports two parameters:
+
+                <itemizedlist>
+                    <listitem><para>
+                        <emphasis>"module":</emphasis>
+                        The top-level depot location or directory to fetch. The
+                        value of this parameter can also point to a single file
+                        within the depot, in which case the local file path will
+                        include the module path.
+                    </para></listitem>
+                    <listitem><para>
+                        <emphasis>"remotepath":</emphasis>
+                        When used with the value "<filename>keep</filename>",
+                        the fetcher will mirror the full depot paths locally
+                        for the specified location, even in combination with
+                        the <filename>module</filename> parameter.
+                    </para></listitem>
+                </itemizedlist>
+            </para>
+
+            <para>
+                Here is an example use of the the <filename>module</filename>
+                parameter:
+
+                <literallayout class='monospaced'>
+    SRC_URI = "p4://user:passwd@example-depot/main;module=source/..."
+                </literallayout>
+
+                In this case, the content of the top-level directory
+                <filename>source/</filename>  will be fetched to
+                <filename>${P4DIR}</filename>, including the directory itself.
+                The top-level directory will be accesible at
+                <filename>${P4DIR}/source/</filename>.
+            </para>
+
+            <para>
+                Here is an example use of the the <filename>remotepath</filename>
+                parameter:
+
+                <literallayout class='monospaced'>
+    SRC_URI = "p4://user:passwd@example-depot/main;module=source/...;remotepath=keep"
+                </literallayout>
+
+                In this case, the content of the top-level directory
+                <filename>source/</filename>  will be fetched to
+                <filename>${P4DIR}</filename>, but the complete depot paths will
+                be mirrored locally. The top-level directory will be accessible
+                at <filename>${P4DIR}/example-depot/main/source/</filename>.
+            </para>
         </section>
 
         <section id='repo-fetcher'>

bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.xml

@@ -1124,6 +1124,67 @@
             </glossdef>
         </glossentry>
 
+        <glossentry id='var-BBFILES_DYNAMIC'><glossterm>BBFILES_DYNAMIC</glossterm>
+            <info>
+                BBFILES_DYNAMIC[doc] = "Activates content depending on presence of identified layers."
+            </info>
+            <glossdef>
+                <para role="glossdeffirst">
+                    Activates content depending on presence of identified layers.
+                    You identify the layers by the collections that the layers
+                    define.
+                </para>
+
+                <para>
+                    Use the <filename>BBFILES_DYNAMIC</filename> variable to
+                    avoid <filename>.bbappend</filename> files whose
+                    corresponding <filename>.bb</filename> file is in a layer
+                    that attempts to modify other layers through
+                    <filename>.bbappend</filename> but does not want to
+                    introduce a hard dependency on those other layers.
+                </para>
+
+                <para>
+                    Additionally you can prefix the rule with "!" to add 
+                    <filename>.bbappend</filename> and <filename>.bb</filename> files
+                    in case a layer is not present.
+                    Use this avoid hard dependency on those other layers.
+                </para>
+
+                <para>
+                    Use the following form for
+                    <filename>BBFILES_DYNAMIC</filename>:
+                    <literallayout class='monospaced'>
+                        <replaceable>collection_name</replaceable>:<replaceable>filename_pattern</replaceable>
+                    </literallayout>
+                    The following example identifies two collection names and
+                    two filename patterns:
+                    <literallayout class='monospaced'>
+                        BBFILES_DYNAMIC += "\
+                            clang-layer:${LAYERDIR}/bbappends/meta-clang/*/*/*.bbappend \
+                            core:${LAYERDIR}/bbappends/openembedded-core/meta/*/*/*.bbappend \
+                        "
+                    </literallayout>
+                    When the collection name is prefixed with "!" it will add the file pattern in case
+                    the layer is absent:
+                    <literallayout class='monospaced'>
+                        BBFILES_DYNAMIC += "\
+                            !clang-layer:${LAYERDIR}/backfill/meta-clang/*/*/*.bb \
+                        "
+                    </literallayout>
+                    
+                    This next example shows an error message that occurs
+                    because invalid entries are found, which cause parsing to
+                    abort:
+                    <literallayout class='monospaced'>
+                    ERROR: BBFILES_DYNAMIC entries must be of the form {!}&lt;collection name&gt;:&lt;filename pattern&gt;, not:
+                        /work/my-layer/bbappends/meta-security-isafw/*/*/*.bbappend
+                        /work/my-layer/bbappends/openembedded-core/meta/*/*/*.bbappend
+                    </literallayout>
+                </para>
+            </glossdef>
+        </glossentry>
+
         <glossentry id='var-bb-BBINCLUDED'><glossterm>BBINCLUDED</glossterm>
             <glossdef>
                 <para>

bitbake/lib/bb/build.py

@@ -16,7 +16,9 @@ import os
 import sys
 import logging
 import glob
+import itertools
 import time
+import re
 import stat
 import bb
 import bb.msg
@@ -303,20 +305,60 @@ def exec_func_python(func, d, runfile, cwd=None):
 
 def shell_trap_code():
     return '''#!/bin/sh\n
+__BITBAKE_LAST_LINE=0
+
 # Emit a useful diagnostic if something fails:
-bb_exit_handler() {
+bb_sh_exit_handler() {
     ret=$?
-    case $ret in
-    0)  ;;
-    *)  case $BASH_VERSION in
-        "") echo "WARNING: exit code $ret from a shell command.";;
-        *)  echo "WARNING: ${BASH_SOURCE[0]}:${BASH_LINENO[0]} exit $ret from '$BASH_COMMAND'";;
-        esac
-        exit $ret
-    esac
+    if [ "$ret" != 0 ]; then
+        echo "WARNING: exit code $ret from a shell command."
+    fi
+    exit $ret
 }
-trap 'bb_exit_handler' 0
-set -e
+
+bb_bash_exit_handler() {
+    ret=$?
+    { set +x; } > /dev/null
+    trap "" DEBUG
+    if [ "$ret" != 0 ]; then
+        echo "WARNING: ${BASH_SOURCE[0]}:${__BITBAKE_LAST_LINE} exit $ret from '$1'"
+
+        echo "WARNING: Backtrace (BB generated script): "
+        for i in $(seq 1 $((${#FUNCNAME[@]} - 1))); do
+            if [ "$i" -eq 1 ]; then
+                echo -e "\t#$((i)): ${FUNCNAME[$i]}, ${BASH_SOURCE[$((i-1))]}, line ${__BITBAKE_LAST_LINE}"
+            else
+                echo -e "\t#$((i)): ${FUNCNAME[$i]}, ${BASH_SOURCE[$((i-1))]}, line ${BASH_LINENO[$((i-1))]}"
+            fi
+        done
+    fi
+    exit $ret
+}
+
+bb_bash_debug_handler() {
+    local line=${BASH_LINENO[0]}
+    # For some reason the DEBUG trap trips with lineno=1 when scripts exit; ignore it
+    if [ "$line" -eq 1 ]; then
+        return
+    fi
+
+    # Track the line number of commands as they execute. This is so we can have access to the failing line number
+    # in the EXIT trap. See http://gnu-bash.2382.n7.nabble.com/trap-echo-quot-trap-exit-on-LINENO-quot-EXIT-gt-wrong-linenumber-td3666.html
+    if [ "${FUNCNAME[1]}" != "bb_bash_exit_handler" ]; then
+        __BITBAKE_LAST_LINE=$line
+    fi
+}
+
+case $BASH_VERSION in
+"") trap 'bb_sh_exit_handler' 0
+    set -e
+    ;;
+*)  trap 'bb_bash_exit_handler "$BASH_COMMAND"' 0
+    trap '{ bb_bash_debug_handler; } 2>/dev/null' DEBUG
+    set -e
+    shopt -s extdebug
+    ;;
+esac
 '''
 
 def create_progress_handler(func, progress, logfile, d):
@@ -346,7 +388,7 @@ def create_progress_handler(func, progress, logfile, d):
             cls_obj = functools.reduce(resolve, cls.split("."), bb.utils._context)
             if not cls_obj:
                 # Fall-back on __builtins__
-                cls_obj = functools.reduce(lambda x, y: x.get(y), cls.split("."), __builtins__)
+                cls_obj = functools.reduce(resolve, cls.split("."), __builtins__)
             if cls_obj:
                 return cls_obj(d, outfile=logfile, otherargs=otherargs)
             bb.warn('%s: unknown custom progress handler in task progress varflag value "%s", ignoring' % (func, cls))
@@ -398,7 +440,13 @@ exit $ret
 
     progress = d.getVarFlag(func, 'progress')
     if progress:
-        logfile = create_progress_handler(func, progress, logfile, d)
+        try:
+            logfile = create_progress_handler(func, progress, logfile, d)
+        except:
+            from traceback import format_exc
+            logger.error("Failed to create progress handler")
+            logger.error(format_exc())
+            raise
 
     fifobuffer = bytearray()
     def readfifo(data):
@@ -450,6 +498,62 @@ exit $ret
             bb.debug(2, "Executing shell function %s" % func)
             with open(os.devnull, 'r+') as stdin, logfile:
                 bb.process.run(cmd, shell=False, stdin=stdin, log=logfile, extrafiles=[(fifo,readfifo)])
+        except bb.process.ExecutionError as exe:
+            # Find the backtrace that the shell trap generated
+            backtrace_marker_regex = re.compile(r"WARNING: Backtrace \(BB generated script\)")
+            stdout_lines = (exe.stdout or "").split("\n")
+            backtrace_start_line = None
+            for i, line in enumerate(reversed(stdout_lines)):
+                if backtrace_marker_regex.search(line):
+                    backtrace_start_line = len(stdout_lines) - i
+                    break
+
+            # Read the backtrace frames, starting at the location we just found
+            backtrace_entry_regex = re.compile(r"#(?P<frameno>\d+): (?P<funcname>[^\s]+), (?P<file>.+?), line ("
+                                               r"?P<lineno>\d+)")
+            backtrace_frames = []
+            if backtrace_start_line:
+                for line in itertools.islice(stdout_lines, backtrace_start_line, None):
+                    match = backtrace_entry_regex.search(line)
+                    if match:
+                        backtrace_frames.append(match.groupdict())
+
+            with open(runfile, "r") as script:
+                script_lines = [line.rstrip() for line in script.readlines()]
+
+            # For each backtrace frame, search backwards in the script (from the line number called out by the frame),
+            # to find the comment that emit_vars injected when it wrote the script. This will give us the metadata
+            # filename (e.g. .bb or .bbclass) and line number where the shell function was originally defined.
+            script_metadata_comment_regex = re.compile(r"# line: (?P<lineno>\d+), file: (?P<file>.+)")
+            better_frames = []
+            # Skip the very last frame since it's just the call to the shell task in the body of the script
+            for frame in backtrace_frames[:-1]:
+                # Check whether the frame corresponds to a function defined in the script vs external script.
+                if os.path.samefile(frame["file"], runfile):
+                    # Search backwards from the frame lineno to locate the comment that BB injected
+                    i = int(frame["lineno"]) - 1
+                    while i >= 0:
+                        match = script_metadata_comment_regex.match(script_lines[i])
+                        if match:
+                            # Calculate the relative line in the function itself
+                            relative_line_in_function = int(frame["lineno"]) - i - 2
+                            # Calculate line in the function as declared in the metadata
+                            metadata_function_line = relative_line_in_function + int(match["lineno"])
+                            better_frames.append("#{frameno}: {funcname}, {file}, line {lineno}".format(
+                                frameno=frame["frameno"],
+                                funcname=frame["funcname"],
+                                file=match["file"],
+                                lineno=metadata_function_line
+                            ))
+                            break
+                        i -= 1
+                else:
+                    better_frames.append("#{frameno}: {funcname}, {file}, line {lineno}".format(**frame))
+
+            if better_frames:
+                better_frames = ("\t{0}".format(frame) for frame in better_frames)
+                exe.extra_message = "\nBacktrace (metadata-relative locations):\n{0}".format("\n".join(better_frames))
+            raise
         finally:
             os.unlink(fifopath)
 

bitbake/lib/bb/cache.py

@@ -26,7 +26,7 @@ import re
 
 logger = logging.getLogger("BitBake.Cache")
 
-__cache_version__ = "152"
+__cache_version__ = "153"
 
 def getCacheFile(path, filename, mc, data_hash):
     mcspec = ''

bitbake/lib/bb/command.py

@@ -84,7 +84,7 @@ class Command:
         if command not in CommandsAsync.__dict__:
             return None, "No such command"
         self.currentAsyncCommand = (command, commandline)
-        self.cooker.configuration.server_register_idlecallback(self.cooker.runCommands, self.cooker)
+        self.cooker.idleCallBackRegister(self.cooker.runCommands, self.cooker)
         return True, None
 
     def runAsyncCommand(self):
@@ -396,7 +396,7 @@ class CommandsSync:
         def sortkey(x):
             vfn, _ = x
             realfn, _, mc = bb.cache.virtualfn2realfn(vfn)
-            return (-command.cooker.collections[mc].calc_bbfile_priority(realfn), vfn)
+            return (-command.cooker.collections[mc].calc_bbfile_priority(realfn)[0], vfn)
 
         skipdict = OrderedDict(sorted(command.cooker.skiplist.items(), key=sortkey))
         return list(skipdict.items())
@@ -723,10 +723,10 @@ class CommandsAsync:
         """
         Find signature info files via the signature generator
         """
-        pn = params[0]
+        (mc, pn) = bb.runqueue.split_mc(params[0])
         taskname = params[1]
         sigs = params[2]
-        res = bb.siggen.find_siginfo(pn, taskname, sigs, command.cooker.data)
-        bb.event.fire(bb.event.FindSigInfoResult(res), command.cooker.data)
+        res = bb.siggen.find_siginfo(pn, taskname, sigs, command.cooker.databuilder.mcdata[mc])
+        bb.event.fire(bb.event.FindSigInfoResult(res), command.cooker.databuilder.mcdata[mc])
         command.finishAsyncCommand()
     findSigInfo.needcache = False

bitbake/lib/bb/cooker.py

@@ -148,7 +148,7 @@ class BBCooker:
     Manages one bitbake build run
     """
 
-    def __init__(self, configuration, featureSet=None):
+    def __init__(self, configuration, featureSet=None, idleCallBackRegister=None):
         self.recipecaches = None
         self.skiplist = {}
         self.featureset = CookerFeatures()
@@ -158,6 +158,8 @@ class BBCooker:
 
         self.configuration = configuration
 
+        self.idleCallBackRegister = idleCallBackRegister
+
         bb.debug(1, "BBCooker starting %s" % time.time())
         sys.stdout.flush()
 
@@ -210,7 +212,7 @@ class BBCooker:
             cooker.process_inotify_updates()
             return 1.0
 
-        self.configuration.server_register_idlecallback(_process_inotify_updates, self)
+        self.idleCallBackRegister(_process_inotify_updates, self)
 
         # TOSTOP must not be set or our children will hang when they output
         try:
@@ -626,6 +628,7 @@ class BBCooker:
         current = 0
         runlist = []
         for k in fulltargetlist:
+            origk = k
             mc = ""
             if k.startswith("mc:"):
                 mc = k.split(":")[1]
@@ -635,6 +638,10 @@ class BBCooker:
                 k2 = k.split(":do_")
                 k = k2[0]
                 ktask = k2[1]
+
+            if mc not in self.multiconfigs:
+                 bb.fatal("Multiconfig dependency %s depends on nonexistent multiconfig configuration named %s" % (origk, mc))
+
             taskdata[mc].add_provider(localdata[mc], self.recipecaches[mc], k)
             current += 1
             if not ktask.startswith("do_"):
@@ -670,7 +677,7 @@ class BBCooker:
                         l = k.split(':')
                         depmc = l[2]
                         if depmc not in self.multiconfigs:
-                            bb.fatal("Multiconfig dependency %s depends on nonexistent mc configuration %s" % (k,depmc))
+                            bb.fatal("Multiconfig dependency %s depends on nonexistent multiconfig configuration named configuration %s" % (k,depmc))
                         else:
                             logger.debug(1, "Adding providers for multiconfig dependency %s" % l[3])
                             taskdata[depmc].add_provider(localdata[depmc], self.recipecaches[depmc], l[3])
@@ -1418,7 +1425,7 @@ class BBCooker:
                 return True
             return retval
 
-        self.configuration.server_register_idlecallback(buildFileIdle, rq)
+        self.idleCallBackRegister(buildFileIdle, rq)
 
     def buildTargets(self, targets, task):
         """
@@ -1489,7 +1496,7 @@ class BBCooker:
         if 'universe' in targets:
             rq.rqdata.warn_multi_bb = True
 
-        self.configuration.server_register_idlecallback(buildTargetsIdle, rq)
+        self.idleCallBackRegister(buildTargetsIdle, rq)
 
 
     def getAllKeysWithFlags(self, flaglist):
@@ -1588,7 +1595,7 @@ class BBCooker:
             self.show_appends_with_no_recipes()
             self.handlePrefProviders()
             for mc in self.multiconfigs:
-                self.recipecaches[mc].bbfile_priority = self.collections[mc].collection_priorities(self.recipecaches[mc].pkg_fn, self.data)
+                self.recipecaches[mc].bbfile_priority = self.collections[mc].collection_priorities(self.recipecaches[mc].pkg_fn, self.parser.mcfilelist[mc], self.data)
             self.state = state.running
 
             # Send an event listing all stamps reachable after parsing
@@ -1609,7 +1616,7 @@ class BBCooker:
             raise NothingToBuild
 
         ignore = (self.data.getVar("ASSUME_PROVIDED") or "").split()
-        for pkg in pkgs_to_build:
+        for pkg in pkgs_to_build.copy():
             if pkg in ignore:
                 parselog.warning("Explicit target \"%s\" is in ASSUME_PROVIDED, ignoring" % pkg)
             if pkg.startswith("multiconfig:"):
@@ -1704,14 +1711,11 @@ class CookerCollectFiles(object):
         # the shortest.  This allows nested layers to be properly evaluated.
         self.bbfile_config_priorities = sorted(priorities, key=lambda tup: tup[1], reverse=True)
 
-    def calc_bbfile_priority( self, filename, matched = None ):
+    def calc_bbfile_priority(self, filename):
         for _, _, regex, pri in self.bbfile_config_priorities:
             if regex.match(filename):
-                if matched is not None:
-                    if not regex in matched:
-                        matched.add(regex)
-                return pri
-        return 0
+                return pri, regex
+        return 0, None
 
     def get_bbfiles(self):
         """Get list of default .bb files by reading out the current directory"""
@@ -1744,7 +1748,7 @@ class CookerCollectFiles(object):
         config.setVar("BBFILES", " ".join(files))
 
         # Sort files by priority
-        files.sort( key=lambda fileitem: self.calc_bbfile_priority(fileitem) )
+        files.sort( key=lambda fileitem: self.calc_bbfile_priority(fileitem)[0] )
 
         if not len(files):
             files = self.get_bbfiles()
@@ -1866,39 +1870,62 @@ class CookerCollectFiles(object):
                 filelist.append(filename)
         return tuple(filelist)
 
-    def collection_priorities(self, pkgfns, d):
+    def collection_priorities(self, pkgfns, fns, d):
+        # Return the priorities of the entries in pkgfns
+        # Also check that all the regexes in self.bbfile_config_priorities are used
+        # (but to do that we need to ensure skipped recipes aren't counted, nor
+        # collections in BBFILE_PATTERN_IGNORE_EMPTY)
 
         priorities = {}
+        seen = set()
+        matched = set()
+
+        matched_regex = set()
+        unmatched_regex = set()
+        for _, _, regex, _ in self.bbfile_config_priorities:
+            unmatched_regex.add(regex)
 
         # Calculate priorities for each file
-        matched = set()
         for p in pkgfns:
             realfn, cls, mc = bb.cache.virtualfn2realfn(p)
-            priorities[p] = self.calc_bbfile_priority(realfn, matched)
+            priorities[p], regex = self.calc_bbfile_priority(realfn)
+            if regex in unmatched_regex:
+                matched_regex.add(regex)
+                unmatched_regex.remove(regex)
+            seen.add(realfn)
+            if regex:
+                matched.add(realfn)
 
-        unmatched = set()
-        for _, _, regex, pri in self.bbfile_config_priorities:
-            if not regex in matched:
-                unmatched.add(regex)
-
-        # Don't show the warning if the BBFILE_PATTERN did match .bbappend files
-        def find_bbappend_match(regex):
+        if unmatched_regex:
+            # Account for bbappend files
             for b in self.bbappends:
                 (bbfile, append) = b
-                if regex.match(append):
-                    # If the bbappend is matched by already "matched set", return False
-                    for matched_regex in matched:
-                        if matched_regex.match(append):
-                            return False
-                    return True
-            return False
+                seen.add(append)
 
-        for unmatch in unmatched.copy():
-            if find_bbappend_match(unmatch):
-                unmatched.remove(unmatch)
+            # Account for skipped recipes
+            seen.update(fns)
+
+            seen.difference_update(matched)
+
+            def already_matched(fn):
+                for regex in matched_regex:
+                    if regex.match(fn):
+                        return True
+                return False
+
+            for unmatch in unmatched_regex.copy():
+                for fn in seen:
+                    if unmatch.match(fn):
+                        # If the bbappend or file was already matched by another regex, skip it
+                        # e.g. for a layer within a layer, the outer regex could match, the inner
+                        # regex may match nothing and we should warn about that
+                        if already_matched(fn):
+                            continue
+                        unmatched_regex.remove(unmatch)
+                        break
 
         for collection, pattern, regex, _ in self.bbfile_config_priorities:
-            if regex in unmatched:
+            if regex in unmatched_regex:
                 if d.getVar('BBFILE_PATTERN_IGNORE_EMPTY_%s' % collection) != '1':
                     collectlog.warning("No bb files in %s matched BBFILE_PATTERN_%s '%s'" % (self.mc if self.mc else 'default',
                                                                                              collection, pattern))

bitbake/lib/bb/cookerdata.py

@@ -143,16 +143,10 @@ class CookerConfiguration(object):
                 setattr(self, key, parameters.options.__dict__[key])
         self.env = parameters.environment.copy()
 
-    def setServerRegIdleCallback(self, srcb):
-        self.server_register_idlecallback = srcb
-
     def __getstate__(self):
         state = {}
         for key in self.__dict__.keys():
-            if key == "server_register_idlecallback":
-                state[key] = None
-            else:
-                state[key] = getattr(self, key)
+            state[key] = getattr(self, key)
         return state
 
     def __setstate__(self,state):
@@ -387,10 +381,13 @@ class CookerDataBuilder(object):
                     invalid.append(entry)
                     continue
                 l, f = parts
-                if l in collections:
+                invert = l[0] == "!"
+                if invert:
+                    l = l[1:]
+                if (l in collections and not invert) or (l not in collections and invert):
                     data.appendVar("BBFILES", " " + f)
             if invalid:
-                bb.fatal("BBFILES_DYNAMIC entries must be of the form <collection name>:<filename pattern>, not:\n    %s" % "\n    ".join(invalid))
+                bb.fatal("BBFILES_DYNAMIC entries must be of the form {!}<collection name>:<filename pattern>, not:\n    %s" % "\n    ".join(invalid))
 
             layerseries = set((data.getVar("LAYERSERIES_CORENAMES") or "").split())
             collections_tmp = collections[:]

bitbake/lib/bb/data.py

@@ -161,6 +161,12 @@ def emit_var(var, o=sys.__stdout__, d = init(), all=False):
         return True
 
     if func:
+        # Write a comment indicating where the shell function came from (line number and filename) to make it easier
+        # for the user to diagnose task failures. This comment is also used by build.py to determine the metadata
+        # location of shell functions.
+        o.write("# line: {0}, file: {1}\n".format(
+            d.getVarFlag(var, "lineno", False),
+            d.getVarFlag(var, "filename", False)))
         # NOTE: should probably check for unbalanced {} within the var
         val = val.rstrip('\n')
         o.write("%s() {\n%s\n}\n" % (varExpanded, val))

bitbake/lib/bb/fetch2/git.py

@@ -355,7 +355,7 @@ class Git(FetchMethod):
               runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir)
 
             runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, repourl), d, workdir=ud.clonedir)
-            fetch_cmd = "LANG=C %s fetch -f --prune --progress %s refs/*:refs/*" % (ud.basecmd, repourl)
+            fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, repourl)
             if ud.proto.lower() != 'file':
                 bb.fetch2.check_network_access(d, fetch_cmd, ud.url)
             progresshandler = GitProgressHandler(d)

bitbake/lib/bb/fetch2/perforce.py

@@ -1,6 +1,20 @@
 """
 BitBake 'Fetch' implementation for perforce
 
+Supported SRC_URI options are:
+
+- module
+   The top-level location to fetch while preserving the remote paths
+
+   The value of module can point to either a directory or a file. The result,
+   in both cases, is that the fetcher will preserve all file paths starting
+   from the module path. That is, the top-level directory in the module value
+   will also be the top-level directory in P4DIR.
+
+- remotepath
+   If the value "keep" is given, the full depot location of each file is
+   preserved in P4DIR. This option overrides the effect of the module option.
+
 """
 
 # Copyright (C) 2003, 2004  Chris Larson
@@ -17,6 +31,36 @@ from   bb.fetch2 import FetchError
 from   bb.fetch2 import logger
 from   bb.fetch2 import runfetchcmd
 
+class PerforceProgressHandler (bb.progress.BasicProgressHandler):
+    """
+    Implements basic progress information for perforce, based on the number of
+    files to be downloaded.
+
+    The p4 print command will print one line per file, therefore it can be used
+    to "count" the number of files already completed and give an indication of
+    the progress.
+    """
+    def __init__(self, d, num_files):
+        self._num_files = num_files
+        self._count = 0
+        super(PerforceProgressHandler, self).__init__(d)
+
+        # Send an initial progress event so the bar gets shown
+        self._fire_progress(-1)
+
+    def write(self, string):
+        self._count = self._count + 1
+
+        percent = int(100.0 * float(self._count) / float(self._num_files))
+
+        # In case something goes wrong, we try to preserve our sanity
+        if percent > 100:
+            percent = 100
+
+        self.update(percent)
+
+        super(PerforceProgressHandler, self).write(string)
+
 class Perforce(FetchMethod):
     """ Class to fetch from perforce repositories """
     def supports(self, ud, d):
@@ -58,14 +102,33 @@ class Perforce(FetchMethod):
             logger.debug(1, 'Determined P4PORT to be: %s' % ud.host)
             if not ud.host:
                 raise FetchError('Could not determine P4PORT from P4CONFIG')
-	
+
+        # Fetcher options
+        ud.module = ud.parm.get('module')
+        ud.keepremotepath = (ud.parm.get('remotepath', '') == 'keep')
+
         if ud.path.find('/...') >= 0:
             ud.pathisdir = True
         else:
             ud.pathisdir = False
 
+        # Avoid using the "/..." syntax in SRC_URI when a module value is given
+        if ud.pathisdir and ud.module:
+            raise FetchError('SRC_URI depot path cannot not end in /... when a module value is given')
+
         cleanedpath = ud.path.replace('/...', '').replace('/', '.')
         cleanedhost = ud.host.replace(':', '.')
+
+        # Merge the path and module into the final depot location
+        if ud.module:
+            if ud.module.find('/') == 0:
+                raise FetchError('module cannot begin with /')
+            ud.path = os.path.join(ud.path, ud.module)
+
+            # Append the module path to the local pkg name
+            cleanedmodule = ud.module.replace('/...', '').replace('/', '.')
+            cleanedpath += '--%s' % cleanedmodule
+
         ud.pkgdir = os.path.join(ud.dldir, cleanedhost, cleanedpath)
 
         ud.setup_revisions(d)
@@ -95,10 +158,20 @@ class Perforce(FetchMethod):
             pathnrev = '%s' % (ud.path)
 
         if depot_filename:
-            if ud.pathisdir: # Remove leading path to obtain filename
+            if ud.keepremotepath:
+                # preserve everything, remove the leading //
+                filename = depot_filename.lstrip('/')
+            elif ud.module:
+                # remove everything up to the module path
+                modulepath = ud.module.rstrip('/...')
+                filename = depot_filename[depot_filename.rfind(modulepath):]
+            elif ud.pathisdir:
+                # Remove leading (visible) path to obtain the filepath
                 filename = depot_filename[len(ud.path)-1:]
             else:
+                # Remove everything, except the filename
                 filename = depot_filename[depot_filename.rfind('/'):]
+
             filename = filename[:filename.find('#')] # Remove trailing '#rev'
 
         if command == 'changes':
@@ -150,10 +223,12 @@ class Perforce(FetchMethod):
         bb.utils.remove(ud.pkgdir, True)
         bb.utils.mkdirhier(ud.pkgdir)
 
+        progresshandler = PerforceProgressHandler(d, len(filelist))
+
         for afile in filelist:
             p4fetchcmd = self._buildp4command(ud, d, 'print', afile)
             bb.fetch2.check_network_access(d, p4fetchcmd, ud.url)
-            runfetchcmd(p4fetchcmd, d, workdir=ud.pkgdir)
+            runfetchcmd(p4fetchcmd, d, workdir=ud.pkgdir, log=progresshandler)
 
         runfetchcmd('tar -czf %s p4' % (ud.localpath), d, cleanup=[ud.localpath], workdir=ud.pkgdir)
 

bitbake/lib/bb/msg.py

@@ -280,7 +280,7 @@ def setLoggingConfig(defaultconfig, userconfigfile=None):
     logconfig = copy.deepcopy(defaultconfig)
 
     if userconfigfile:
-        with open(userconfigfile, 'r') as f:
+        with open(os.path.normpath(userconfigfile), 'r') as f:
             if userconfigfile.endswith('.yml') or userconfigfile.endswith('.yaml'):
                 import yaml
                 userconfig = yaml.load(f)

bitbake/lib/bb/parse/ast.py

@@ -244,12 +244,14 @@ class AddTaskNode(AstNode):
         bb.build.addtask(self.func, self.before, self.after, data)
 
 class DelTaskNode(AstNode):
-    def __init__(self, filename, lineno, func):
+    def __init__(self, filename, lineno, tasks):
         AstNode.__init__(self, filename, lineno)
-        self.func = func
+        self.tasks = tasks
 
     def eval(self, data):
-        bb.build.deltask(self.func, data)
+        tasks = data.expand(self.tasks).split()
+        for task in tasks:
+            bb.build.deltask(task, data)
 
 class BBHandlerNode(AstNode):
     def __init__(self, filename, lineno, fns):
@@ -305,7 +307,7 @@ def handleAddTask(statements, filename, lineno, m):
     statements.append(AddTaskNode(filename, lineno, func, before, after))
 
 def handleDelTask(statements, filename, lineno, m):
-    func = m.group("func")
+    func = m.group(1)
     if func is None:
         return
 

bitbake/lib/bb/parse/parse_py/BBHandler.py

@@ -26,7 +26,7 @@ __func_start_regexp__    = re.compile(r"(((?P<py>python)|(?P<fr>fakeroot))\s*)*(
 __inherit_regexp__       = re.compile(r"inherit\s+(.+)" )
 __export_func_regexp__   = re.compile(r"EXPORT_FUNCTIONS\s+(.+)" )
 __addtask_regexp__       = re.compile(r"addtask\s+(?P<func>\w+)\s*((before\s*(?P<before>((.*(?=after))|(.*))))|(after\s*(?P<after>((.*(?=before))|(.*)))))*")
-__deltask_regexp__       = re.compile(r"deltask\s+(?P<func>\w+)(?P<ignores>.*)")
+__deltask_regexp__       = re.compile(r"deltask\s+(.+)")
 __addhandler_regexp__    = re.compile(r"addhandler\s+(.+)" )
 __def_regexp__           = re.compile(r"def\s+(\w+).*:" )
 __python_func_regexp__   = re.compile(r"(\s+.*)|(^$)|(^#)" )
@@ -238,9 +238,6 @@ def feeder(lineno, s, fn, root, statements, eof=False):
 
     m = __deltask_regexp__.match(s)
     if m:
-        # Check and warn "for deltask task1 task2"
-        if m.group('ignores'):
-            logger.warning('deltask ignored: "%s"' % m.group('ignores'))
         ast.handleDelTask(statements, fn, lineno, m)
         return
 

bitbake/lib/bb/process.py

@@ -41,6 +41,7 @@ class ExecutionError(CmdError):
         self.exitcode = exitcode
         self.stdout = stdout
         self.stderr = stderr
+        self.extra_message = None
 
     def __str__(self):
         message = ""
@@ -51,7 +52,7 @@ class ExecutionError(CmdError):
         if message:
             message = ":\n" + message
         return (CmdError.__str__(self) +
-                " with exit code %s" % self.exitcode + message)
+                " with exit code %s" % self.exitcode + message + (self.extra_message or ""))
 
 class Popen(subprocess.Popen):
     defaults = {

bitbake/lib/bb/progress.py

@@ -14,7 +14,27 @@ import bb.event
 import bb.build
 from bb.build import StdoutNoopContextManager
 
-class ProgressHandler(object):
+
+# from https://stackoverflow.com/a/14693789/221061
+ANSI_ESCAPE_REGEX = re.compile(r'\x1B\[[0-?]*[ -/]*[@-~]')
+
+
+def filter_color(string):
+    """
+    Filter ANSI escape codes out of |string|, return new string
+    """
+    return ANSI_ESCAPE_REGEX.sub('', string)
+
+
+def filter_color_n(string):
+    """
+    Filter ANSI escape codes out of |string|, returns tuple of
+    (new string, # of ANSI codes removed)
+    """
+    return ANSI_ESCAPE_REGEX.subn('', string)
+
+
+class ProgressHandler:
     """
     Base class that can pretend to be a file object well enough to be
     used to build objects to intercept console output and determine the
@@ -55,6 +75,7 @@ class ProgressHandler(object):
             self._lastevent = ts
             self._progress = progress
 
+
 class LineFilterProgressHandler(ProgressHandler):
     """
     A ProgressHandler variant that provides the ability to filter out
@@ -66,7 +87,7 @@ class LineFilterProgressHandler(ProgressHandler):
     """
     def __init__(self, d, outfile=None):
         self._linebuffer = ''
-        super(LineFilterProgressHandler, self).__init__(d, outfile)
+        super().__init__(d, outfile)
 
     def write(self, string):
         self._linebuffer += string
@@ -80,41 +101,44 @@ class LineFilterProgressHandler(ProgressHandler):
             lbreakpos = line.rfind('\r') + 1
             if lbreakpos:
                 line = line[lbreakpos:]
-            if self.writeline(line):
-                super(LineFilterProgressHandler, self).write(line)
+            if self.writeline(filter_color(line)):
+                super().write(line)
 
     def writeline(self, line):
         return True
 
+
 class BasicProgressHandler(ProgressHandler):
     def __init__(self, d, regex=r'(\d+)%', outfile=None):
-        super(BasicProgressHandler, self).__init__(d, outfile)
+        super().__init__(d, outfile)
         self._regex = re.compile(regex)
         # Send an initial progress event so the bar gets shown
         self._fire_progress(0)
 
     def write(self, string):
-        percs = self._regex.findall(string)
+        percs = self._regex.findall(filter_color(string))
         if percs:
             progress = int(percs[-1])
             self.update(progress)
-        super(BasicProgressHandler, self).write(string)
+        super().write(string)
+
 
 class OutOfProgressHandler(ProgressHandler):
     def __init__(self, d, regex, outfile=None):
-        super(OutOfProgressHandler, self).__init__(d, outfile)
+        super().__init__(d, outfile)
         self._regex = re.compile(regex)
         # Send an initial progress event so the bar gets shown
         self._fire_progress(0)
 
     def write(self, string):
-        nums = self._regex.findall(string)
+        nums = self._regex.findall(filter_color(string))
         if nums:
             progress = (float(nums[-1][0]) / float(nums[-1][1])) * 100
             self.update(progress)
-        super(OutOfProgressHandler, self).write(string)
+        super().write(string)
 
-class MultiStageProgressReporter(object):
+
+class MultiStageProgressReporter:
     """
     Class which allows reporting progress without the caller
     having to know where they are in the overall sequence. Useful
@@ -199,6 +223,7 @@ class MultiStageProgressReporter(object):
           value is considered to be out of stage_total, otherwise it should
           be a percentage value from 0 to 100.
         """
+        progress = None
         if self._stage_total:
             stage_progress = (float(stage_progress) / self._stage_total) * 100
         if self._stage < 0:
@@ -207,9 +232,10 @@ class MultiStageProgressReporter(object):
             progress = self._base_progress + (stage_progress * self._stage_weights[self._stage])
         else:
             progress = self._base_progress
-        if progress > 100:
-            progress = 100
-        self._fire_progress(progress)
+        if progress:
+            if progress > 100:
+                progress = 100
+            self._fire_progress(progress)
 
     def finish(self):
         if self._finished:
@@ -230,6 +256,7 @@ class MultiStageProgressReporter(object):
                     out.append('Up to finish: %d' % stage_weight)
             bb.warn('Stage times:\n  %s' % '\n  '.join(out))
 
+
 class MultiStageProcessProgressReporter(MultiStageProgressReporter):
     """
     Version of MultiStageProgressReporter intended for use with
@@ -238,7 +265,7 @@ class MultiStageProcessProgressReporter(MultiStageProgressReporter):
     def __init__(self, d, processname, stage_weights, debug=False):
         self._processname = processname
         self._started = False
-        MultiStageProgressReporter.__init__(self, d, stage_weights, debug)
+        super().__init__(d, stage_weights, debug)
 
     def start(self):
         if not self._started:
@@ -255,13 +282,14 @@ class MultiStageProcessProgressReporter(MultiStageProgressReporter):
         MultiStageProgressReporter.finish(self)
         bb.event.fire(bb.event.ProcessFinished(self._processname), self._data)
 
+
 class DummyMultiStageProcessProgressReporter(MultiStageProgressReporter):
     """
     MultiStageProcessProgressReporter that takes the calls and does nothing
     with them (to avoid a bunch of "if progress_reporter:" checks)
     """
     def __init__(self):
-        MultiStageProcessProgressReporter.__init__(self, "", None, [])
+        super().__init__(None, [])
 
     def _fire_progress(self, taskprogress, rate=None):
         pass

bitbake/lib/bb/pysh/pyshyacc.py

@@ -570,6 +570,7 @@ def p_linebreak(p):
 
 def p_separator_op(p):                 
     """separator_op : COMMA
+                    | COMMA COMMA
                     | AMP"""
     p[0] = p[1]
 

bitbake/lib/bb/runqueue.py

@@ -2967,7 +2967,12 @@ class runQueuePipe():
             while index != -1 and self.queue.startswith(b"<event>"):
                 try:
                     event = pickle.loads(self.queue[7:index])
-                except ValueError as e:
+                except (ValueError, pickle.UnpicklingError, AttributeError, IndexError) as e:
+                    if isinstance(e, pickle.UnpicklingError) and "truncated" in str(e):
+                        # The pickled data could contain "</event>" so search for the next occurance
+                        # unpickling again, this should be the only way an unpickle error could occur
+                        index = self.queue.find(b"</event>", index + 1)
+                        continue
                     bb.msg.fatal("RunQueue", "failed load pickle '%s': '%s'" % (e, self.queue[7:index]))
                 bb.event.fire_from_worker(event, self.d)
                 if isinstance(event, taskUniHashUpdate):
@@ -2979,7 +2984,7 @@ class runQueuePipe():
             while index != -1 and self.queue.startswith(b"<exitcode>"):
                 try:
                     task, status = pickle.loads(self.queue[10:index])
-                except ValueError as e:
+                except (ValueError, pickle.UnpicklingError, AttributeError, IndexError) as e:
                     bb.msg.fatal("RunQueue", "failed load pickle '%s': '%s'" % (e, self.queue[10:index]))
                 self.rqexec.runqueue_process_waitpid(task, status)
                 found = True

bitbake/lib/bb/server/process.py

@@ -34,12 +34,11 @@ logger = logging.getLogger('BitBake')
 class ProcessTimeout(SystemExit):
     pass
 
-class ProcessServer(multiprocessing.Process):
+class ProcessServer():
     profile_filename = "profile.log"
     profile_processed_filename = "profile.log.processed"
 
-    def __init__(self, lock, sock, sockname):
-        multiprocessing.Process.__init__(self)
+    def __init__(self, lock, sock, sockname, server_timeout, xmlrpcinterface):
         self.command_channel = False
         self.command_channel_reply = False
         self.quit = False
@@ -47,8 +46,9 @@ class ProcessServer(multiprocessing.Process):
         self.next_heartbeat = time.time()
 
         self.event_handle = None
+        self.hadanyui = False
         self.haveui = False
-        self.lastui = False
+        self.maxuiwait = 30
         self.xmlrpc = False
 
         self._idlefuns = {}
@@ -57,6 +57,9 @@ class ProcessServer(multiprocessing.Process):
         self.sock = sock
         self.sockname = sockname
 
+        self.server_timeout = server_timeout
+        self.xmlrpcinterface = xmlrpcinterface
+
     def register_idle_function(self, function, data):
         """Register a function to be called while the server is idle"""
         assert hasattr(function, '__call__')
@@ -155,6 +158,7 @@ class ProcessServer(multiprocessing.Process):
                 print("No timeout, exiting.")
                 self.quit = True
 
+        self.lastui = time.time()
         while not self.quit:
             if self.sock in ready:
                 while select.select([self.sock],[],[],0)[0]:
@@ -187,15 +191,23 @@ class ProcessServer(multiprocessing.Process):
                     self.command_channel_reply = writer
 
                     self.haveui = True
+                    self.hadanyui = True
 
                 except (EOFError, OSError):
                     disconnect_client(self, fds)
 
-            if not self.timeout == -1.0 and not self.haveui and self.lastui and self.timeout and \
+            if not self.timeout == -1.0 and not self.haveui and self.timeout and \
                     (self.lastui + self.timeout) < time.time():
                 print("Server timeout, exiting.")
                 self.quit = True
 
+            # If we don't see a UI connection within maxuiwait, its unlikely we're going to see
+            # one. We have had issue with processes hanging indefinitely so timing out UI-less
+            # servers is useful.
+            if not self.hadanyui and not self.xmlrpc and not self.timeout and (self.lastui + self.maxuiwait) < time.time():
+                print("No UI connection within max timeout, exiting to avoid infinite loop.")
+                self.quit = True
+
             if self.command_channel in ready:
                 try:
                     command = self.command_channel.get()
@@ -220,10 +232,13 @@ class ProcessServer(multiprocessing.Process):
 
         print("Exiting")
         # Remove the socket file so we don't get any more connections to avoid races
-        os.unlink(self.sockname)
+        try:
+            os.unlink(self.sockname)
+        except:
+            pass
         self.sock.close()
 
-        try: 
+        try:
             self.cooker.shutdown(True)
             self.cooker.notifier.stop()
             self.cooker.confignotifier.stop()
@@ -232,6 +247,10 @@ class ProcessServer(multiprocessing.Process):
 
         self.cooker.post_serve()
 
+        # Flush logs before we release the lock
+        sys.stdout.flush()
+        sys.stderr.flush()
+
         # Finally release the lockfile but warn about other processes holding it open
         lock = self.bitbake_lock
         lockfile = lock.name
@@ -243,7 +262,7 @@ class ProcessServer(multiprocessing.Process):
                 lock = bb.utils.lockfile(lockfile, shared=False, retry=False, block=True)
                 if lock:
                     # We hold the lock so we can remove the file (hide stale pid data)
-                    bb.utils.remove(lockfile)
+                    # via unlockfile.
                     bb.utils.unlockfile(lock)
                     return
 
@@ -331,7 +350,9 @@ class ServerCommunicator():
     def runCommand(self, command):
         self.connection.send(command)
         if not self.recv.poll(30):
-            raise ProcessTimeout("Timeout while waiting for a reply from the bitbake server")
+            logger.info("No reply from server in 30s")
+            if not self.recv.poll(30):
+                raise ProcessTimeout("Timeout while waiting for a reply from the bitbake server (60s)")
         return self.recv.get()
 
     def updateFeatureSet(self, featureset):
@@ -452,23 +473,25 @@ class BitBakeServer(object):
         print(self.start_log_format % (os.getpid(), datetime.datetime.now().strftime(self.start_log_datetime_format)))
         sys.stdout.flush()
 
-        server = ProcessServer(self.bitbake_lock, self.sock, self.sockname)
-        self.configuration.setServerRegIdleCallback(server.register_idle_function)
-        os.close(self.readypipe)
-        writer = ConnectionWriter(self.readypipein)
         try:
-            self.cooker = bb.cooker.BBCooker(self.configuration, self.featureset)
-        except bb.BBHandledException:
-            return None
-        writer.send("r")
-        writer.close()
-        server.cooker = self.cooker
-        server.server_timeout = self.configuration.server_timeout
-        server.xmlrpcinterface = self.configuration.xmlrpcinterface
-        print("Started bitbake server pid %d" % os.getpid())
-        sys.stdout.flush()
+            server = ProcessServer(self.bitbake_lock, self.sock, self.sockname, self.configuration.server_timeout, self.configuration.xmlrpcinterface)
+            os.close(self.readypipe)
+            writer = ConnectionWriter(self.readypipein)
+            try:
+                self.cooker = bb.cooker.BBCooker(self.configuration, self.featureset, server.register_idle_function)
+            except bb.BBHandledException:
+                return None
+            writer.send("r")
+            writer.close()
+            server.cooker = self.cooker
+            print("Started bitbake server pid %d" % os.getpid())
+            sys.stdout.flush()
 
-        server.start()
+            server.run()
+        finally:
+            # Flush any ,essages/errors to the logfile before exit
+            sys.stdout.flush()
+            sys.stderr.flush()
 
 def connectProcessServer(sockname, featureset):
     # Connect to socket

bitbake/lib/bb/siggen.py

@@ -14,6 +14,7 @@ import simplediff
 from bb.checksum import FileChecksumCache
 from bb import runqueue
 import hashserv
+import hashserv.client
 
 logger = logging.getLogger('BitBake.SigGen')
 hashequiv_logger = logging.getLogger('BitBake.SigGen.HashEquiv')

bitbake/lib/bb/taskdata.py

@@ -21,8 +21,13 @@ def re_match_strings(target, strings):
     Whether or not the string 'target' matches
     any one string of the strings which can be regular expression string
     """
-    return any(name == target or re.match(name, target)
-               for name in strings)
+    for name in strings:
+        if name.startswith("^") or name.endswith("$"):
+            if re.match(name, target):
+                return True
+        elif name == target:
+            return True
+    return False
 
 class TaskEntry:
     def __init__(self):

bitbake/lib/bb/tests/color.py

@@ -0,0 +1,95 @@
+#
+# BitBake Test for ANSI color code filtering
+#
+# Copyright (C) 2020  Agilent Technologies, Inc.
+# Author: Chris Laplante <chris.laplante@agilent.com>
+#
+# SPDX-License-Identifier: MIT
+#
+
+import unittest
+import bb.progress
+import bb.data
+import bb.event
+from bb.progress import filter_color, filter_color_n
+import io
+import re
+
+
+class ProgressWatcher:
+    def __init__(self):
+        self._reports = []
+
+    def handle_event(self, event):
+        self._reports.append((event.progress, event.rate))
+
+    def reports(self):
+        return self._reports
+
+
+class ColorCodeTests(unittest.TestCase):
+    def setUp(self):
+        self.d = bb.data.init()
+        self._progress_watcher = ProgressWatcher()
+        bb.event.register("bb.build.TaskProgress", self._progress_watcher.handle_event)
+
+    def tearDown(self):
+        bb.event.remove("bb.build.TaskProgress", None)
+
+    def test_filter_color(self):
+        input_string = "~~~~~~~~~~~~^~~~~~~~"
+        filtered = filter_color(input_string)
+        self.assertEqual(filtered, "~~~~~~~~~~~~^~~~~~~~")
+
+    def test_filter_color_n(self):
+        input_string = "~~~~~~~~~~~~^~~~~~~~"
+        filtered, code_count = filter_color_n(input_string)
+        self.assertEqual(filtered, "~~~~~~~~~~~~^~~~~~~~")
+        self.assertEqual(code_count, 4)
+
+    def test_LineFilterProgressHandler_color_filtering(self):
+        class CustomProgressHandler(bb.progress.LineFilterProgressHandler):
+            PROGRESS_REGEX = re.compile(r"Progress: (?P<progress>\d+)%")
+
+            def writeline(self, line):
+                match = self.PROGRESS_REGEX.match(line)
+                if match:
+                    self.update(int(match.group("progress")))
+                    return False
+                return True
+
+        buffer = io.StringIO()
+        handler = CustomProgressHandler(self.d, buffer)
+        handler.write("Program output!\n")
+        handler.write("More output!\n")
+        handler.write("Progress: 10%\n") # 10%
+        handler.write("Even more\n")
+        handler.write("Progress: 50%\n") # 50%
+        handler.write("Progress: 60%\n") # 60%
+        handler.write("Progress: 100%\n") # 100%
+
+        expected = [(10, None), (50, None), (60, None), (100, None)]
+        self.assertEqual(self._progress_watcher.reports(), expected)
+
+        self.assertEqual(buffer.getvalue(), "Program output!\nMore output!\nEven more\n")
+
+    def test_BasicProgressHandler_color_filtering(self):
+        buffer = io.StringIO()
+        handler = bb.progress.BasicProgressHandler(self.d, outfile=buffer)
+        handler.write("1%\n") # 1%
+        handler.write("2%\n") # 2%
+        handler.write("10%\n") # 10%
+        handler.write("100%\n") # 100%
+
+        expected = [(0, None), (1, None), (2, None), (10, None), (100, None)]
+        self.assertListEqual(self._progress_watcher.reports(), expected)
+
+    def test_OutOfProgressHandler_color_filtering(self):
+        buffer = io.StringIO()
+        handler = bb.progress.OutOfProgressHandler(self.d, r'(\d+) of (\d+)', outfile=buffer)
+        handler.write("Text text 1 of 5") # 1/5
+        handler.write("Text text 3 of 5") # 3/5
+        handler.write("Text text 5 of 5") # 5/5
+
+        expected = [(0, None), (20.0, None), (60.0, None), (100.0, None)]
+        self.assertListEqual(self._progress_watcher.reports(), expected)

bitbake/lib/bb/tests/cooker.py

@@ -60,7 +60,7 @@ class CookerTest(unittest.TestCase):
         log_handler = LogHandler()
         logger.addHandler(log_handler)
         collection = bb.cooker.CookerCollectFiles(bbfile_config_priorities)
-        collection.collection_priorities(pkgfns, self.d)
+        collection.collection_priorities(pkgfns, pkgfns, self.d)
         logger.removeHandler(log_handler)
 
         # Should be empty (no generated messages)

bitbake/lib/bb/tests/fetch.py

@@ -1031,7 +1031,7 @@ class SVNTest(FetcherTest):
 
         bb.process.run("svn co %s svnfetch_co" % self.repo_url, cwd=self.tempdir)
         # Github will emulate SVN.  Use this to check if we're downloding...
-        bb.process.run("svn propset svn:externals 'bitbake http://github.com/openembedded/bitbake' .",
+        bb.process.run("svn propset svn:externals 'bitbake svn://vcs.pcre.org/pcre2/code' .",
                        cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk'))
         bb.process.run("svn commit --non-interactive -m 'Add external'",
                        cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk'))
@@ -1152,7 +1152,8 @@ class FetchLatestVersionTest(FetcherTest):
         ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4", "9b1db6b8060bd00b121a692f942404a24ae2960f", "")
             : "1.99.4",
         # version pattern "vX.Y"
-        ("mtd-utils", "git://git.infradead.org/mtd-utils.git", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "")
+        # mirror of git.infradead.org since network issues interfered with testing
+        ("mtd-utils", "git://git.yoctoproject.org/mtd-utils.git", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "")
             : "1.5.0",
         # version pattern "pkg_name-X.Y"
         ("presentproto", "git://anongit.freedesktop.org/git/xorg/proto/presentproto", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "")

bitbake/lib/bb/tests/parse.py

@@ -178,7 +178,10 @@ python () {
 addtask do_patch after do_foo after do_unpack before do_configure before do_compile
 addtask do_fetch do_patch
 
-deltask do_fetch do_patch
+MYVAR = "do_patch"
+EMPTYVAR = ""
+deltask do_fetch ${MYVAR} ${EMPTYVAR}
+deltask ${EMPTYVAR}
 """
     def test_parse_addtask_deltask(self):
         import sys
@@ -189,6 +192,5 @@ deltask do_fetch do_patch
         self.assertTrue("addtask contained multiple 'before' keywords" in stdout)
         self.assertTrue("addtask contained multiple 'after' keywords" in stdout)
         self.assertTrue('addtask ignored: " do_patch"' in stdout)
-        self.assertTrue('deltask ignored: " do_patch"' in stdout)
         #self.assertTrue('dependent task do_foo for do_patch does not exist' in stdout)
 

bitbake/lib/bb/ui/taskexp.py

@@ -8,9 +8,16 @@
 #
 
 import sys
-import gi
-gi.require_version('Gtk', '3.0')
-from gi.repository import Gtk, Gdk, GObject
+
+try:
+    import gi
+    gi.require_version('Gtk', '3.0')
+    from gi.repository import Gtk, Gdk, GObject
+except ValueError:
+    sys.exit("FATAL: Gtk version needs to be 3.0")
+except ImportError:
+    sys.exit("FATAL: Gtk ui could not load the required gi python module")
+
 import threading
 from xmlrpc import client
 import bb

bitbake/lib/bb/ui/teamcity.py

@@ -167,8 +167,6 @@ def main(server, eventHandler, params):
         forcelevel = bb.msg.BBLogFormatter.ERROR
     else:
         forcelevel = bb.msg.BBLogFormatter.WARNING
-    bb.msg.addDefaultlogFilter(console, bb.msg.BBLogFilterStdOut, forcelevel)
-    bb.msg.addDefaultlogFilter(errconsole, bb.msg.BBLogFilterStdErr)
     console.setFormatter(format)
     errconsole.setFormatter(format)
     if not bb.msg.has_console_handler(logger):

bitbake/lib/hashserv/__init__.py

@@ -6,12 +6,20 @@
 from contextlib import closing
 import re
 import sqlite3
+import itertools
+import json
 
 UNIX_PREFIX = "unix://"
 
 ADDR_TYPE_UNIX = 0
 ADDR_TYPE_TCP = 1
 
+# The Python async server defaults to a 64K receive buffer, so we hardcode our
+# maximum chunk size. It would be better if the client and server reported to
+# each other what the maximum chunk sizes were, but that will slow down the
+# connection setup with a round trip delay so I'd rather not do that unless it
+# is necessary
+DEFAULT_MAX_CHUNK = 32 * 1024
 
 def setup_database(database, sync=True):
     db = sqlite3.connect(database)
@@ -66,6 +74,20 @@ def parse_address(addr):
         return (ADDR_TYPE_TCP, (host, int(port)))
 
 
+def chunkify(msg, max_chunk):
+    if len(msg) < max_chunk - 1:
+        yield ''.join((msg, "\n"))
+    else:
+        yield ''.join((json.dumps({
+                'chunk-stream': None
+            }), "\n"))
+
+        args = [iter(msg)] * (max_chunk - 1)
+        for m in map(''.join, itertools.zip_longest(*args, fillvalue='')):
+            yield ''.join(itertools.chain(m, "\n"))
+        yield "\n"
+
+
 def create_server(addr, dbname, *, sync=True):
     from . import server
     db = setup_database(dbname, sync=sync)

bitbake/lib/hashserv/client.py

@@ -7,6 +7,7 @@ import json
 import logging
 import socket
 import os
+from . import chunkify, DEFAULT_MAX_CHUNK
 
 
 logger = logging.getLogger('hashserv.client')
@@ -25,6 +26,7 @@ class Client(object):
         self.reader = None
         self.writer = None
         self.mode = self.MODE_NORMAL
+        self.max_chunk = DEFAULT_MAX_CHUNK
 
     def connect_tcp(self, address, port):
         def connect_sock():
@@ -58,7 +60,7 @@ class Client(object):
             self.reader = self._socket.makefile('r', encoding='utf-8')
             self.writer = self._socket.makefile('w', encoding='utf-8')
 
-            self.writer.write('OEHASHEQUIV 1.0\n\n')
+            self.writer.write('OEHASHEQUIV 1.1\n\n')
             self.writer.flush()
 
             # Restore mode if the socket is being re-created
@@ -91,18 +93,35 @@ class Client(object):
                 count += 1
 
     def send_message(self, msg):
-        def proc():
-            self.writer.write('%s\n' % json.dumps(msg))
-            self.writer.flush()
-
-            l = self.reader.readline()
-            if not l:
+        def get_line():
+            line = self.reader.readline()
+            if not line:
                 raise HashConnectionError('Connection closed')
 
-            if not l.endswith('\n'):
+            if not line.endswith('\n'):
                 raise HashConnectionError('Bad message %r' % message)
 
-            return json.loads(l)
+            return line
+
+        def proc():
+            for c in chunkify(json.dumps(msg), self.max_chunk):
+                self.writer.write(c)
+            self.writer.flush()
+
+            l = get_line()
+
+            m = json.loads(l)
+            if 'chunk-stream' in m:
+                lines = []
+                while True:
+                    l = get_line().rstrip('\n')
+                    if not l:
+                        break
+                    lines.append(l)
+
+                m = json.loads(''.join(lines))
+
+            return m
 
         return self._send_wrapper(proc)
 
@@ -155,6 +174,14 @@ class Client(object):
         m['unihash'] = unihash
         return self.send_message({'report-equiv': m})
 
+    def get_taskhash(self, method, taskhash, all_properties=False):
+        self._set_mode(self.MODE_NORMAL)
+        return self.send_message({'get': {
+            'taskhash': taskhash,
+            'method': method,
+            'all': all_properties
+        }})
+
     def get_stats(self):
         self._set_mode(self.MODE_NORMAL)
         return self.send_message({'get-stats': None})

bitbake/lib/hashserv/server.py

@@ -13,6 +13,7 @@ import os
 import signal
 import socket
 import time
+from . import chunkify, DEFAULT_MAX_CHUNK
 
 logger = logging.getLogger('hashserv.server')
 
@@ -107,12 +108,29 @@ class Stats(object):
         return {k: getattr(self, k) for k in ('num', 'total_time', 'max_time', 'average', 'stdev')}
 
 
+class ClientError(Exception):
+    pass
+
 class ServerClient(object):
+    FAST_QUERY = 'SELECT taskhash, method, unihash FROM tasks_v2 WHERE method=:method AND taskhash=:taskhash ORDER BY created ASC LIMIT 1'
+    ALL_QUERY =  'SELECT *                         FROM tasks_v2 WHERE method=:method AND taskhash=:taskhash ORDER BY created ASC LIMIT 1'
+
     def __init__(self, reader, writer, db, request_stats):
         self.reader = reader
         self.writer = writer
         self.db = db
         self.request_stats = request_stats
+        self.max_chunk = DEFAULT_MAX_CHUNK
+
+        self.handlers = {
+            'get': self.handle_get,
+            'report': self.handle_report,
+            'report-equiv': self.handle_equivreport,
+            'get-stream': self.handle_get_stream,
+            'get-stats': self.handle_get_stats,
+            'reset-stats': self.handle_reset_stats,
+            'chunk-stream': self.handle_chunk,
+        }
 
     async def process_requests(self):
         try:
@@ -125,7 +143,11 @@ class ServerClient(object):
                 return
 
             (proto_name, proto_version) = protocol.decode('utf-8').rstrip().split()
-            if proto_name != 'OEHASHEQUIV' or proto_version != '1.0':
+            if proto_name != 'OEHASHEQUIV':
+                return
+
+            proto_version = tuple(int(v) for v in proto_version.split('.'))
+            if proto_version < (1, 0) or proto_version > (1, 1):
                 return
 
             # Read headers. Currently, no headers are implemented, so look for
@@ -140,40 +162,34 @@ class ServerClient(object):
                     break
 
             # Handle messages
-            handlers = {
-                'get': self.handle_get,
-                'report': self.handle_report,
-                'report-equiv': self.handle_equivreport,
-                'get-stream': self.handle_get_stream,
-                'get-stats': self.handle_get_stats,
-                'reset-stats': self.handle_reset_stats,
-            }
-
             while True:
                 d = await self.read_message()
                 if d is None:
                     break
-
-                for k in handlers.keys():
-                    if k in d:
-                        logger.debug('Handling %s' % k)
-                        if 'stream' in k:
-                            await handlers[k](d[k])
-                        else:
-                            with self.request_stats.start_sample() as self.request_sample, \
-                                    self.request_sample.measure():
-                                await handlers[k](d[k])
-                        break
-                else:
-                    logger.warning("Unrecognized command %r" % d)
-                    break
-
+                await self.dispatch_message(d)
                 await self.writer.drain()
+        except ClientError as e:
+            logger.error(str(e))
         finally:
             self.writer.close()
 
+    async def dispatch_message(self, msg):
+        for k in self.handlers.keys():
+            if k in msg:
+                logger.debug('Handling %s' % k)
+                if 'stream' in k:
+                    await self.handlers[k](msg[k])
+                else:
+                    with self.request_stats.start_sample() as self.request_sample, \
+                            self.request_sample.measure():
+                        await self.handlers[k](msg[k])
+                return
+
+        raise ClientError("Unrecognized command %r" % msg)
+
     def write_message(self, msg):
-        self.writer.write(('%s\n' % json.dumps(msg)).encode('utf-8'))
+        for c in chunkify(json.dumps(msg), self.max_chunk):
+            self.writer.write(c.encode('utf-8'))
 
     async def read_message(self):
         l = await self.reader.readline()
@@ -191,14 +207,38 @@ class ServerClient(object):
             logger.error('Bad message from client: %r' % message)
             raise e
 
+    async def handle_chunk(self, request):
+        lines = []
+        try:
+            while True:
+                l = await self.reader.readline()
+                l = l.rstrip(b"\n").decode("utf-8")
+                if not l:
+                    break
+                lines.append(l)
+
+            msg = json.loads(''.join(lines))
+        except (json.JSONDecodeError, UnicodeDecodeError) as e:
+            logger.error('Bad message from client: %r' % message)
+            raise e
+
+        if 'chunk-stream' in msg:
+            raise ClientError("Nested chunks are not allowed")
+
+        await self.dispatch_message(msg)
+
     async def handle_get(self, request):
         method = request['method']
         taskhash = request['taskhash']
 
-        row = self.query_equivalent(method, taskhash)
+        if request.get('all', False):
+            row = self.query_equivalent(method, taskhash, self.ALL_QUERY)
+        else:
+            row = self.query_equivalent(method, taskhash, self.FAST_QUERY)
+
         if row is not None:
             logger.debug('Found equivalent task %s -> %s', (row['taskhash'], row['unihash']))
-            d = {k: row[k] for k in ('taskhash', 'method', 'unihash')}
+            d = {k: row[k] for k in row.keys()}
 
             self.write_message(d)
         else:
@@ -228,7 +268,7 @@ class ServerClient(object):
 
                 (method, taskhash) = l.split()
                 #logger.debug('Looking up %s %s' % (method, taskhash))
-                row = self.query_equivalent(method, taskhash)
+                row = self.query_equivalent(method, taskhash, self.FAST_QUERY)
                 if row is not None:
                     msg = ('%s\n' % row['unihash']).encode('utf-8')
                     #logger.debug('Found equivalent task %s -> %s', (row['taskhash'], row['unihash']))
@@ -328,7 +368,7 @@ class ServerClient(object):
             # Fetch the unihash that will be reported for the taskhash. If the
             # unihash matches, it means this row was inserted (or the mapping
             # was already valid)
-            row = self.query_equivalent(data['method'], data['taskhash'])
+            row = self.query_equivalent(data['method'], data['taskhash'], self.FAST_QUERY)
 
             if row['unihash'] == data['unihash']:
                 logger.info('Adding taskhash equivalence for %s with unihash %s',
@@ -354,12 +394,11 @@ class ServerClient(object):
         self.request_stats.reset()
         self.write_message(d)
 
-    def query_equivalent(self, method, taskhash):
+    def query_equivalent(self, method, taskhash, query):
         # This is part of the inner loop and must be as fast as possible
         try:
             cursor = self.db.cursor()
-            cursor.execute('SELECT taskhash, method, unihash FROM tasks_v2 WHERE method=:method AND taskhash=:taskhash ORDER BY created ASC LIMIT 1',
-                           {'method': method, 'taskhash': taskhash})
+            cursor.execute(query, {'method': method, 'taskhash': taskhash})
             return cursor.fetchone()
         except:
             cursor.close()

bitbake/lib/hashserv/tests.py

@@ -99,6 +99,29 @@ class TestHashEquivalenceServer(object):
         result = self.client.get_unihash(self.METHOD, taskhash)
         self.assertEqual(result, unihash)
 
+    def test_huge_message(self):
+        # Simple test that hashes can be created
+        taskhash = 'c665584ee6817aa99edfc77a44dd853828279370'
+        outhash = '3c979c3db45c569f51ab7626a4651074be3a9d11a84b1db076f5b14f7d39db44'
+        unihash = '90e9bc1d1f094c51824adca7f8ea79a048d68824'
+
+        result = self.client.get_unihash(self.METHOD, taskhash)
+        self.assertIsNone(result, msg='Found unexpected task, %r' % result)
+
+        siginfo = "0" * (self.client.max_chunk * 4)
+
+        result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash, {
+            'outhash_siginfo': siginfo
+        })
+        self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
+
+        result = self.client.get_taskhash(self.METHOD, taskhash, True)
+        self.assertEqual(result['taskhash'], taskhash)
+        self.assertEqual(result['unihash'], unihash)
+        self.assertEqual(result['method'], self.METHOD)
+        self.assertEqual(result['outhash'], outhash)
+        self.assertEqual(result['outhash_siginfo'], siginfo)
+
     def test_stress(self):
         def query_server(failures):
             client = Client(self.server.address)

documentation/bsp-guide/bsp.xml

@@ -2158,7 +2158,7 @@
                     <ulink url='&YOCTO_DOCS_REF_URL;#var-IMAGE_BOOT_FILES'><filename>IMAGE_BOOT_FILES</filename></ulink>:
                     Files installed into the device's boot partition
                     when preparing the image using the Wic tool
-                    with the <filename>bootimg-partition</filename>
+                    with the <filename>bootimg-partition</filename> or <filename>bootimg-efi</filename>
                     source plugin.
                     </para></listitem>
             </itemizedlist>

documentation/dev-manual/dev-manual-common-tasks.xml

@@ -3190,7 +3190,7 @@
                         building an image.
                         </para></listitem>
                     <listitem><para>
-                        <filename>virtual/mesa</filename>:
+                        <filename>virtual/libgbm</filename>:
                         Provides <filename>gbm.pc</filename>.
                         </para></listitem>
                     <listitem><para>

documentation/mega-manual/mega-manual-customization.xsl

@@ -1,4 +1,5 @@
 <?xml version='1.0'?>
+<!--SPDX-License-Identifier: CC-BY-2.0-UK-->
 <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns="http://www.w3.org/1999/xhtml" xmlns:fo="http://www.w3.org/1999/XSL/Format" version="1.0">
 
   <xsl:import href="http://downloads.yoctoproject.org/mirror/docbook-mirror/docbook-xsl-1.76.1/xhtml/docbook.xsl" />

documentation/mega-manual/mega-manual.xml

@@ -1,7 +1,7 @@
 <!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
 "http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"
 [<!ENTITY % poky SYSTEM "../poky.ent"> %poky; ] >
-
+<!--SPDX-License-Identifier: CC-BY-2.0-UK-->
 
 <book id='mega-manual' lang='en'
       xmlns:xi="http://www.w3.org/2003/XInclude"

documentation/mega-manual/mega-style.css

@@ -1,4 +1,6 @@
 /*
+   SPDX-License-Identifier: CC-BY-2.0-UK
+
    Generic XHTML / DocBook XHTML CSS Stylesheet.
 
    Browser wrangling and typographic design by

documentation/overview-manual/overview-manual-concepts.xml

@@ -1,6 +1,7 @@
 <!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
 "http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"
 [<!ENTITY % poky SYSTEM "../poky.ent"> %poky; ] >
+<!--SPDX-License-Identifier: CC-BY-2.0-UK-->
 
 <chapter id=' overview-manual-concepts'>
 <title>Yocto Project Concepts</title>

documentation/overview-manual/overview-manual-customization.xsl

@@ -1,4 +1,6 @@
 <?xml version='1.0'?>
+<!--SPDX-License-Identifier: CC-BY-2.0-UK-->
+
 <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns="http://www.w3.org/1999/xhtml" xmlns:fo="http://www.w3.org/1999/XSL/Format" version="1.0">
 
   <xsl:import href="http://downloads.yoctoproject.org/mirror/docbook-mirror/docbook-xsl-1.76.1/xhtml/docbook.xsl" />

documentation/overview-manual/overview-manual-development-environment.xml

@@ -1,6 +1,7 @@
 <!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
 "http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"
 [<!ENTITY % poky SYSTEM "../poky.ent"> %poky; ] >
+<!--SPDX-License-Identifier: CC-BY-2.0-UK-->
 
 <chapter id='overview-development-environment'>
 <title>The Yocto Project Development Environment</title>

documentation/overview-manual/overview-manual-intro.xml

@@ -1,6 +1,7 @@
 <!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
 "http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"
 [<!ENTITY % poky SYSTEM "../poky.ent"> %poky; ] >
+<!--SPDX-License-Identifier: CC-BY-2.0-UK-->
 
 <chapter id='overview-manual-intro'>
 

documentation/overview-manual/overview-manual-style.css

@@ -1,4 +1,6 @@
 /*
+   SPDX-License-Identifier: CC-BY-2.0-UK
+
    Generic XHTML / DocBook XHTML CSS Stylesheet.
 
    Browser wrangling and typographic design by

documentation/overview-manual/overview-manual-yp-intro.xml

@@ -1,6 +1,7 @@
 <!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
 "http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"
 [<!ENTITY % poky SYSTEM "../poky.ent"> %poky; ] >
+<!--SPDX-License-Identifier: CC-BY-2.0-UK-->
 
 <chapter id='overview-yp'>
     <title>Introducing the Yocto Project</title>

documentation/overview-manual/overview-manual.xml

@@ -1,6 +1,7 @@
 <!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
 "http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"
 [<!ENTITY % poky SYSTEM "../poky.ent"> %poky; ] >
+<!--SPDX-License-Identifier: CC-BY-2.0-UK-->
 
 <book id='overview-manual' lang='en'
       xmlns:xi="http://www.w3.org/2003/XInclude"

documentation/ref-manual/faq.xml

@@ -1,6 +1,7 @@
 <!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
 "http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"
 [<!ENTITY % poky SYSTEM "../poky.ent"> %poky; ] >
+<!--SPDX-License-Identifier: CC-BY-2.0-UK-->
 
 <chapter id='faq'>
 <title>FAQ</title>

documentation/ref-manual/migration.xml

@@ -1,6 +1,7 @@
 <!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
 "http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"
 [<!ENTITY % poky SYSTEM "../poky.ent"> %poky; ] >
+<!--SPDX-License-Identifier: CC-BY-2.0-UK-->
 
 <chapter id='migration'>
 <title>Migrating to a Newer Yocto Project Release</title>

documentation/ref-manual/ref-classes.xml

@@ -1,6 +1,7 @@
 <!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
 "http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"
 [<!ENTITY % poky SYSTEM "../poky.ent"> %poky; ] >
+<!--SPDX-License-Identifier: CC-BY-2.0-UK-->
 
 <chapter id='ref-classes'>
 <title>Classes</title>
@@ -1879,8 +1880,82 @@ This check was removed for YP 2.3 release
 
     <para>
         The <filename>kernel-fitimage</filename> class provides support to
-        pack zImages.
+        pack a kernel Image, device trees and a RAM disk into a single
+        FIT image. In theory, a FIT image can support any number of kernels,
+        RAM disks and device-trees.
+        However, <filename>kernel-fitimage</filename> currently only supports
+        limited usescases: just one kernel image, an optional RAM disk, and
+        any number of device tree.
     </para>
+
+    <para>
+        To create a FIT image, it is required that
+        <filename><link linkend='var-KERNEL_CLASSES'>KERNEL_CLASSES</link></filename>
+        is set to "kernel-fitimage" and
+        <filename><link linkend='var-KERNEL_IMAGETYPE'>KERNEL_IMAGETYPE</link></filename>
+        is set to "fitImage".
+    </para>
+
+    <para>
+        The options for the device tree compiler passed to mkimage -D feature
+        when creating the FIT image are specified using the
+        <filename><link linkend='var-UBOOT_MKIMAGE_DTCOPTS'>UBOOT_MKIMAGE_DTCOPTS</link></filename>
+        variable.
+    </para>
+
+    <para>
+        Only a single kernel can be added to the FIT image created by
+        <filename>kernel-fitimage</filename> and the kernel image in FIT is
+        mandatory.
+        The address where the kernel image is to be loaded by U-boot is
+        specified by
+        <filename><link linkend='var-UBOOT_LOADADDRESS'>UBOOT_LOADADDRESS</link></filename>
+        and the entrypoint by
+        <filename><link linkend='var-UBOOT_ENTRYPOINT'>UBOOT_ENTRYPOINT</link></filename>.
+    </para>
+
+    <para>
+        Multiple device trees can be added to the FIT image created by
+        <filename>kernel-fitimage</filename> and the device tree is optional.
+        The address where the device tree is to be loaded by U-boot is
+        specified by
+        <filename><link linkend='var-UBOOT_DTBO_LOADADDRESS'>UBOOT_DTBO_LOADADDRESS</link></filename>
+        for device tree overlays and by
+        <filename><link linkend='var-UBOOT_DTB_LOADADDRESS'>UBOOT_DTB_LOADADDRESS</link></filename>
+        for device tree binaries.
+    </para>
+
+    <para>
+        Only a single RAM disk can be added to the FIT image created by
+        <filename>kernel-fitimage</filename> and the RAM disk in FIT is
+        optional.
+        The address where the RAM disk image is to be loaded by U-boot
+        is specified by
+        <filename><link linkend='var-UBOOT_RD_LOADADDRESS'>UBOOT_RD_LOADADDRESS</link></filename>
+        and the entrypoint by
+        <filename><link linkend='var-UBOOT_RD_ENTRYPOINT'>UBOOT_RD_ENTRYPOINT</link></filename>.
+        The ramdisk is added to FIT image when
+        <filename><link linkend='var-INITRAMFS_IMAGE'>INITRAMFS_IMAGE</link></filename>
+        is specified.
+    </para>
+
+    <para>
+        The FIT image generated by <filename>kernel-fitimage</filename> class
+        is signed when the variables
+        <filename><link linkend='var-UBOOT_SIGN_ENABLE'>UBOOT_SIGN_ENABLE</link></filename>,
+        <filename><link linkend='var-UBOOT_MKIMAGE_DTCOPTS'>UBOOT_MKIMAGE_DTCOPTS</link></filename>,
+        <filename><link linkend='var-UBOOT_SIGN_KEYDIR'>UBOOT_SIGN_KEYDIR</link></filename>
+        and
+        <filename><link linkend='var-UBOOT_SIGN_KEYNAME'>UBOOT_SIGN_KEYNAME</link></filename>
+        are set appropriately.
+        The default values used for
+        <filename><link linkend='var-FIT_HASH_ALG'>FIT_HASH_ALG</link></filename>
+        and
+        <filename><link linkend='var-FIT_SIGN_ALG'>FIT_SIGN_ALG</link></filename>
+        in <filename>kernel-fitimage</filename> are "sha256" and "rsa2048"
+        respectively.
+    </para>
+
 </section>
 
 <section id='ref-classes-kernel-grub'>

documentation/ref-manual/ref-devtool-reference.xml

@@ -1,6 +1,7 @@
 <!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
 "http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"
 [<!ENTITY % poky SYSTEM "../poky.ent"> %poky; ] >
+<!--SPDX-License-Identifier: CC-BY-2.0-UK-->
 
 <chapter id='ref-devtool-reference'>
     <title><filename>devtool</filename> Quick Reference</title>

documentation/ref-manual/ref-features.xml

@@ -1,6 +1,7 @@
 <!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
 "http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"
 [<!ENTITY % poky SYSTEM "../poky.ent"> %poky; ] >
+<!--SPDX-License-Identifier: CC-BY-2.0-UK-->
 
 <chapter id='ref-features'>
     <title>Features</title>

documentation/ref-manual/ref-images.xml

@@ -1,6 +1,7 @@
 <!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
 "http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"
 [<!ENTITY % poky SYSTEM "../poky.ent"> %poky; ] >
+<!--SPDX-License-Identifier: CC-BY-2.0-UK-->
 
 <chapter id='ref-images'>
     <title>Images</title>

documentation/ref-manual/ref-kickstart.xml

@@ -1,6 +1,7 @@
 <!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
 "http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"
 [<!ENTITY % poky SYSTEM "../poky.ent"> %poky; ] >
+<!--SPDX-License-Identifier: CC-BY-2.0-UK-->
 
 <chapter id='ref-kickstart'>
 <title>OpenEmbedded Kickstart (<filename>.wks</filename>) Reference</title>

documentation/ref-manual/ref-manual-customization.xsl

@@ -1,4 +1,6 @@
 <?xml version='1.0'?>
+<!--SPDX-License-Identifier: CC-BY-2.0-UK-->
+
 <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns="http://www.w3.org/1999/xhtml" xmlns:fo="http://www.w3.org/1999/XSL/Format" version="1.0">
 
   <xsl:import href="http://downloads.yoctoproject.org/mirror/docbook-mirror/docbook-xsl-1.76.1/xhtml/docbook.xsl" />

documentation/ref-manual/ref-qa-checks.xml

@@ -1,6 +1,7 @@
 <!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
 "http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"
 [<!ENTITY % poky SYSTEM "../poky.ent"> %poky; ] >
+<!--SPDX-License-Identifier: CC-BY-2.0-UK-->
 
 <chapter id='ref-qa-checks'>
 <title>QA Error and Warning Messages</title>

documentation/ref-manual/ref-release-process.xml

@@ -1,6 +1,7 @@
 <!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
 "http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"
 [<!ENTITY % poky SYSTEM "../poky.ent"> %poky; ] >
+<!--SPDX-License-Identifier: CC-BY-2.0-UK-->
 
 <chapter id='ref-release-process'>
 <title>Yocto Project Releases and the Stable Release Process</title>

documentation/ref-manual/ref-structure.xml

@@ -1,6 +1,7 @@
 <!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
 "http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"
 [<!ENTITY % poky SYSTEM "../poky.ent"> %poky; ] >
+<!--SPDX-License-Identifier: CC-BY-2.0-UK-->
 
 <chapter id='ref-structure'>
 

documentation/ref-manual/ref-style.css

@@ -1,4 +1,7 @@
 /*
+
+   SPDX-License-Identifier: CC-BY-2.0-UK
+
    Generic XHTML / DocBook XHTML CSS Stylesheet.
 
    Browser wrangling and typographic design by

documentation/ref-manual/ref-system-requirements.xml

@@ -1,6 +1,7 @@
 <!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
 "http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"
 [<!ENTITY % poky SYSTEM "../poky.ent"> %poky; ] >
+<!--SPDX-License-Identifier: CC-BY-2.0-UK-->
 
 <chapter id='ref-manual-system-requirements'>
 <title>System Requirements</title>
@@ -93,11 +94,12 @@
             <itemizedlist>
                 <listitem><para>Ubuntu 16.04 (LTS)</para></listitem>
                 <listitem><para>Ubuntu 18.04 (LTS)</para></listitem>
-                <listitem><para>Ubuntu 19.04</para></listitem>
-                <listitem><para>Fedora 28</para></listitem>
-                <listitem><para>Fedora 29</para></listitem>
+                <listitem><para>Ubuntu 20.04</para></listitem>
                 <listitem><para>Fedora 30</para></listitem>
+                <listitem><para>Fedora 31</para></listitem>
+                <listitem><para>Fedora 32</para></listitem>
                 <listitem><para>CentOS 7.x</para></listitem>
+                <listitem><para>CentOS 8.x</para></listitem>
                 <listitem><para>Debian GNU/Linux 8.x (Jessie)</para></listitem>
                 <listitem><para>Debian GNU/Linux 9.x (Stretch)</para></listitem>
                 <listitem><para>Debian GNU/Linux 10.x (Buster)</para></listitem>

documentation/ref-manual/ref-tasks.xml

@@ -1,6 +1,7 @@
 <!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
 "http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"
 [<!ENTITY % poky SYSTEM "../poky.ent"> %poky; ] >
+<!--SPDX-License-Identifier: CC-BY-2.0-UK-->
 
 <chapter id='ref-tasks'>
 <title>Tasks</title>

documentation/ref-manual/ref-terms.xml

@@ -1,6 +1,7 @@
 <!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
 "http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"
 [<!ENTITY % poky SYSTEM "../poky.ent"> %poky; ] >
+<!--SPDX-License-Identifier: CC-BY-2.0-UK-->
 
 <chapter id='ref-terms'>
 <title>Yocto Project Terms</title>

documentation/ref-manual/ref-variables.xml

@@ -1,6 +1,7 @@
 <!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
 "http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"
 [<!ENTITY % poky SYSTEM "../poky.ent"> %poky; ] >
+<!--SPDX-License-Identifier: CC-BY-2.0-UK-->
 
 <!-- Dummy chapter -->
 <chapter id='ref-variables-glos'>
@@ -4990,6 +4991,30 @@
             </glossdef>
         </glossentry>
 
+        <glossentry id='var-FIT_HASH_ALG'><glossterm>FIT_HASH_ALG</glossterm>
+            <info>
+               FIT_HASH_ALG[doc] = "Specifies the hash algorithm used in creating the FIT Image."
+            </info>
+            <glossdef>
+                <para role="glossdeffirst">
+                    Specifies the hash algorithm used in creating the FIT Image.
+                    For e.g. sha256.
+                </para>
+            </glossdef>
+        </glossentry>
+
+        <glossentry id='var-FIT_SIGN_ALG'><glossterm>FIT_SIGN_ALG</glossterm>
+            <info>
+               FIT_SIGN_ALG[doc] = "Specifies the signature algorithm used in creating the FIT Image."
+            </info>
+            <glossdef>
+                <para role="glossdeffirst">
+                    Specifies the signature algorithm used in creating the FIT Image.
+                    For e.g. rsa2048.
+                </para>
+            </glossdef>
+        </glossentry>
+
         <glossentry id='var-FONT_EXTRA_RDEPENDS'><glossterm>FONT_EXTRA_RDEPENDS</glossterm>
             <info>
                 FONT_EXTRA_RDEPENDS[doc] = "When a recipe inherits the fontcache class, this variable specifies runtime dependencies for font packages. This variable defaults to 'fontconfig-utils'."
@@ -5702,7 +5727,7 @@
                 <para role="glossdeffirst">
                     A space-separated list of files installed into the
                     boot partition when preparing an image using the Wic tool
-                    with the <filename>bootimg-partition</filename> source
+                    with the <filename>bootimg-partition</filename> or <filename>bootimg-efi</filename> source
                     plugin.
                     By default, the files are installed under the same name as
                     the source files.
@@ -15959,6 +15984,38 @@
             </glossdef>
         </glossentry>
 
+        <glossentry id='var-UBOOT_DTB_LOADADDRESS'><glossterm>UBOOT_DTB_LOADADDRESS</glossterm>
+            <info>
+               UBOOT_DTB_LOADADDRESS[doc] = "Specifies the load address for the dtb."
+            </info>
+            <glossdef>
+                <para role="glossdeffirst">
+                    Specifies the load address for the dtb image used by U-boot.
+                    During FIT image creation, the
+                    <filename>UBOOT_DTB_LOADADDRESS</filename> variable is used
+                    in <filename>kernel-fitimage</filename> class to specify the
+                    load address to be used in creating the dtb sections of
+                    Image Tree Source for the FIT image.
+                </para>
+            </glossdef>
+        </glossentry>
+
+        <glossentry id='var-UBOOT_DTBO_LOADADDRESS'><glossterm>UBOOT_DTBO_LOADADDRESS</glossterm>
+            <info>
+               UBOOT_DTBO_LOADADDRESS[doc] = "Specifies the load address for the dtbo."
+            </info>
+            <glossdef>
+                <para role="glossdeffirst">
+                    Specifies the load address for the dtbo image used by U-boot.
+                    During FIT image creation, the
+                    <filename>UBOOT_DTBO_LOADADDRESS</filename> variable is used
+                    in <filename>kernel-fitimage</filename> class to specify the
+                    load address to be used in creating the dtbo sections of
+                    Image Tree Source for the FIT image.
+                </para>
+            </glossdef>
+        </glossentry>
+
         <glossentry id='var-UBOOT_ENTRYPOINT'><glossterm>UBOOT_ENTRYPOINT</glossterm>
             <info>
                UBOOT_ENTRYPOINT[doc] = "Specifies the entry point for the U-Boot image."
@@ -16044,6 +16101,51 @@
             </glossdef>
         </glossentry>
 
+        <glossentry id='var-UBOOT_MKIMAGE_DTCOPTS'><glossterm>UBOOT_MKIMAGE_DTCOPTS</glossterm>
+            <info>
+               UBOOT_MKIMAGE_DTCOPTS[doc] = "Options for the device tree compiler passed to mkimage '-D' feature."
+            </info>
+            <glossdef>
+                <para role="glossdeffirst">
+                    Options for the device tree compiler passed to mkimage '-D'
+                    feature while creating FIT image in
+                    <filename>kernel-fitimage</filename> class.
+                </para>
+            </glossdef>
+        </glossentry>
+
+        <glossentry id='var-UBOOT_RD_LOADADDRESS'><glossterm>UBOOT_RD_LOADADDRESS</glossterm>
+            <info>
+               UBOOT_RD_LOADADDRESS[doc] = "Specifies the load address for the ramdisk image."
+            </info>
+            <glossdef>
+                <para role="glossdeffirst">
+                    Specifies the load address for the RAM disk image.
+                    During FIT image creation, the
+                    <filename>UBOOT_RD_LOADADDRESS</filename> variable is used
+                    in <filename>kernel-fitimage</filename> class to specify the
+                    load address to be used in creating the Image Tree Source for
+                    the FIT image.
+                </para>
+            </glossdef>
+        </glossentry>
+
+        <glossentry id='var-UBOOT_RD_ENTRYPOINT'><glossterm>UBOOT_RD_ENTRYPOINT</glossterm>
+            <info>
+               UBOOT_RD_ENTRYPOINT[doc] = "Specifies the entrypoint for the ramdisk image."
+            </info>
+            <glossdef>
+                <para role="glossdeffirst">
+                    Specifies the entrypoint for the RAM disk image.
+                    During FIT image creation, the
+                    <filename>UBOOT_RD_ENTRYPOINT</filename> variable is used
+                    in <filename>kernel-fitimage</filename> class to specify the
+                    entrypoint to be used in creating the Image Tree Source for
+                    the FIT image.
+                </para>
+            </glossdef>
+        </glossentry>
+
         <glossentry id='var-UBOOT_SUFFIX'><glossterm>UBOOT_SUFFIX</glossterm>
             <info>
                UBOOT_SUFFIX[doc] = "Points to the generated U-Boot extension."
@@ -16062,6 +16164,47 @@
             </glossdef>
         </glossentry>
 
+        <glossentry id='var-UBOOT_SIGN_ENABLE'><glossterm>UBOOT_SIGN_ENABLE</glossterm>
+            <info>
+               UBOOT_SIGN_KEYDIR[doc] = "Enable signing of FIT image."
+            </info>
+            <glossdef>
+                <para role="glossdeffirst">
+                    Enable signing of FIT image. The default value is "0".
+                </para>
+            </glossdef>
+        </glossentry>
+
+        <glossentry id='var-UBOOT_SIGN_KEYDIR'><glossterm>UBOOT_SIGN_KEYDIR</glossterm>
+            <info>
+               UBOOT_SIGN_KEYDIR[doc] = "Location of the directory containing the RSA key and certificate used for signing FIT image."
+            </info>
+            <glossdef>
+                <para role="glossdeffirst">
+                    Location of the directory containing the RSA key and
+                    certificate used for signing FIT image.
+                </para>
+            </glossdef>
+        </glossentry>
+
+        <glossentry id='var-UBOOT_SIGN_KEYNAME'><glossterm>UBOOT_SIGN_KEYNAME</glossterm>
+            <info>
+               UBOOT_SIGN_KEYNAME[doc] = "The name of keys used for signing U-boot FIT image"
+            </info>
+            <glossdef>
+                <para role="glossdeffirst">
+                    The name of keys used for signing U-boot FIT image stored in
+                    <filename><link linkend='var-UBOOT_SIGN_KEYDIR'>UBOOT_SIGN_KEYDIR</link></filename>
+                    directory. For e.g. dev.key key and dev.crt certificate
+                    stored in
+                    <filename><link linkend='var-UBOOT_SIGN_KEYDIR'>UBOOT_SIGN_KEYDIR</link></filename>
+                    directory will have
+                    <filename><link linkend='var-UBOOT_SIGN_KEYNAME'>UBOOT_SIGN_KEYNAME</link></filename>
+                    set to "dev".
+                </para>
+            </glossdef>
+        </glossentry>
+
         <glossentry id='var-UBOOT_TARGET'><glossterm>UBOOT_TARGET</glossterm>
             <info>
                UBOOT_TARGET[doc] = "Specifies the target used for building U-Boot."

documentation/ref-manual/ref-varlocality.xml

@@ -1,6 +1,7 @@
 <!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
 "http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"
 [<!ENTITY % poky SYSTEM "../poky.ent"> %poky; ] >
+<!--SPDX-License-Identifier: CC-BY-2.0-UK-->
 
 <chapter id='ref-varlocality'>
     <title>Variable Context</title>

documentation/ref-manual/resources.xml

@@ -1,6 +1,7 @@
 <!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
 "http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"
 [<!ENTITY % poky SYSTEM "../poky.ent"> %poky; ] >
+<!--SPDX-License-Identifier: CC-BY-2.0-UK-->
 
 <chapter id='resources'>
 <title>Contributions and Additional Information</title>

meta-skeleton/recipes-baremetal/baremetal-examples/baremetal-helloworld_git.bb

@@ -10,10 +10,19 @@ SRC_URI = "git://github.com/aehs29/baremetal-helloqemu.git;protocol=https;branch
 
 S = "${WORKDIR}/git/"
 
-# These examples are not meant to be built when using either musl or glibc
-COMPATIBLE_HOST_libc-musl_class-target = "null"
-COMPATIBLE_HOST_libc-glibc_class-target = "null"
+# The following variables should be set to accomodate each application
+BAREMETAL_BINNAME ?= "hello_baremetal_${MACHINE}"
+IMAGE_LINK_NAME ?= "baremetal-helloworld-image-${MACHINE}"
+IMAGE_NAME_SUFFIX ?= ""
 
+# Baremetal-Image creates the proper wiring, assumes the output is provided in
+# binary and ELF format, installed on ${base_libdir}/firmware/ , we want a
+# package to be created since we might have some way of updating the baremetal
+# firmware from Linux
+inherit baremetal-image
+
+
+# These parameters are app specific for this example
 # This will be translated automatically to the architecture and
 # machine that QEMU uses on OE, e.g. -machine virt -cpu cortex-a57
 # but the examples can also be run on other architectures/machines
@@ -25,82 +34,17 @@ BAREMETAL_QEMUARCH_qemuarmv5 = "versatile"
 BAREMETAL_QEMUARCH_qemuarm = "arm"
 BAREMETAL_QEMUARCH_qemuarm64 = "aarch64"
 
-
 EXTRA_OEMAKE_append = " QEMUARCH=${BAREMETAL_QEMUARCH} V=1"
 
+
+# Install binaries on the proper location for baremetal-image to fetch and deploy
 do_install(){
-    install -d ${D}/${datadir}
-    install -m 755 ${B}build/hello_baremetal_${BAREMETAL_QEMUARCH}.bin ${D}/${datadir}/hello_baremetal_${MACHINE}.bin
-    install -m 755 ${B}build/hello_baremetal_${BAREMETAL_QEMUARCH}.elf ${D}/${datadir}/hello_baremetal_${MACHINE}.elf
-}
-
-# Borrowed from meta-freertos
-inherit rootfs-postcommands
-inherit deploy
-IMGDEPLOYDIR ?= "${WORKDIR}/deploy-${PN}-image-complete"
-do_deploy[dirs] = "${DEPLOYDIR} ${DEPLOY_DIR_IMAGE}"
-do_rootfs[dirs] = "${DEPLOYDIR} ${DEPLOY_DIR_IMAGE}"
-DEPLOYDIR = "${IMGDEPLOYDIR}"
-IMAGE_LINK_NAME ?= "baremetal-helloworld-image-${MACHINE}"
-IMAGE_NAME_SUFFIX ?= ""
-
-do_deploy(){
-    install ${D}/${datadir}/hello_baremetal_${MACHINE}.bin ${DEPLOYDIR}/${IMAGE_LINK_NAME}.bin
-    install ${D}/${datadir}/hello_baremetal_${MACHINE}.elf ${DEPLOYDIR}/${IMAGE_LINK_NAME}.elf
-}
-
-do_image(){
-    :
+    install -d ${D}/${base_libdir}/firmware
+    install -m 755 ${B}build/hello_baremetal_${BAREMETAL_QEMUARCH}.bin ${D}/${base_libdir}/firmware/${BAREMETAL_BINNAME}.bin
+    install -m 755 ${B}build/hello_baremetal_${BAREMETAL_QEMUARCH}.elf ${D}/${base_libdir}/firmware/${BAREMETAL_BINNAME}.elf
 }
 
 FILES_${PN} += " \
-    ${datadir}/hello_baremetal_${MACHINE}.bin \
-    ${datadir}/hello_baremetal_${MACHINE}.elf \
+    ${base_libdir}/firmware/${BAREMETAL_BINNAME}.bin \
+    ${base_libdir}/firmware/${BAREMETAL_BINNAME}.elf \
 "
-
-python do_rootfs(){
-    from oe.utils import execute_pre_post_process
-    from pathlib import Path
-
-    # Write empty manifest testdate file
-    deploy_dir = d.getVar('DEPLOYDIR')
-    link_name = d.getVar('IMAGE_LINK_NAME')
-    manifest_name = d.getVar('IMAGE_MANIFEST')
-
-    Path(manifest_name).touch()
-    if os.path.exists(manifest_name) and link_name:
-        manifest_link = deploy_dir + "/" + link_name + ".manifest"
-        if os.path.lexists(manifest_link):
-            os.remove(manifest_link)
-        os.symlink(os.path.basename(manifest_name), manifest_link)
-    execute_pre_post_process(d, d.getVar('ROOTFS_POSTPROCESS_COMMAND'))
-}
-
-# QEMU generic FreeRTOS parameters
-QB_DEFAULT_KERNEL = "${IMAGE_LINK_NAME}.bin"
-QB_MEM = "-m 256"
-QB_OPT_APPEND = "-nographic"
-QB_DEFAULT_FSTYPE = "bin"
-QB_DTB = ""
-
-# This next part is necessary to trick the build system into thinking
-# its building an image recipe so it generates the qemuboot.conf
-addtask do_deploy after do_write_qemuboot_conf before do_build
-addtask do_rootfs before do_deploy after do_install
-addtask do_image after do_rootfs before do_build
-inherit qemuboot
-
-# Based on image.bbclass to make sure we build qemu
-python(){
-    # do_addto_recipe_sysroot doesnt exist for all recipes, but we need it to have
-    # /usr/bin on recipe-sysroot (qemu) populated
-    def extraimage_getdepends(task):
-        deps = ""
-        for dep in (d.getVar('EXTRA_IMAGEDEPENDS') or "").split():
-        # Make sure we only add it for qemu
-            if 'qemu' in dep:
-                deps += " %s:%s" % (dep, task)
-        return deps
-    d.appendVarFlag('do_image', 'depends', extraimage_getdepends('do_addto_recipe_sysroot'))
-    d.appendVarFlag('do_image', 'depends', extraimage_getdepends('do_populate_sysroot'))
-}

meta-yocto-bsp/recipes-kernel/linux/linux-yocto_5.4.bbappend

@@ -7,17 +7,17 @@ KMACHINE_genericx86 ?= "common-pc"
 KMACHINE_genericx86-64 ?= "common-pc-64"
 KMACHINE_beaglebone-yocto ?= "beaglebone"
 
-SRCREV_machine_genericx86 ?= "29f44c85c379c38f15e544828e7e77b3c008f378"
-SRCREV_machine_genericx86-64 ?= "29f44c85c379c38f15e544828e7e77b3c008f378"
-SRCREV_machine_edgerouter ?= "f4d7dbafb103e4f782323017c239c548871c1567"
-SRCREV_machine_beaglebone-yocto ?= "f4d7dbafb103e4f782323017c239c548871c1567"
+SRCREV_machine_genericx86 ?= "9fc2fb2e73466a520ee9a3c48b3ca2f5b21415dc"
+SRCREV_machine_genericx86-64 ?= "9fc2fb2e73466a520ee9a3c48b3ca2f5b21415dc"
+SRCREV_machine_edgerouter ?= "9fc2fb2e73466a520ee9a3c48b3ca2f5b21415dc"
+SRCREV_machine_beaglebone-yocto ?= "9fc2fb2e73466a520ee9a3c48b3ca2f5b21415dc"
 
 COMPATIBLE_MACHINE_genericx86 = "genericx86"
 COMPATIBLE_MACHINE_genericx86-64 = "genericx86-64"
 COMPATIBLE_MACHINE_edgerouter = "edgerouter"
 COMPATIBLE_MACHINE_beaglebone-yocto = "beaglebone-yocto"
 
-LINUX_VERSION_genericx86 = "5.4.40"
-LINUX_VERSION_genericx86-64 = "5.4.40"
-LINUX_VERSION_edgerouter = "5.4.20"
-LINUX_VERSION_beaglebone-yocto = "5.4.20"
+LINUX_VERSION_genericx86 = "5.4.54"
+LINUX_VERSION_genericx86-64 = "5.4.54"
+LINUX_VERSION_edgerouter = "5.4.54"
+LINUX_VERSION_beaglebone-yocto = "5.4.54"

meta/classes/archiver.bbclass

@@ -582,8 +582,8 @@ do_deploy_archives[sstate-outputdirs] = "${DEPLOY_DIR_SRC}"
 addtask do_deploy_archives_setscene
 
 addtask do_ar_original after do_unpack
-addtask do_unpack_and_patch after do_patch
-addtask do_ar_patched after do_unpack_and_patch before do_preconfigure do_configure
+addtask do_unpack_and_patch after do_patch do_preconfigure
+addtask do_ar_patched after do_unpack_and_patch
 addtask do_ar_configured after do_unpack_and_patch
 addtask do_ar_mirror after do_fetch
 addtask do_dumpdata

meta/classes/autotools.bbclass

@@ -5,7 +5,7 @@ def autotools_dep_prepend(d):
     pn = d.getVar('PN')
     deps = ''
 
-    if pn in ['autoconf-native', 'automake-native', 'help2man-native']:
+    if pn in ['autoconf-native', 'automake-native']:
         return deps
     deps += 'autoconf-native automake-native '
 

meta/classes/baremetal-image.bbclass

@@ -0,0 +1,99 @@
+# Baremetal image class
+#
+# This class is meant to be inherited by recipes for baremetal/RTOS applications
+# It contains code that would be used by all of them, every recipe just needs to
+# override certain variables.
+#
+# For scalability purposes, code within this class focuses on the "image" wiring
+# to satisfy the OpenEmbedded image creation and testing infrastructure.
+#
+# See meta-skeleton for a working example.
+
+
+# Toolchain should be baremetal or newlib based.
+# TCLIBC="baremetal" or TCLIBC="newlib"
+COMPATIBLE_HOST_libc-musl_class-target = "null"
+COMPATIBLE_HOST_libc-glibc_class-target = "null"
+
+
+inherit rootfs-postcommands
+
+# Set some defaults, but these should be overriden by each recipe if required
+IMGDEPLOYDIR ?= "${WORKDIR}/deploy-${PN}-image-complete"
+BAREMETAL_BINNAME ?= "hello_baremetal_${MACHINE}"
+IMAGE_LINK_NAME ?= "baremetal-helloworld-image-${MACHINE}"
+IMAGE_NAME_SUFFIX ?= ""
+
+do_rootfs[dirs] = "${IMGDEPLOYDIR} ${DEPLOY_DIR_IMAGE}"
+
+do_image(){
+    install ${D}/${base_libdir}/firmware/${BAREMETAL_BINNAME}.bin ${IMGDEPLOYDIR}/${IMAGE_LINK_NAME}.bin
+    install ${D}/${base_libdir}/firmware/${BAREMETAL_BINNAME}.elf ${IMGDEPLOYDIR}/${IMAGE_LINK_NAME}.elf
+}
+
+do_image_complete(){
+    :
+}
+
+python do_rootfs(){
+    from oe.utils import execute_pre_post_process
+    from pathlib import Path
+
+    # Write empty manifest file to satisfy test infrastructure
+    deploy_dir = d.getVar('IMGDEPLOYDIR')
+    link_name = d.getVar('IMAGE_LINK_NAME')
+    manifest_name = d.getVar('IMAGE_MANIFEST')
+
+    Path(manifest_name).touch()
+    if os.path.exists(manifest_name) and link_name:
+        manifest_link = deploy_dir + "/" + link_name + ".manifest"
+        if os.path.lexists(manifest_link):
+            os.remove(manifest_link)
+        os.symlink(os.path.basename(manifest_name), manifest_link)
+    execute_pre_post_process(d, d.getVar('ROOTFS_POSTPROCESS_COMMAND'))
+}
+
+
+# Assure binaries, manifest and qemubootconf are populated on DEPLOY_DIR_IMAGE
+do_image_complete[dirs] = "${TOPDIR}"
+do_image_complete[umask] = "022"
+SSTATETASKS += "do_image_complete"
+SSTATE_SKIP_CREATION_task-image-complete = '1'
+do_image_complete[sstate-inputdirs] = "${IMGDEPLOYDIR}"
+do_image_complete[sstate-outputdirs] = "${DEPLOY_DIR_IMAGE}"
+do_image_complete[stamp-extra-info] = "${MACHINE_ARCH}"
+addtask do_image_complete after do_image before do_build
+
+python do_image_complete_setscene () {
+    sstate_setscene(d)
+}
+addtask do_image_complete_setscene
+
+# QEMU generic Baremetal/RTOS parameters
+QB_DEFAULT_KERNEL ?= "${IMAGE_LINK_NAME}.bin"
+QB_MEM ?= "-m 256"
+QB_DEFAULT_FSTYPE ?= "bin"
+QB_DTB ?= ""
+QB_OPT_APPEND = "-nographic"
+
+# This next part is necessary to trick the build system into thinking
+# its building an image recipe so it generates the qemuboot.conf
+addtask do_rootfs before do_image after do_install
+addtask do_image after do_rootfs before do_image_complete
+addtask do_image_complete after do_image before do_build
+inherit qemuboot
+
+# Based on image.bbclass to make sure we build qemu
+python(){
+    # do_addto_recipe_sysroot doesnt exist for all recipes, but we need it to have
+    # /usr/bin on recipe-sysroot (qemu) populated
+    def extraimage_getdepends(task):
+        deps = ""
+        for dep in (d.getVar('EXTRA_IMAGEDEPENDS') or "").split():
+        # Make sure we only add it for qemu
+            if 'qemu' in dep:
+                deps += " %s:%s" % (dep, task)
+        return deps
+    d.appendVarFlag('do_image', 'depends', extraimage_getdepends('do_addto_recipe_sysroot'))
+    d.appendVarFlag('do_image', 'depends', extraimage_getdepends('do_populate_sysroot'))
+}

meta/classes/buildhistory.bbclass

@@ -429,8 +429,8 @@ def buildhistory_list_installed(d, rootfs_type="image"):
     from oe.sdk import sdk_list_installed_packages
     from oe.utils import format_pkg_list
 
-    process_list = [('file', 'bh_installed_pkgs.txt'),\
-                    ('deps', 'bh_installed_pkgs_deps.txt')]
+    process_list = [('file', 'bh_installed_pkgs_%s.txt' % os.getpid()),\
+                    ('deps', 'bh_installed_pkgs_deps_%s.txt' % os.getpid())]
 
     if rootfs_type == "image":
         pkgs = image_list_installed_packages(d)
@@ -460,9 +460,10 @@ buildhistory_get_installed() {
 
 	# Get list of installed packages
 	pkgcache="$1/installed-packages.tmp"
-	cat ${WORKDIR}/bh_installed_pkgs.txt | sort > $pkgcache && rm ${WORKDIR}/bh_installed_pkgs.txt
+	cat ${WORKDIR}/bh_installed_pkgs_${PID}.txt | sort > $pkgcache && rm ${WORKDIR}/bh_installed_pkgs_${PID}.txt
 
 	cat $pkgcache | awk '{ print $1 }' > $1/installed-package-names.txt
+
 	if [ -s $pkgcache ] ; then
 		cat $pkgcache | awk '{ print $2 }' | xargs -n1 basename > $1/installed-packages.txt
 	else
@@ -471,8 +472,8 @@ buildhistory_get_installed() {
 
 	# Produce dependency graph
 	# First, quote each name to handle characters that cause issues for dot
-	sed 's:\([^| ]*\):"\1":g' ${WORKDIR}/bh_installed_pkgs_deps.txt > $1/depends.tmp &&
-		rm ${WORKDIR}/bh_installed_pkgs_deps.txt
+	sed 's:\([^| ]*\):"\1":g' ${WORKDIR}/bh_installed_pkgs_deps_${PID}.txt > $1/depends.tmp &&
+		rm ${WORKDIR}/bh_installed_pkgs_deps_${PID}.txt
 	# Remove lines with rpmlib(...) and config(...) dependencies, change the
 	# delimiter from pipe to "->", set the style for recommend lines and
 	# turn versioned dependencies into edge labels.

meta/classes/cmake.bbclass

@@ -21,23 +21,6 @@ python() {
         d.setVarFlag("do_compile", "progress", r"outof:^\[(\d+)/(\d+)\]\s+")
     else:
         bb.fatal("Unknown CMake Generator %s" % generator)
-
-    # C/C++ Compiler (without cpu arch/tune arguments)
-    if not d.getVar('OECMAKE_C_COMPILER'):
-        cc_list = d.getVar('CC').split()
-        if cc_list[0] == 'ccache':
-            d.setVar('OECMAKE_C_COMPILER_LAUNCHER', cc_list[0])
-            d.setVar('OECMAKE_C_COMPILER', cc_list[1])
-        else:
-            d.setVar('OECMAKE_C_COMPILER', cc_list[0])
-
-    if not d.getVar('OECMAKE_CXX_COMPILER'):
-        cxx_list = d.getVar('CXX').split()
-        if cxx_list[0] == 'ccache':
-            d.setVar('OECMAKE_CXX_COMPILER_LAUNCHER', cxx_list[0])
-            d.setVar('OECMAKE_CXX_COMPILER', cxx_list[1])
-        else:
-            d.setVar('OECMAKE_CXX_COMPILER', cxx_list[0])
 }
 OECMAKE_AR ?= "${AR}"
 
@@ -51,8 +34,23 @@ OECMAKE_CXX_LINK_FLAGS ?= "${HOST_CC_ARCH} ${TOOLCHAIN_OPTIONS} ${CXXFLAGS} ${LD
 CXXFLAGS += "${HOST_CC_ARCH} ${TOOLCHAIN_OPTIONS}"
 CFLAGS += "${HOST_CC_ARCH} ${TOOLCHAIN_OPTIONS}"
 
-OECMAKE_C_COMPILER_LAUNCHER ?= ""
-OECMAKE_CXX_COMPILER_LAUNCHER ?= ""
+def oecmake_map_compiler(compiler, d):
+    args = d.getVar(compiler).split()
+    if args[0] == "ccache":
+        return args[1], args[0]
+    return args[0], ""
+
+# C/C++ Compiler (without cpu arch/tune arguments)
+OECMAKE_C_COMPILER ?= "${@oecmake_map_compiler('CC', d)[0]}"
+OECMAKE_C_COMPILER_LAUNCHER ?= "${@oecmake_map_compiler('CC', d)[1]}"
+OECMAKE_CXX_COMPILER ?= "${@oecmake_map_compiler('CXX', d)[0]}"
+OECMAKE_CXX_COMPILER_LAUNCHER ?= "${@oecmake_map_compiler('CXX', d)[1]}"
+
+# clear compiler vars for allarch to avoid sig hash difference
+OECMAKE_C_COMPILER_allarch = ""
+OECMAKE_C_COMPILER_LAUNCHER_allarch = ""
+OECMAKE_CXX_COMPILER_allarch = ""
+OECMAKE_CXX_COMPILER_LAUNCHER_allarch = ""
 
 OECMAKE_RPATH ?= ""
 OECMAKE_PERLNATIVE_DIR ??= ""
@@ -70,15 +68,22 @@ CMAKE_BUILD_PARALLEL_LEVEL_task-install = "${@oe.utils.parallel_make(d, True)}"
 OECMAKE_TARGET_COMPILE ?= "all"
 OECMAKE_TARGET_INSTALL ?= "install"
 
+def map_host_os_to_system_name(host_os):
+    if host_os.startswith('mingw'):
+        return 'Windows'
+    if host_os.startswith('linux'):
+        return 'Linux'
+    return host_os
+
 # CMake expects target architectures in the format of uname(2),
 # which do not always match TARGET_ARCH, so all the necessary
 # conversions should happen here.
-def map_target_arch_to_uname_arch(target_arch):
-    if target_arch == "powerpc":
+def map_host_arch_to_uname_arch(host_arch):
+    if host_arch == "powerpc":
         return "ppc"
-    if target_arch == "powerpc64":
+    if host_arch == "powerpc64":
         return "ppc64"
-    return target_arch
+    return host_arch
 
 cmake_do_generate_toolchain_file() {
 	if [ "${BUILD_SYS}" = "${HOST_SYS}" ]; then
@@ -88,8 +93,8 @@ cmake_do_generate_toolchain_file() {
 # CMake system name must be something like "Linux".
 # This is important for cross-compiling.
 $cmake_crosscompiling
-set( CMAKE_SYSTEM_NAME `echo ${TARGET_OS} | sed -e 's/^./\u&/' -e 's/^\(Linux\).*/\1/'` )
-set( CMAKE_SYSTEM_PROCESSOR ${@map_target_arch_to_uname_arch(d.getVar('TARGET_ARCH'))} )
+set( CMAKE_SYSTEM_NAME ${@map_host_os_to_system_name(d.getVar('HOST_OS'))} )
+set( CMAKE_SYSTEM_PROCESSOR ${@map_host_arch_to_uname_arch(d.getVar('HOST_ARCH'))} )
 set( CMAKE_C_COMPILER ${OECMAKE_C_COMPILER} )
 set( CMAKE_CXX_COMPILER ${OECMAKE_CXX_COMPILER} )
 set( CMAKE_C_COMPILER_LAUNCHER ${OECMAKE_C_COMPILER_LAUNCHER} )

meta/classes/cml1.bbclass

@@ -1,3 +1,13 @@
+# returns all the elements from the src uri that are .cfg files
+def find_cfgs(d):
+    sources=src_patches(d, True)
+    sources_list=[]
+    for s in sources:
+        if s.endswith('.cfg'):
+            sources_list.append(s)
+
+    return sources_list
+
 cml1_do_configure() {
 	set -e
 	unset CFLAGS CPPFLAGS CXXFLAGS LDFLAGS
@@ -17,12 +27,16 @@ CROSS_CURSES_INC = '-DCURSES_LOC="<curses.h>"'
 TERMINFO = "${STAGING_DATADIR_NATIVE}/terminfo"
 
 KCONFIG_CONFIG_COMMAND ??= "menuconfig"
+KCONFIG_CONFIG_ROOTDIR ??= "${B}"
 python do_menuconfig() {
     import shutil
 
+    config = os.path.join(d.getVar('KCONFIG_CONFIG_ROOTDIR'), ".config")
+    configorig = os.path.join(d.getVar('KCONFIG_CONFIG_ROOTDIR'), ".config.orig")
+
     try:
-        mtime = os.path.getmtime(".config")
-        shutil.copy(".config", ".config.orig")
+        mtime = os.path.getmtime(config)
+        shutil.copy(config, configorig)
     except OSError:
         mtime = 0
 
@@ -32,7 +46,7 @@ python do_menuconfig() {
     # FIXME this check can be removed when the minimum bitbake version has been bumped
     if hasattr(bb.build, 'write_taint'):
         try:
-            newmtime = os.path.getmtime(".config")
+            newmtime = os.path.getmtime(config)
         except OSError:
             newmtime = 0
 
@@ -42,7 +56,7 @@ python do_menuconfig() {
 }
 do_menuconfig[depends] += "ncurses-native:do_populate_sysroot"
 do_menuconfig[nostamp] = "1"
-do_menuconfig[dirs] = "${B}"
+do_menuconfig[dirs] = "${KCONFIG_CONFIG_ROOTDIR}"
 addtask menuconfig after do_configure
 
 python do_diffconfig() {
@@ -51,8 +65,8 @@ python do_diffconfig() {
 
     workdir = d.getVar('WORKDIR')
     fragment = workdir + '/fragment.cfg'
-    configorig = '.config.orig'
-    config = '.config'
+    configorig = os.path.join(d.getVar('KCONFIG_CONFIG_ROOTDIR'), ".config.orig")
+    config = os.path.join(d.getVar('KCONFIG_CONFIG_ROOTDIR'), ".config")
 
     try:
         md5newconfig = bb.utils.md5_file(configorig)
@@ -75,5 +89,5 @@ python do_diffconfig() {
 }
 
 do_diffconfig[nostamp] = "1"
-do_diffconfig[dirs] = "${B}"
+do_diffconfig[dirs] = "${KCONFIG_CONFIG_ROOTDIR}"
 addtask diffconfig

meta/classes/cve-check.bbclass

@@ -30,6 +30,9 @@ CVE_CHECK_DB_FILE ?= "${CVE_CHECK_DB_DIR}/nvdcve_1.1.db"
 
 CVE_CHECK_LOG ?= "${T}/cve.log"
 CVE_CHECK_TMP_FILE ?= "${TMPDIR}/cve_check"
+CVE_CHECK_SUMMARY_DIR ?= "${LOG_DIR}/cve"
+CVE_CHECK_SUMMARY_FILE_NAME ?= "cve-summary"
+CVE_CHECK_SUMMARY_FILE ?= "${CVE_CHECK_SUMMARY_DIR}/${CVE_CHECK_SUMMARY_FILE_NAME}"
 
 CVE_CHECK_DIR ??= "${DEPLOY_DIR}/cve"
 CVE_CHECK_MANIFEST ?= "${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.cve"
@@ -46,6 +49,32 @@ CVE_CHECK_PN_WHITELIST ?= ""
 # 
 CVE_CHECK_WHITELIST ?= ""
 
+python cve_save_summary_handler () {
+    import shutil
+    import datetime
+
+    cve_tmp_file = d.getVar("CVE_CHECK_TMP_FILE")
+
+    cve_summary_name = d.getVar("CVE_CHECK_SUMMARY_FILE_NAME")
+    cvelogpath = d.getVar("CVE_CHECK_SUMMARY_DIR")
+    bb.utils.mkdirhier(cvelogpath)
+
+    timestamp = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
+    cve_summary_file = os.path.join(cvelogpath, "%s-%s.txt" % (cve_summary_name, timestamp))
+
+    shutil.copyfile(cve_tmp_file, cve_summary_file)
+
+    if cve_summary_file and os.path.exists(cve_summary_file):
+        cvefile_link = os.path.join(cvelogpath, cve_summary_name)
+
+        if os.path.exists(os.path.realpath(cvefile_link)):
+            os.remove(cvefile_link)
+        os.symlink(os.path.basename(cve_summary_file), cvefile_link)
+}
+
+addhandler cve_save_summary_handler
+cve_save_summary_handler[eventmask] = "bb.event.BuildCompleted"
+
 python do_cve_check () {
     """
     Check recipe for patched and unpatched CVEs
@@ -331,5 +360,8 @@ def cve_write_data(d, patched, unpatched, whitelisted, cve_data):
             f.write(write_string)
 
     if d.getVar("CVE_CHECK_CREATE_MANIFEST") == "1":
+        cvelogpath = d.getVar("CVE_CHECK_SUMMARY_DIR")
+        bb.utils.mkdirhier(cvelogpath)
+
         with open(d.getVar("CVE_CHECK_TMP_FILE"), "a") as f:
             f.write("%s" % write_string)

meta/classes/deploy.bbclass

@@ -8,4 +8,5 @@ python do_deploy_setscene () {
 }
 addtask do_deploy_setscene
 do_deploy[dirs] = "${DEPLOYDIR} ${B}"
+do_deploy[cleandirs] = "${DEPLOYDIR}"
 do_deploy[stamp-extra-info] = "${MACHINE_ARCH}"

meta/classes/gtk-icon-cache.bbclass

@@ -1,6 +1,15 @@
 FILES_${PN} += "${datadir}/icons/hicolor"
 
-DEPENDS +=" ${@['hicolor-icon-theme', '']['${BPN}' == 'hicolor-icon-theme']} gtk+3-native"
+#gtk+3 reqiure GTK3DISTROFEATURES, DEPENDS on it make all the
+#recipes inherit this class require GTK3DISTROFEATURES
+inherit features_check
+ANY_OF_DISTRO_FEATURES = "${GTK3DISTROFEATURES}"
+
+DEPENDS +=" ${@['hicolor-icon-theme', '']['${BPN}' == 'hicolor-icon-theme']} \
+            ${@['gdk-pixbuf', '']['${BPN}' == 'gdk-pixbuf']} \
+            ${@['gtk+3', '']['${BPN}' == 'gtk+3']} \
+            gtk+3-native \
+"
 
 PACKAGE_WRITE_DEPS += "gtk+3-native gdk-pixbuf-native"
 
@@ -48,9 +57,18 @@ python populate_packages_append () {
         bb.note("adding hicolor-icon-theme dependency to %s" % pkg)
         rdepends = ' ' + d.getVar('MLPREFIX', False) + "hicolor-icon-theme"
         d.appendVar('RDEPENDS_%s' % pkg, rdepends)
-    
+
+        #gtk_icon_cache_postinst depend on gdk-pixbuf and gtk+3
+        bb.note("adding gdk-pixbuf dependency to %s" % pkg)
+        rdepends = ' ' + d.getVar('MLPREFIX', False) + "gdk-pixbuf"
+        d.appendVar('RDEPENDS_%s' % pkg, rdepends)
+
+        bb.note("adding gtk+3 dependency to %s" % pkg)
+        rdepends = ' ' + d.getVar('MLPREFIX', False) + "gtk+3"
+        d.appendVar('RDEPENDS_%s' % pkg, rdepends)
+
         bb.note("adding gtk-icon-cache postinst and postrm scripts to %s" % pkg)
-        
+
         postinst = d.getVar('pkg_postinst_%s' % pkg)
         if not postinst:
             postinst = '#!/bin/sh\n'

meta/classes/gtk-immodules-cache.bbclass

@@ -22,6 +22,7 @@ else
         gtk-query-immodules-2.0 > ${libdir}/gtk-2.0/2.10.0/immodules.cache
     fi
     if [ ! -z `which gtk-query-immodules-3.0` ]; then
+        mkdir -p ${libdir}/gtk-3.0/3.0.0
         gtk-query-immodules-3.0 > ${libdir}/gtk-3.0/3.0.0/immodules.cache
     fi
 fi

meta/classes/image.bbclass

@@ -548,14 +548,14 @@ def get_rootfs_size(d):
     if rootfs_maxsize:
         rootfs_maxsize_int = int(rootfs_maxsize)
         if base_size > rootfs_maxsize_int:
-            bb.fatal("The rootfs size %d(K) overrides IMAGE_ROOTFS_MAXSIZE: %d(K)" % \
+            bb.fatal("The rootfs size %d(K) exceeds IMAGE_ROOTFS_MAXSIZE: %d(K)" % \
                 (base_size, rootfs_maxsize_int))
 
     # Check the initramfs size against INITRAMFS_MAXSIZE (if set)
     if image_fstypes == initramfs_fstypes != ''  and initramfs_maxsize:
         initramfs_maxsize_int = int(initramfs_maxsize)
         if base_size > initramfs_maxsize_int:
-            bb.error("The initramfs size %d(K) overrides INITRAMFS_MAXSIZE: %d(K)" % \
+            bb.error("The initramfs size %d(K) exceeds INITRAMFS_MAXSIZE: %d(K)" % \
                 (base_size, initramfs_maxsize_int))
             bb.error("You can set INITRAMFS_MAXSIZE a larger value. Usually, it should")
             bb.fatal("be less than 1/2 of ram size, or you may fail to boot it.\n")

meta/classes/insane.bbclass

@@ -364,14 +364,14 @@ def package_qa_check_arch(path,name,d, elf, messages):
             target_os == "linux-gnu_ilp32" or re.match(r'mips64.*32', d.getVar('DEFAULTTUNE')))
     is_bpf = (oe.qa.elf_machine_to_string(elf.machine()) == "BPF")
     if not ((machine == elf.machine()) or is_32 or is_bpf):
-        package_qa_add_message(messages, "arch", "Architecture did not match (%s, expected %s) on %s" % \
-                 (oe.qa.elf_machine_to_string(elf.machine()), oe.qa.elf_machine_to_string(machine), package_qa_clean_path(path,d)))
+        package_qa_add_message(messages, "arch", "Architecture did not match (%s, expected %s) in %s" % \
+                 (oe.qa.elf_machine_to_string(elf.machine()), oe.qa.elf_machine_to_string(machine), package_qa_clean_path(path, d, name)))
     elif not ((bits == elf.abiSize()) or is_32 or is_bpf):
-        package_qa_add_message(messages, "arch", "Bit size did not match (%d to %d) %s on %s" % \
-                 (bits, elf.abiSize(), bpn, package_qa_clean_path(path,d)))
+        package_qa_add_message(messages, "arch", "Bit size did not match (%d, expected %d) in %s" % \
+                 (elf.abiSize(), bits, package_qa_clean_path(path, d, name)))
     elif not ((littleendian == elf.isLittleEndian()) or is_bpf):
-        package_qa_add_message(messages, "arch", "Endiannes did not match (%d to %d) on %s" % \
-                 (littleendian, elf.isLittleEndian(), package_qa_clean_path(path,d)))
+        package_qa_add_message(messages, "arch", "Endiannes did not match (%d, expected %d) in %s" % \
+                 (elf.isLittleEndian(), littleendian, package_qa_clean_path(path,d, name)))
 
 QAPATHTEST[desktop] = "package_qa_check_desktop"
 def package_qa_check_desktop(path, name, d, elf, messages):
@@ -459,10 +459,6 @@ def package_qa_check_buildpaths(path, name, d, elf, messages):
     if os.path.islink(path):
         return
 
-    # Ignore ipk and deb's CONTROL dir
-    if path.find(name + "/CONTROL/") != -1 or path.find(name + "/DEBIAN/") != -1:
-        return
-
     tmpdir = bytes(d.getVar('TMPDIR'), encoding="utf-8")
     with open(path, 'rb') as f:
         file_content = f.read()
@@ -1034,7 +1030,14 @@ python do_package_qa () {
     pkgfiles = {}
     for pkg in packages:
         pkgfiles[pkg] = []
-        for walkroot, dirs, files in os.walk(os.path.join(pkgdest, pkg)):
+        pkgdir = os.path.join(pkgdest, pkg)
+        for walkroot, dirs, files in os.walk(pkgdir):
+            # Don't walk into top-level CONTROL or DEBIAN directories as these
+            # are temporary directories created by do_package.
+            if walkroot == pkgdir:
+                for control in ("CONTROL", "DEBIAN"):
+                    if control in dirs:
+                        dirs.remove(control)
             for file in files:
                 pkgfiles[pkg].append(os.path.join(walkroot, file))
 

meta/classes/kernel-devicetree.bbclass

@@ -52,7 +52,7 @@ do_configure_append() {
 do_compile_append() {
 	for dtbf in ${KERNEL_DEVICETREE}; do
 		dtb=`normalize_dtb "$dtbf"`
-		oe_runmake $dtb
+		oe_runmake $dtb CC="${KERNEL_CC} $cc_extra " LD="${KERNEL_LD}" ${KERNEL_EXTRA_ARGS}
 	done
 }
 

meta/classes/kernel-fitimage.bbclass

@@ -257,12 +257,21 @@ fitimage_emit_section_config() {
 	# Test if we have any DTBs at all
 	sep=""
 	conf_desc=""
+	conf_node="conf@"
 	kernel_line=""
 	fdt_line=""
 	ramdisk_line=""
 	setup_line=""
 	default_line=""
 
+	# conf node name is selected based on dtb ID if it is present,
+	# otherwise its selected based on kernel ID
+	if [ -n "${3}" ]; then
+		conf_node=$conf_node${3}
+	else
+		conf_node=$conf_node${2}
+	fi
+
 	if [ -n "${2}" ]; then
 		conf_desc="Linux kernel"
 		sep=", "
@@ -287,12 +296,18 @@ fitimage_emit_section_config() {
 	fi
 
 	if [ "${6}" = "1" ]; then
-		default_line="default = \"conf@${3}\";"
+		# default node is selected based on dtb ID if it is present,
+		# otherwise its selected based on kernel ID
+		if [ -n "${3}" ]; then
+			default_line="default = \"conf@${3}\";"
+		else
+			default_line="default = \"conf@${2}\";"
+		fi
 	fi
 
 	cat << EOF >> ${1}
                 ${default_line}
-                conf@${3} {
+                $conf_node {
 			description = "${6} ${conf_desc}";
 			${kernel_line}
 			${fdt_line}
@@ -434,6 +449,13 @@ fitimage_assemble() {
 	#
 	fitimage_emit_section_maint ${1} confstart
 
+	# kernel-fitimage.bbclass currently only supports a single kernel (no less or
+	# more) to be added to the FIT image along with 0 or more device trees and
+	# 0 or 1 ramdisk.
+	# If a device tree is to be part of the FIT image, then select
+	# the default configuration to be used is based on the dtbcount. If there is
+	# no dtb present than select the default configuation to be based on
+	# the kernelcount.
 	if [ -n "${DTBS}" ]; then
 		i=1
 		for DTB in ${DTBS}; do
@@ -445,6 +467,9 @@ fitimage_assemble() {
 			fi
 			i=`expr ${i} + 1`
 		done
+	else
+		defaultconfigcount=1
+		fitimage_emit_section_config ${1} "${kernelcount}" "" "${ramdiskcount}" "${setupcount}" "${defaultconfigcount}"
 	fi
 
 	fitimage_emit_section_maint ${1} sectend

meta/classes/kernel-yocto.bbclass

@@ -87,6 +87,13 @@ def get_machine_branch(d, default):
 
 do_kernel_metadata() {
 	set +e
+
+	if [ -n "$1" ]; then
+		mode="$1"
+	else
+		mode="patch"
+	fi
+
 	cd ${S}
 	export KMETA=${KMETA}
 
@@ -120,47 +127,59 @@ do_kernel_metadata() {
 	if [ -n "${KBUILD_DEFCONFIG}" ]; then
 		if [ -f "${S}/arch/${ARCH}/configs/${KBUILD_DEFCONFIG}" ]; then
 			if [ -f "${WORKDIR}/defconfig" ]; then
-				# If the two defconfig's are different, warn that we didn't overwrite the
-				# one already placed in WORKDIR by the fetcher.
+				# If the two defconfig's are different, warn that we overwrote the
+				# one already placed in WORKDIR
 				cmp "${WORKDIR}/defconfig" "${S}/arch/${ARCH}/configs/${KBUILD_DEFCONFIG}"
 				if [ $? -ne 0 ]; then
-					bbwarn "defconfig detected in WORKDIR. ${KBUILD_DEFCONFIG} skipped"
-				else
-					cp -f ${S}/arch/${ARCH}/configs/${KBUILD_DEFCONFIG} ${WORKDIR}/defconfig
+					bbdebug 1 "detected SRC_URI or unpatched defconfig in WORKDIR. ${KBUILD_DEFCONFIG} copied over it"
 				fi
+				cp -f ${S}/arch/${ARCH}/configs/${KBUILD_DEFCONFIG} ${WORKDIR}/defconfig
 			else
 				cp -f ${S}/arch/${ARCH}/configs/${KBUILD_DEFCONFIG} ${WORKDIR}/defconfig
 			fi
-			sccs="${WORKDIR}/defconfig"
+			in_tree_defconfig="${WORKDIR}/defconfig"
 		else
 			bbfatal "A KBUILD_DEFCONFIG '${KBUILD_DEFCONFIG}' was specified, but not present in the source tree"
 		fi
 	fi
 
-	# was anyone trying to patch the kernel meta data ?, we need to do
-	# this here, since the scc commands migrate the .cfg fragments to the
-	# kernel source tree, where they'll be used later.
-	check_git_config
-	patches="${@" ".join(find_patches(d,'kernel-meta'))}"
-	for p in $patches; do
-	    (
-		cd ${WORKDIR}/kernel-meta
-		git am -s $p
-	    )
-	done
+	if [ "$mode" = "patch" ]; then
+		# was anyone trying to patch the kernel meta data ?, we need to do
+		# this here, since the scc commands migrate the .cfg fragments to the
+		# kernel source tree, where they'll be used later.
+		check_git_config
+		patches="${@" ".join(find_patches(d,'kernel-meta'))}"
+		for p in $patches; do
+		    (
+			cd ${WORKDIR}/kernel-meta
+			git am -s $p
+		    )
+		done
+	fi
 
 	sccs_from_src_uri="${@" ".join(find_sccs(d))}"
 	patches="${@" ".join(find_patches(d,''))}"
 	feat_dirs="${@" ".join(find_kernel_feature_dirs(d))}"
 
-	# a quick check to make sure we don't have duplicate defconfigs
-	# If there's a defconfig in the SRC_URI, did we also have one from
-	# the KBUILD_DEFCONFIG processing above ?
-	if [ -n "$sccs" ]; then
-	    # we did have a defconfig from above. remove any that might be in the src_uri
-	    sccs_from_src_uri=$(echo $sccs_from_src_uri | awk '{ if ($0!="defconfig") { print $0 } }' RS=' ')
+	# a quick check to make sure we don't have duplicate defconfigs If
+	# there's a defconfig in the SRC_URI, did we also have one from the
+	# KBUILD_DEFCONFIG processing above ?
+	src_uri_defconfig=$(echo $sccs_from_src_uri | awk '(match($0, "defconfig") != 0) { print $0 }' RS=' ')
+	# drop and defconfig's from the src_uri variable, we captured it just above here if it existed
+	sccs_from_src_uri=$(echo $sccs_from_src_uri | awk '(match($0, "defconfig") == 0) { print $0 }' RS=' ')
+
+	if [ -n "$in_tree_defconfig" ]; then
+		sccs_defconfig=$in_tree_defconfig
+		if [ -n "$src_uri_defconfig" ]; then
+			bbwarn "[NOTE]: defconfig was supplied both via KBUILD_DEFCONFIG and SRC_URI. Dropping SRC_URI defconfig"
+		fi
+	else
+		# if we didn't have an in-tree one, make our defconfig the one
+		# from the src_uri. Note: there may not have been one from the
+		# src_uri, so this can be an empty variable.
+		sccs_defconfig=$src_uri_defconfig
 	fi
-	sccs="$sccs $sccs_from_src_uri"
+	sccs="$sccs_from_src_uri"
 
 	# check for feature directories/repos/branches that were part of the
 	# SRC_URI. If they were supplied, we convert them into include directives
@@ -187,11 +206,10 @@ do_kernel_metadata() {
 	# expand kernel features into their full path equivalents
 	bsp_definition=$(spp ${includes} --find -DKMACHINE=${KMACHINE} -DKTYPE=${LINUX_KERNEL_TYPE})
 	if [ -z "$bsp_definition" ]; then
-		echo "$sccs" | grep -q defconfig
-		if [ $? -ne 0 ]; then
+		if [ -z "$sccs_defconfig" ]; then
 			bbfatal_log "Could not locate BSP definition for ${KMACHINE}/${LINUX_KERNEL_TYPE} and no defconfig was provided"
 		fi
-
+	else
 		# if the bsp definition has "define KMETA_EXTERNAL_BSP t",
 		# then we need to set a flag that will instruct the next
 		# steps to use the BSP as both configuration and patches.
@@ -202,13 +220,40 @@ do_kernel_metadata() {
 	fi
 	meta_dir=$(kgit --meta)
 
-	# run1: pull all the configuration fragments, no matter where they come from
-	elements="`echo -n ${bsp_definition} ${sccs} ${patches} ${KERNEL_FEATURES}`"
-	if [ -n "${elements}" ]; then
-		echo "${bsp_definition}" > ${S}/${meta_dir}/bsp_definition
-		scc --force -o ${S}/${meta_dir}:cfg,merge,meta ${includes} ${bsp_definition} ${sccs} ${patches} ${KERNEL_FEATURES}
-		if [ $? -ne 0 ]; then
-			bbfatal_log "Could not generate configuration queue for ${KMACHINE}."
+	KERNEL_FEATURES_FINAL=""
+	if [ -n "${KERNEL_FEATURES}" ]; then
+		for feature in ${KERNEL_FEATURES}; do
+			feature_found=f
+			for d in $includes; do
+				path_to_check=$(echo $d | sed 's/-I//g')
+				if [ "$feature_found" = "f" ] && [ -e "$path_to_check/$feature" ]; then
+				    feature_found=t
+				fi
+			done
+			if [ "$feature_found" = "f" ]; then
+				if [ -n "${KERNEL_DANGLING_FEATURES_WARN_ONLY}" ]; then
+				    bbwarn "Feature '$feature' not found, but KERNEL_DANGLING_FEATURES_WARN_ONLY is set"
+				    bbwarn "This may cause runtime issues, dropping feature and allowing configuration to continue"
+				else
+				    bberror "Feature '$feature' not found, this will cause configuration failures."
+				    bberror "Check the SRC_URI for meta-data repositories or directories that may be missing"
+				    bbfatal_log "Set KERNEL_DANGLING_FEATURES_WARN_ONLY to ignore this issue"
+				fi
+			else
+				KERNEL_FEATURES_FINAL="$KERNEL_FEATURES_FINAL $feature"
+			fi
+		done
+        fi
+
+	if [ "$mode" = "config" ]; then
+		# run1: pull all the configuration fragments, no matter where they come from
+		elements="`echo -n ${bsp_definition} $sccs_defconfig ${sccs} ${patches} $KERNEL_FEATURES_FINAL`"
+		if [ -n "${elements}" ]; then
+			echo "${bsp_definition}" > ${S}/${meta_dir}/bsp_definition
+			scc --force -o ${S}/${meta_dir}:cfg,merge,meta ${includes} $sccs_defconfig $bsp_definition $sccs $patches $KERNEL_FEATURES_FINAL
+			if [ $? -ne 0 ]; then
+				bbfatal_log "Could not generate configuration queue for ${KMACHINE}."
+			fi
 		fi
 	fi
 
@@ -219,12 +264,14 @@ do_kernel_metadata() {
 		sccs="${bsp_definition} ${sccs}"
 	fi
 
-	# run2: only generate patches for elements that have been passed on the SRC_URI
-	elements="`echo -n ${sccs} ${patches} ${KERNEL_FEATURES}`"
-	if [ -n "${elements}" ]; then
-		scc --force -o ${S}/${meta_dir}:patch --cmds patch ${includes} ${sccs} ${patches} ${KERNEL_FEATURES}
-		if [ $? -ne 0 ]; then
-			bbfatal_log "Could not generate configuration queue for ${KMACHINE}."
+	if [ "$mode" = "patch" ]; then
+		# run2: only generate patches for elements that have been passed on the SRC_URI
+		elements="`echo -n ${sccs} ${patches} $KERNEL_FEATURES_FINAL`"
+		if [ -n "${elements}" ]; then
+			scc --force -o ${S}/${meta_dir}:patch --cmds patch ${includes} ${sccs} ${patches} $KERNEL_FEATURES_FINAL
+			if [ $? -ne 0 ]; then
+				bbfatal_log "Could not generate configuration queue for ${KMACHINE}."
+			fi
 		fi
 	fi
 }
@@ -328,6 +375,8 @@ do_kernel_configme[depends] += "bc-native:do_populate_sysroot bison-native:do_po
 do_kernel_configme[depends] += "kern-tools-native:do_populate_sysroot"
 do_kernel_configme[dirs] += "${S} ${B}"
 do_kernel_configme() {
+	do_kernel_metadata config
+
 	# translate the kconfig_mode into something that merge_config.sh
 	# understands
 	case ${KCONFIG_MODE} in
@@ -370,6 +419,67 @@ do_kernel_configme() {
 }
 
 addtask kernel_configme before do_configure after do_patch
+addtask config_analysis
+
+do_config_analysis[depends] = "virtual/kernel:do_configure"
+do_config_analysis[depends] += "kern-tools-native:do_populate_sysroot"
+
+CONFIG_AUDIT_FILE ?= "${WORKDIR}/config-audit.txt"
+CONFIG_ANALYSIS_FILE ?= "${WORKDIR}/config-analysis.txt"
+
+python do_config_analysis() {
+    import re, string, sys, subprocess
+
+    s = d.getVar('S')
+
+    env = os.environ.copy()
+    env['PATH'] = "%s:%s%s" % (d.getVar('PATH'), s, "/scripts/util/")
+    env['LD'] = d.getVar('KERNEL_LD')
+    env['CC'] = d.getVar('KERNEL_CC')
+    env['ARCH'] = d.getVar('ARCH')
+    env['srctree'] = s
+
+    # read specific symbols from the kernel recipe or from local.conf
+    # i.e.: CONFIG_ANALYSIS_pn-linux-yocto-dev = 'NF_CONNTRACK LOCALVERSION'
+    config = d.getVar( 'CONFIG_ANALYSIS' )
+    if not config:
+       config = [ "" ]
+    else:
+       config = config.split()
+
+    for c in config:
+        for action in ["analysis","audit"]:
+            if action == "analysis":
+                try:
+                    analysis = subprocess.check_output(['symbol_why.py', '--dotconfig',  '{}'.format( d.getVar('B') + '/.config' ), '--blame', c], cwd=s, env=env ).decode('utf-8')
+                except subprocess.CalledProcessError as e:
+                    bb.fatal( "config analysis failed: %s" % e.output.decode('utf-8'))
+
+                outfile = d.getVar( 'CONFIG_ANALYSIS_FILE' )
+
+            if action == "audit":
+                try:
+                    analysis = subprocess.check_output(['symbol_why.py', '--dotconfig',  '{}'.format( d.getVar('B') + '/.config' ), '--summary', '--extended', '--sanity', c], cwd=s, env=env ).decode('utf-8')
+                except subprocess.CalledProcessError as e:
+                    bb.fatal( "config analysis failed: %s" % e.output.decode('utf-8'))
+
+                outfile = d.getVar( 'CONFIG_AUDIT_FILE' )
+
+            if c:
+                outdir = os.path.dirname( outfile )
+                outname = os.path.basename( outfile )
+                outfile = outdir + '/'+ c + '-' + outname
+
+            if config and os.path.isfile(outfile):
+                os.remove(outfile)
+
+            with open(outfile, 'w+') as f:
+                f.write( analysis )
+
+            bb.warn( "Configuration {} executed, see: {} for details".format(action,outfile ))
+            if c:
+                bb.warn( analysis )
+}
 
 python do_kernel_configcheck() {
     import re, string, sys, subprocess
@@ -379,57 +489,89 @@ python do_kernel_configcheck() {
     # meta-series for processing
     kmeta = d.getVar("KMETA") or "meta"
     if not os.path.exists(kmeta):
-        kmeta = "." + kmeta
+        kmeta = subprocess.check_output(['kgit', '--meta'], cwd=d.getVar('S')).decode('utf-8').rstrip()
 
     s = d.getVar('S')
 
     env = os.environ.copy()
     env['PATH'] = "%s:%s%s" % (d.getVar('PATH'), s, "/scripts/util/")
-    env['LD'] = "${KERNEL_LD}"
+    env['LD'] = d.getVar('KERNEL_LD')
+    env['CC'] = d.getVar('KERNEL_CC')
+    env['ARCH'] = d.getVar('ARCH')
+    env['srctree'] = s
 
     try:
         configs = subprocess.check_output(['scc', '--configs', '-o', s + '/.kernel-meta'], env=env).decode('utf-8')
     except subprocess.CalledProcessError as e:
         bb.fatal( "Cannot gather config fragments for audit: %s" % e.output.decode("utf-8") )
 
-    try:
-        subprocess.check_call(['kconf_check', '--report', '-o',
-                '%s/%s/cfg' % (s, kmeta), d.getVar('B') + '/.config', s, configs], cwd=s, env=env)
-    except subprocess.CalledProcessError:
-        # The configuration gathering can return different exit codes, but
-        # we interpret them based on the KCONF_AUDIT_LEVEL variable, so we catch
-        # everything here, and let the run continue.
-        pass
-
     config_check_visibility = int(d.getVar("KCONF_AUDIT_LEVEL") or 0)
     bsp_check_visibility = int(d.getVar("KCONF_BSP_AUDIT_LEVEL") or 0)
 
-    # if config check visibility is non-zero, report dropped configuration values
-    mismatch_file = d.expand("${S}/%s/cfg/mismatch.txt" % kmeta)
-    if os.path.exists(mismatch_file):
-        if config_check_visibility:
-            with open (mismatch_file, "r") as myfile:
+    # if config check visibility is "1", that's the lowest level of audit. So
+    # we add the --classify option to the run, since classification will
+    # streamline the output to only report options that could be boot issues,
+    # or are otherwise required for proper operation.
+    extra_params = ""
+    if config_check_visibility == 1:
+       extra_params = "--classify"
+
+    # category #1: mismatches
+    try:
+        analysis = subprocess.check_output(['symbol_why.py', '--dotconfig',  '{}'.format( d.getVar('B') + '/.config' ), '--mismatches', extra_params], cwd=s, env=env ).decode('utf-8')
+    except subprocess.CalledProcessError as e:
+        bb.fatal( "config analysis failed: %s" % e.output.decode('utf-8'))
+
+    if analysis:
+        outfile = "{}/{}/cfg/mismatch.txt".format( s, kmeta )
+        if os.path.isfile(outfile):
+           os.remove(outfile)
+        with open(outfile, 'w+') as f:
+            f.write( analysis )
+
+        if config_check_visibility and os.stat(outfile).st_size > 0:
+            with open (outfile, "r") as myfile:
                 results = myfile.read()
                 bb.warn( "[kernel config]: specified values did not make it into the kernel's final configuration:\n\n%s" % results)
 
-    if bsp_check_visibility:
-        invalid_file = d.expand("${S}/%s/cfg/invalid.cfg" % kmeta)
-        if os.path.exists(invalid_file) and os.stat(invalid_file).st_size > 0:
-            with open (invalid_file, "r") as myfile:
-                results = myfile.read()
-                bb.warn( "[kernel config]: This BSP sets config options that are not offered anywhere within this kernel:\n\n%s" % results)
-        errors_file = d.expand("${S}/%s/cfg/fragment_errors.txt" % kmeta)
-        if os.path.exists(errors_file) and os.stat(errors_file).st_size > 0:
-            with open (errors_file, "r") as myfile:
-               results = myfile.read()
-               bb.warn( "[kernel config]: This BSP contains fragments with errors:\n\n%s" % results)
+    # category #2: invalid fragment elements
+    extra_params = ""
+    if bsp_check_visibility > 1:
+        extra_params = "--strict"
+    try:
+        analysis = subprocess.check_output(['symbol_why.py', '--dotconfig',  '{}'.format( d.getVar('B') + '/.config' ), '--invalid', extra_params], cwd=s, env=env ).decode('utf-8')
+    except subprocess.CalledProcessError as e:
+        bb.fatal( "config analysis failed: %s" % e.output.decode('utf-8'))
 
-    # if the audit level is greater than two, we report if a fragment has overriden
-    # a value from a base fragment. This is really only used for new kernel introduction
-    if bsp_check_visibility > 2:
-        redefinition_file = d.expand("${S}/%s/cfg/redefinition.txt" % kmeta)
-        if os.path.exists(redefinition_file) and os.stat(redefinition_file).st_size > 0:
-            with open (redefinition_file, "r") as myfile:
+    if analysis:
+        outfile = "{}/{}/cfg/invalid.txt".format(s,kmeta)
+        if os.path.isfile(outfile):
+           os.remove(outfile)
+        with open(outfile, 'w+') as f:
+            f.write( analysis )
+
+        if bsp_check_visibility and os.stat(outfile).st_size > 0:
+            with open (outfile, "r") as myfile:
+               results = myfile.read()
+               bb.warn( "[kernel config]: This BSP contains fragments with warnings:\n\n%s" % results)
+
+    # category #3: redefined options (this is pretty verbose and is debug only)
+    try:
+        analysis = subprocess.check_output(['symbol_why.py', '--dotconfig',  '{}'.format( d.getVar('B') + '/.config' ), '--sanity'], cwd=s, env=env ).decode('utf-8')
+    except subprocess.CalledProcessError as e:
+        bb.fatal( "config analysis failed: %s" % e.output.decode('utf-8'))
+
+    if analysis:
+        outfile = "{}/{}/cfg/redefinition.txt".format(s,kmeta)
+        if os.path.isfile(outfile):
+           os.remove(outfile)
+        with open(outfile, 'w+') as f:
+            f.write( analysis )
+
+        # if the audit level is greater than two, we report if a fragment has overriden
+        # a value from a base fragment. This is really only used for new kernel introduction
+        if bsp_check_visibility > 2 and os.stat(outfile).st_size > 0:
+            with open (outfile, "r") as myfile:
                 results = myfile.read()
                 bb.warn( "[kernel config]: This BSP has configuration options defined in more than one config, with differing values:\n\n%s" % results)
 }

meta/classes/kernel.bbclass

@@ -4,10 +4,12 @@ KERNEL_PACKAGE_NAME ??= "kernel"
 KERNEL_DEPLOYSUBDIR ??= "${@ "" if (d.getVar("KERNEL_PACKAGE_NAME") == "kernel") else d.getVar("KERNEL_PACKAGE_NAME") }"
 
 PROVIDES += "${@ "virtual/kernel" if (d.getVar("KERNEL_PACKAGE_NAME") == "kernel") else "" }"
-DEPENDS += "virtual/${TARGET_PREFIX}binutils virtual/${TARGET_PREFIX}gcc kmod-native bc-native lzop-native bison-native"
+DEPENDS += "virtual/${TARGET_PREFIX}binutils virtual/${TARGET_PREFIX}gcc kmod-native bc-native bison-native"
+DEPENDS += "${@bb.utils.contains("INITRAMFS_FSTYPES", "cpio.lzo", "lzop-native", "", d)}"
+DEPENDS += "${@bb.utils.contains("INITRAMFS_FSTYPES", "cpio.lz4", "lz4-native", "", d)}"
 PACKAGE_WRITE_DEPS += "depmodwrapper-cross"
 
-do_deploy[depends] += "depmodwrapper-cross:do_populate_sysroot"
+do_deploy[depends] += "depmodwrapper-cross:do_populate_sysroot gzip-native:do_populate_sysroot"
 do_clean[depends] += "make-mod-scripts:do_clean"
 
 CVE_PRODUCT ?= "linux_kernel"
@@ -94,6 +96,25 @@ python __anonymous () {
         d.appendVar('RDEPENDS_%s-image' % kname, ' %s-image-%s' % (kname, typelower))
         d.setVar('PKG_%s-image-%s' % (kname,typelower), '%s-image-%s-${KERNEL_VERSION_PKG_NAME}' % (kname, typelower))
         d.setVar('ALLOW_EMPTY_%s-image-%s' % (kname, typelower), '1')
+        d.setVar('pkg_postinst_%s-image-%s' % (kname,typelower), """set +e
+if [ -n "$D" ]; then
+    ln -sf %s-${KERNEL_VERSION} $D/${KERNEL_IMAGEDEST}/%s > /dev/null 2>&1
+else
+    ln -sf %s-${KERNEL_VERSION} ${KERNEL_IMAGEDEST}/%s > /dev/null 2>&1
+    if [ $? -ne 0 ]; then
+        echo "Filesystem on ${KERNEL_IMAGEDEST}/ doesn't support symlinks, falling back to copied image (%s)."
+        install -m 0644 ${KERNEL_IMAGEDEST}/%s-${KERNEL_VERSION} ${KERNEL_IMAGEDEST}/%s
+    fi
+fi
+set -e
+""" % (type, type, type, type, type, type, type))
+        d.setVar('pkg_postrm_%s-image-%s' % (kname,typelower), """set +e
+if [ -f "${KERNEL_IMAGEDEST}/%s" -o -L "${KERNEL_IMAGEDEST}/%s" ]; then
+    rm -f ${KERNEL_IMAGEDEST}/%s  > /dev/null 2>&1
+fi
+set -e
+""" % (type, type, type))
+
 
     image = d.getVar('INITRAMFS_IMAGE')
     # If the INTIRAMFS_IMAGE is set but the INITRAMFS_IMAGE_BUNDLE is set to 0,
@@ -191,6 +212,8 @@ UBOOT_LOADADDRESS ?= "${UBOOT_ENTRYPOINT}"
 KERNEL_EXTRA_ARGS ?= ""
 
 EXTRA_OEMAKE = " HOSTCC="${BUILD_CC} ${BUILD_CFLAGS} ${BUILD_LDFLAGS}" HOSTCPP="${BUILD_CPP}""
+EXTRA_OEMAKE += " HOSTCXX="${BUILD_CXX} ${BUILD_CXXFLAGS} ${BUILD_LDFLAGS}""
+
 KERNEL_ALT_IMAGETYPE ??= ""
 
 copy_initramfs() {
@@ -210,7 +233,7 @@ copy_initramfs() {
 				;;
 			*lz4)
 				echo "lz4 decompressing image"
-				lz4 -df ${B}/usr/${INITRAMFS_IMAGE_NAME}.$img
+				lz4 -df ${B}/usr/${INITRAMFS_IMAGE_NAME}.$img ${B}/usr/${INITRAMFS_IMAGE_NAME}.cpio
 				break
 				;;
 			*lzo)
@@ -385,9 +408,6 @@ kernel_do_install() {
 	install -d ${D}/boot
 	for imageType in ${KERNEL_IMAGETYPES} ; do
 		install -m 0644 ${KERNEL_OUTPUT_DIR}/${imageType} ${D}/${KERNEL_IMAGEDEST}/${imageType}-${KERNEL_VERSION}
-		if [ "${KERNEL_PACKAGE_NAME}" = "kernel" ]; then
-			ln -sf ${imageType}-${KERNEL_VERSION} ${D}/${KERNEL_IMAGEDEST}/${imageType}
-		fi
 	done
 	install -m 0644 System.map ${D}/boot/System.map-${KERNEL_VERSION}
 	install -m 0644 .config ${D}/boot/config-${KERNEL_VERSION}
@@ -463,7 +483,7 @@ do_shared_workdir () {
 
 	# Copy files required for module builds
 	cp System.map $kerneldir/System.map-${KERNEL_VERSION}
-	cp Module.symvers $kerneldir/
+	[ -e Module.symvers ] && cp Module.symvers $kerneldir/
 	cp .config $kerneldir/
 	mkdir -p $kerneldir/include/config
 	cp include/config/kernel.release $kerneldir/include/config/kernel.release
@@ -729,8 +749,6 @@ kernel_do_deploy() {
 		done
 	fi
 }
-do_deploy[cleandirs] = "${DEPLOYDIR}"
-do_deploy[dirs] = "${DEPLOYDIR} ${B}"
 do_deploy[prefuncs] += "package_get_auto_pr"
 
 addtask deploy after do_populate_sysroot do_packagedata

meta/classes/meson.bbclass

@@ -98,6 +98,7 @@ strip = ${@meson_array('STRIP', d)}
 readelf = ${@meson_array('READELF', d)}
 pkgconfig = 'pkg-config'
 llvm-config = 'llvm-config${LLVMVERSION}'
+cups-config = 'cups-config'
 
 [properties]
 needs_exe_wrapper = true

meta/classes/mime.bbclass

@@ -24,7 +24,18 @@ if [ "x$D" != "x" ]; then
 		mimedir=${MIMEDIR}
 else
 	echo "Updating MIME database... this may take a while."
-	update-mime-database $D${MIMEDIR}
+	# $D${MIMEDIR}/packages belong to package shared-mime-info-data,
+	# packages like libfm-mime depend on shared-mime-info-data.
+	# after shared-mime-info-data uninstalled, $D${MIMEDIR}/packages
+	# is removed, but update-mime-database need this dir to update
+	# database, workaround to create one and remove it later
+	if [ ! -d $D${MIMEDIR}/packages ]; then
+		mkdir -p $D${MIMEDIR}/packages
+		update-mime-database $D${MIMEDIR}
+		rmdir --ignore-fail-on-non-empty $D${MIMEDIR}/packages
+	else
+		update-mime-database $D${MIMEDIR}
+fi
 fi
 }
 

meta/classes/nativesdk.bbclass

@@ -9,6 +9,7 @@ NATIVESDKLIBC ?= "libc-glibc"
 LIBCOVERRIDE = ":${NATIVESDKLIBC}"
 CLASSOVERRIDE = "class-nativesdk"
 MACHINEOVERRIDES = ""
+MACHINE_FEATURES = ""
 
 MULTILIBS = ""
 
@@ -57,7 +58,7 @@ EXTRA_OECONF_GCC_FLOAT = ""
 
 CPPFLAGS = "${BUILDSDK_CPPFLAGS}"
 CFLAGS = "${BUILDSDK_CFLAGS}"
-CXXFLAGS = "${BUILDSDK_CFLAGS}"
+CXXFLAGS = "${BUILDSDK_CXXFLAGS}"
 LDFLAGS = "${BUILDSDK_LDFLAGS}"
 
 # Change to place files in SDKPATH

meta/classes/package.bbclass

@@ -1039,7 +1039,7 @@ python split_and_strip_files () {
 
     dvar = d.getVar('PKGD')
     pn = d.getVar('PN')
-    targetos = d.getVar('TARGET_OS')
+    hostos = d.getVar('HOST_OS')
 
     oldcwd = os.getcwd()
     os.chdir(dvar)
@@ -1194,7 +1194,7 @@ python split_and_strip_files () {
     if (d.getVar('INHIBIT_PACKAGE_DEBUG_SPLIT') != '1'):
         results = oe.utils.multiprocess_launch(splitdebuginfo, list(elffiles), d, extraargs=(dvar, debugdir, debuglibdir, debugappend, debugsrcdir, d))
 
-        if debugsrcdir and not targetos.startswith("mingw"):
+        if debugsrcdir and not hostos.startswith("mingw"):
             if (d.getVar('PACKAGE_DEBUG_STATIC_SPLIT') == '1'):
                 results = oe.utils.multiprocess_launch(splitstaticdebuginfo, staticlibs, d, extraargs=(dvar, debugstaticdir, debugstaticlibdir, debugstaticappend, debugsrcdir, d))
             else:
@@ -1761,7 +1761,7 @@ python package_do_shlibs() {
     else:
         shlib_pkgs = packages.split()
 
-    targetos = d.getVar('TARGET_OS')
+    hostos = d.getVar('HOST_OS')
 
     workdir = d.getVar('WORKDIR')
 
@@ -1912,9 +1912,9 @@ python package_do_shlibs() {
                 soname = None
                 if cpath.islink(file):
                     continue
-                if targetos == "darwin" or targetos == "darwin8":
+                if hostos == "darwin" or hostos == "darwin8":
                     darwin_so(file, needed, sonames, renames, pkgver)
-                elif targetos.startswith("mingw"):
+                elif hostos.startswith("mingw"):
                     mingw_dll(file, needed, sonames, renames, pkgver)
                 elif os.access(file, os.X_OK) or lib_re.match(file):
                     linuxlist.append(file)
@@ -1936,7 +1936,7 @@ python package_do_shlibs() {
         shlibs_file = os.path.join(shlibswork_dir, pkg + ".list")
         if len(sonames):
             with open(shlibs_file, 'w') as fd:
-                for s in sonames:
+                for s in sorted(sonames):
                     if s[0] in shlib_provider and s[1] in shlib_provider[s[0]]:
                         (old_pkg, old_pkgver) = shlib_provider[s[0]][s[1]]
                         if old_pkg != pkg:

meta/classes/packagefeed-stability.bbclass

@@ -1,252 +0,0 @@
-# Class to avoid copying packages into the feed if they haven't materially changed
-#
-# Copyright (C) 2015 Intel Corporation
-# Released under the MIT license (see COPYING.MIT for details)
-#
-# This class effectively intercepts packages as they are written out by
-# do_package_write_*, causing them to be written into a different
-# directory where we can compare them to whatever older packages might
-# be in the "real" package feed directory, and avoid copying the new
-# package to the feed if it has not materially changed. The idea is to
-# avoid unnecessary churn in the packages when dependencies trigger task
-# reexecution (and thus repackaging). Enabling the class is simple:
-#
-# INHERIT += "packagefeed-stability"
-#
-# Caveats:
-# 1) Latest PR values in the build system may not match those in packages
-#    seen on the target (naturally)
-# 2) If you rebuild from sstate without the existing package feed present,
-#    you will lose the "state" of the package feed i.e. the preserved old
-#    package versions. Not the end of the world, but would negate the
-#    entire purpose of this class.
-#
-# Note that running -c cleanall on a recipe will purposely delete the old
-# package files so they will definitely be copied the next time.
-
-python() {
-    if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross', d):
-        return
-    # Package backend agnostic intercept
-    # This assumes that the package_write task is called package_write_<pkgtype>
-    # and that the directory in which packages should be written is
-    # pointed to by the variable DEPLOY_DIR_<PKGTYPE>
-    for pkgclass in (d.getVar('PACKAGE_CLASSES') or '').split():
-        if pkgclass.startswith('package_'):
-            pkgtype = pkgclass.split('_', 1)[1]
-            pkgwritefunc = 'do_package_write_%s' % pkgtype
-            sstate_outputdirs = d.getVarFlag(pkgwritefunc, 'sstate-outputdirs', False)
-            deploydirvar = 'DEPLOY_DIR_%s' % pkgtype.upper()
-            deploydirvarref = '${' + deploydirvar + '}'
-            pkgcomparefunc = 'do_package_compare_%s' % pkgtype
-
-            if bb.data.inherits_class('image', d):
-                d.appendVarFlag('do_rootfs', 'recrdeptask', ' ' + pkgcomparefunc)
-
-            if bb.data.inherits_class('populate_sdk_base', d):
-                d.appendVarFlag('do_populate_sdk', 'recrdeptask', ' ' + pkgcomparefunc)
-
-            if bb.data.inherits_class('populate_sdk_ext', d):
-                d.appendVarFlag('do_populate_sdk_ext', 'recrdeptask', ' ' + pkgcomparefunc)
-
-            d.appendVarFlag('do_build', 'recrdeptask', ' ' + pkgcomparefunc)
-
-            if d.getVarFlag(pkgwritefunc, 'noexec') or not d.getVarFlag(pkgwritefunc, 'task'):
-                # Packaging is disabled for this recipe, we shouldn't do anything
-                continue
-
-            if deploydirvarref in sstate_outputdirs:
-                deplor_dir_pkgtype = d.expand(deploydirvarref + '-prediff')
-                # Set intermediate output directory
-                d.setVarFlag(pkgwritefunc, 'sstate-outputdirs', sstate_outputdirs.replace(deploydirvarref, deplor_dir_pkgtype))
-                # Update SSTATE_DUPWHITELIST to avoid shared location conflicted error
-                d.appendVar('SSTATE_DUPWHITELIST', ' %s' % deplor_dir_pkgtype)
-
-            d.setVar(pkgcomparefunc, d.getVar('do_package_compare', False))
-            d.setVarFlags(pkgcomparefunc, d.getVarFlags('do_package_compare', False))
-            d.appendVarFlag(pkgcomparefunc, 'depends', ' build-compare-native:do_populate_sysroot')
-            bb.build.addtask(pkgcomparefunc, 'do_build', 'do_packagedata ' + pkgwritefunc, d)
-}
-
-# This isn't the real task function - it's a template that we use in the
-# anonymous python code above
-fakeroot python do_package_compare () {
-    currenttask = d.getVar('BB_CURRENTTASK')
-    pkgtype = currenttask.rsplit('_', 1)[1]
-    package_compare_impl(pkgtype, d)
-}
-
-def package_compare_impl(pkgtype, d):
-    import errno
-    import fnmatch
-    import glob
-    import subprocess
-    import oe.sstatesig
-
-    pn = d.getVar('PN')
-    deploydir = d.getVar('DEPLOY_DIR_%s' % pkgtype.upper())
-    prepath = deploydir + '-prediff/'
-
-    # Find out PKGR values are
-    pkgdatadir = d.getVar('PKGDATA_DIR')
-    packages = []
-    try:
-        with open(os.path.join(pkgdatadir, pn), 'r') as f:
-            for line in f:
-                if line.startswith('PACKAGES:'):
-                    packages = line.split(':', 1)[1].split()
-                    break
-    except IOError as e:
-        if e.errno == errno.ENOENT:
-            pass
-
-    if not packages:
-        bb.debug(2, '%s: no packages, nothing to do' % pn)
-        return
-
-    pkgrvalues = {}
-    rpkgnames = {}
-    rdepends = {}
-    pkgvvalues = {}
-    for pkg in packages:
-        with open(os.path.join(pkgdatadir, 'runtime', pkg), 'r') as f:
-            for line in f:
-                if line.startswith('PKGR:'):
-                    pkgrvalues[pkg] = line.split(':', 1)[1].strip()
-                if line.startswith('PKGV:'):
-                    pkgvvalues[pkg] = line.split(':', 1)[1].strip()
-                elif line.startswith('PKG_%s:' % pkg):
-                    rpkgnames[pkg] = line.split(':', 1)[1].strip()
-                elif line.startswith('RDEPENDS_%s:' % pkg):
-                    rdepends[pkg] = line.split(':', 1)[1].strip()
-
-    # Prepare a list of the runtime package names for packages that were
-    # actually produced
-    rpkglist = []
-    for pkg, rpkg in rpkgnames.items():
-        if os.path.exists(os.path.join(pkgdatadir, 'runtime', pkg + '.packaged')):
-            rpkglist.append((rpkg, pkg))
-    rpkglist.sort(key=lambda x: len(x[0]), reverse=True)
-
-    pvu = d.getVar('PV', False)
-    if '$' + '{SRCPV}' in pvu:
-        pvprefix = pvu.split('$' + '{SRCPV}', 1)[0]
-    else:
-        pvprefix = None
-
-    pkgwritetask = 'package_write_%s' % pkgtype
-    files = []
-    docopy = False
-    manifest, _ = oe.sstatesig.sstate_get_manifest_filename(pkgwritetask, d)
-    mlprefix = d.getVar('MLPREFIX')
-    # Copy recipe's all packages if one of the packages are different to make
-    # they have the same PR.
-    with open(manifest, 'r') as f:
-        for line in f:
-            if line.startswith(prepath):
-                srcpath = line.rstrip()
-                if os.path.isfile(srcpath):
-                    destpath = os.path.join(deploydir, os.path.relpath(srcpath, prepath))
-
-                    # This is crude but should work assuming the output
-                    # package file name starts with the package name
-                    # and rpkglist is sorted by length (descending)
-                    pkgbasename = os.path.basename(destpath)
-                    pkgname = None
-                    for rpkg, pkg in rpkglist:
-                        if mlprefix and pkgtype == 'rpm' and rpkg.startswith(mlprefix):
-                            rpkg = rpkg[len(mlprefix):]
-                        if pkgbasename.startswith(rpkg):
-                            pkgr = pkgrvalues[pkg]
-                            destpathspec = destpath.replace(pkgr, '*')
-                            if pvprefix:
-                                pkgv = pkgvvalues[pkg]
-                                if pkgv.startswith(pvprefix):
-                                    pkgvsuffix = pkgv[len(pvprefix):]
-                                    if '+' in pkgvsuffix:
-                                        newpkgv = pvprefix + '*+' + pkgvsuffix.split('+', 1)[1]
-                                        destpathspec = destpathspec.replace(pkgv, newpkgv)
-                            pkgname = pkg
-                            break
-                    else:
-                        bb.warn('Unable to map %s back to package' % pkgbasename)
-                        destpathspec = destpath
-
-                    oldfile = None
-                    if not docopy:
-                        oldfiles = glob.glob(destpathspec)
-                        if oldfiles:
-                            oldfile = oldfiles[-1]
-                            result = subprocess.call(['pkg-diff.sh', oldfile, srcpath])
-                            if result != 0:
-                                docopy = True
-                                bb.note("%s and %s are different, will copy packages" % (oldfile, srcpath))
-                        else:
-                            docopy = True
-                            bb.note("No old packages found for %s, will copy packages" % pkgname)
-
-                    files.append((pkgname, pkgbasename, srcpath, destpath))
-
-    # Remove all the old files and copy again if docopy
-    if docopy:
-        bb.note('Copying packages for recipe %s' % pn)
-        pcmanifest = os.path.join(prepath, d.expand('pkg-compare-manifest-${MULTIMACH_TARGET_SYS}-${PN}'))
-        try:
-            with open(pcmanifest, 'r') as f:
-                for line in f:
-                    fn = line.rstrip()
-                    if fn:
-                        try:
-                            os.remove(fn)
-                            bb.note('Removed old package %s' % fn)
-                        except OSError as e:
-                            if e.errno == errno.ENOENT:
-                                pass
-        except IOError as e:
-            if e.errno == errno.ENOENT:
-                pass
-
-        # Create new manifest
-        with open(pcmanifest, 'w') as f:
-            for pkgname, pkgbasename, srcpath, destpath in files:
-                destdir = os.path.dirname(destpath)
-                bb.utils.mkdirhier(destdir)
-                # Remove allarch rpm pkg if it is already existed (for
-                # multilib), they're identical in theory, but sstate.bbclass
-                # copies it again, so keep align with that.
-                if os.path.exists(destpath) and pkgtype == 'rpm' \
-                        and d.getVar('PACKAGE_ARCH') == 'all':
-                    os.unlink(destpath)
-                if (os.stat(srcpath).st_dev == os.stat(destdir).st_dev):
-                    # Use a hard link to save space
-                    os.link(srcpath, destpath)
-                else:
-                    shutil.copyfile(srcpath, destpath)
-                f.write('%s\n' % destpath)
-    else:
-        bb.note('Not copying packages for recipe %s' % pn)
-
-do_cleansstate[postfuncs] += "pfs_cleanpkgs"
-python pfs_cleanpkgs () {
-    import errno
-    for pkgclass in (d.getVar('PACKAGE_CLASSES') or '').split():
-        if pkgclass.startswith('package_'):
-            pkgtype = pkgclass.split('_', 1)[1]
-            deploydir = d.getVar('DEPLOY_DIR_%s' % pkgtype.upper())
-            prepath = deploydir + '-prediff'
-            pcmanifest = os.path.join(prepath, d.expand('pkg-compare-manifest-${MULTIMACH_TARGET_SYS}-${PN}'))
-            try:
-                with open(pcmanifest, 'r') as f:
-                    for line in f:
-                        fn = line.rstrip()
-                        if fn:
-                            try:
-                                os.remove(fn)
-                            except OSError as e:
-                                if e.errno == errno.ENOENT:
-                                    pass
-                os.remove(pcmanifest)
-            except IOError as e:
-                if e.errno == errno.ENOENT:
-                    pass
-}

meta/classes/populate_sdk_ext.bbclass

@@ -653,7 +653,10 @@ sdk_ext_postinst() {
 
 		# Make sure when the user sets up the environment, they also get
 		# the buildtools-tarball tools in their path.
+		echo "# Save and reset OECORE_NATIVE_SYSROOT as buildtools may change it" >> $env_setup_script
+		echo "SAVED=\"\$OECORE_NATIVE_SYSROOT\"" >> $env_setup_script
 		echo ". $target_sdk_dir/buildtools/environment-setup*" >> $env_setup_script
+		echo "OECORE_NATIVE_SYSROOT=\"\$SAVED\"" >> $env_setup_script
 	fi
 
 	# Allow bitbake environment setup to be ran as part of this sdk.
@@ -676,7 +679,7 @@ sdk_ext_postinst() {
 		# current working directory when first ran, nor will it set $1 when
 		# sourcing a script. That is why this has to look so ugly.
 		LOGFILE="$target_sdk_dir/preparing_build_system.log"
-		sh -c ". buildtools/environment-setup* > $LOGFILE && cd $target_sdk_dir/`dirname ${oe_init_build_env_path}` && set $target_sdk_dir && . $target_sdk_dir/${oe_init_build_env_path} $target_sdk_dir >> $LOGFILE && python $target_sdk_dir/ext-sdk-prepare.py $LOGFILE '${SDK_INSTALL_TARGETS}'" || { echo "printf 'ERROR: this SDK was not fully installed and needs reinstalling\n'" >> $env_setup_script ; exit 1 ; }
+		sh -c ". buildtools/environment-setup* > $LOGFILE && cd $target_sdk_dir/`dirname ${oe_init_build_env_path}` && set $target_sdk_dir && . $target_sdk_dir/${oe_init_build_env_path} $target_sdk_dir >> $LOGFILE && python3 $target_sdk_dir/ext-sdk-prepare.py $LOGFILE '${SDK_INSTALL_TARGETS}'" || { echo "printf 'ERROR: this SDK was not fully installed and needs reinstalling\n'" >> $env_setup_script ; exit 1 ; }
 	fi
 	if [ -e $target_sdk_dir/ext-sdk-prepare.py ]; then
 		rm $target_sdk_dir/ext-sdk-prepare.py

meta/classes/ptest.bbclass

@@ -6,7 +6,7 @@ PTEST_PATH ?= "${libdir}/${BPN}/ptest"
 PTEST_BUILD_HOST_FILES ?= "Makefile"
 PTEST_BUILD_HOST_PATTERN ?= ""
 
-FILES_${PN}-ptest = "${PTEST_PATH}"
+FILES_${PN}-ptest += "${PTEST_PATH}"
 SECTION_${PN}-ptest = "devel"
 ALLOW_EMPTY_${PN}-ptest = "1"
 PTEST_ENABLED = "${@bb.utils.contains('DISTRO_FEATURES', 'ptest', '1', '0', d)}"

meta/classes/qemuboot.bbclass

@@ -4,7 +4,7 @@
 #
 # QB_SYSTEM_NAME: qemu name, e.g., "qemu-system-i386"
 #
-# QB_OPT_APPEND: options to append to qemu, e.g., "-show-cursor"
+# QB_OPT_APPEND: options to append to qemu, e.g., "-device usb-mouse"
 #
 # QB_DEFAULT_KERNEL: default kernel to boot, e.g., "bzImage"
 #
@@ -77,7 +77,7 @@ QB_MEM ?= "-m 256"
 QB_SERIAL_OPT ?= "-serial mon:stdio -serial null"
 QB_DEFAULT_KERNEL ?= "${KERNEL_IMAGETYPE}"
 QB_DEFAULT_FSTYPE ?= "ext4"
-QB_OPT_APPEND ?= "-show-cursor"
+QB_OPT_APPEND ?= ""
 QB_NETWORK_DEVICE ?= "-device virtio-net-pci,netdev=net0,mac=@MAC@"
 QB_CMDLINE_IP_SLIRP ?= "ip=dhcp"
 QB_CMDLINE_IP_TAP ?= "ip=192.168.7.@CLIENT@::192.168.7.@GATEWAY@:255.255.255.0"

meta/classes/reproducible_build.bbclass

@@ -70,100 +70,16 @@ do_deploy_source_date_epoch[sstate-plaindirs] = "${SDE_DEPLOYDIR}"
 addtask do_deploy_source_date_epoch_setscene
 addtask do_deploy_source_date_epoch before do_configure after do_patch
 
-def get_source_date_epoch_from_known_files(d, sourcedir):
-    source_date_epoch = None
-    newest_file = None
-    known_files = set(["NEWS", "ChangeLog", "Changelog", "CHANGES"])
-    for file in known_files:
-        filepath = os.path.join(sourcedir, file)
-        if os.path.isfile(filepath):
-            mtime = int(os.lstat(filepath).st_mtime)
-            # There may be more than one "known_file" present, if so, use the youngest one
-            if not source_date_epoch or mtime > source_date_epoch:
-                source_date_epoch = mtime
-                newest_file = filepath
-    if newest_file:
-        bb.debug(1, "SOURCE_DATE_EPOCH taken from: %s" % newest_file)
-    return source_date_epoch
-
-def find_git_folder(d, sourcedir):
-    # First guess: WORKDIR/git
-    # This is the default git fetcher unpack path
-    workdir = d.getVar('WORKDIR')
-    gitpath = os.path.join(workdir, "git/.git")
-    if os.path.isdir(gitpath):
-        return gitpath
-
-    # Second guess: ${S}
-    gitpath = os.path.join(sourcedir, ".git")
-    if os.path.isdir(gitpath):
-        return gitpath
-
-    # Perhaps there was a subpath or destsuffix specified.
-    # Go looking in the WORKDIR
-    exclude = set(["build", "image", "license-destdir", "patches", "pseudo",
-                   "recipe-sysroot", "recipe-sysroot-native", "sysroot-destdir", "temp"])
-    for root, dirs, files in os.walk(workdir, topdown=True):
-        dirs[:] = [d for d in dirs if d not in exclude]
-        if '.git' in dirs:
-            return root
-
-    bb.warn("Failed to find a git repository in WORKDIR: %s" % workdir)
-    return None
-
-def get_source_date_epoch_from_git(d, sourcedir):
-    source_date_epoch = None
-    if "git://" in d.getVar('SRC_URI'):
-        gitpath = find_git_folder(d, sourcedir)
-        if gitpath:
-            import subprocess
-            source_date_epoch = int(subprocess.check_output(['git','log','-1','--pretty=%ct'], cwd=gitpath))
-            bb.debug(1, "git repository: %s" % gitpath)
-    return source_date_epoch
-
-def get_source_date_epoch_from_youngest_file(d, sourcedir):
-    if sourcedir == d.getVar('WORKDIR'):
-       # These sources are almost certainly not from a tarball
-       return None
-
-    # Do it the hard way: check all files and find the youngest one...
-    source_date_epoch = None
-    newest_file = None
-    for root, dirs, files in os.walk(sourcedir, topdown=True):
-        files = [f for f in files if not f[0] == '.']
-
-        for fname in files:
-            filename = os.path.join(root, fname)
-            try:
-                mtime = int(os.lstat(filename).st_mtime)
-            except ValueError:
-                mtime = 0
-            if not source_date_epoch or mtime > source_date_epoch:
-                source_date_epoch = mtime
-                newest_file = filename
-
-    if newest_file:
-        bb.debug(1, "Newest file found: %s" % newest_file)
-    return source_date_epoch
-
-def fixed_source_date_epoch():
-    bb.debug(1, "No tarball or git repo found to determine SOURCE_DATE_EPOCH")
-    return 0
-
 python create_source_date_epoch_stamp() {
+    import oe.reproducible
+
     epochfile = d.getVar('SDE_FILE')
     # If it exists we need to regenerate as the sources may have changed
     if os.path.isfile(epochfile):
         bb.debug(1, "Deleting existing SOURCE_DATE_EPOCH from: %s" % epochfile)
         os.remove(epochfile)
 
-    sourcedir = d.getVar('S')
-    source_date_epoch = (
-        get_source_date_epoch_from_git(d, sourcedir) or
-        get_source_date_epoch_from_known_files(d, sourcedir) or
-        get_source_date_epoch_from_youngest_file(d, sourcedir) or
-        fixed_source_date_epoch()       # Last resort
-    )
+    source_date_epoch = oe.reproducible.get_source_date_epoch(d, d.getVar('S'))
 
     bb.debug(1, "SOURCE_DATE_EPOCH: %d" % source_date_epoch)
     bb.utils.mkdirhier(d.getVar('SDE_DIR'))

meta/classes/rootfs-postcommands.bbclass

@@ -1,6 +1,6 @@
 
 # Zap the root password if debug-tweaks feature is not enabled
-ROOTFS_POSTPROCESS_COMMAND += '${@bb.utils.contains_any("IMAGE_FEATURES", [ 'debug-tweaks', 'empty-root-password' ], "", "zap_empty_root_password ; ",d)}'
+ROOTFS_POSTPROCESS_COMMAND += '${@bb.utils.contains_any("IMAGE_FEATURES", [ 'debug-tweaks', 'empty-root-password' ], "", "zap_empty_root_password; ",d)}'
 
 # Allow dropbear/openssh to accept logins from accounts with an empty password string if debug-tweaks or allow-empty-password is enabled
 ROOTFS_POSTPROCESS_COMMAND += '${@bb.utils.contains_any("IMAGE_FEATURES", [ 'debug-tweaks', 'allow-empty-password' ], "ssh_allow_empty_password; ", "",d)}'
@@ -12,7 +12,7 @@ ROOTFS_POSTPROCESS_COMMAND += '${@bb.utils.contains_any("IMAGE_FEATURES", [ 'deb
 ROOTFS_POSTPROCESS_COMMAND += '${@bb.utils.contains_any("IMAGE_FEATURES", [ 'debug-tweaks', 'post-install-logging' ], "postinst_enable_logging; ", "",d)}'
 
 # Create /etc/timestamp during image construction to give a reasonably sane default time setting
-ROOTFS_POSTPROCESS_COMMAND += "rootfs_update_timestamp ; "
+ROOTFS_POSTPROCESS_COMMAND += "rootfs_update_timestamp; "
 
 # Tweak the mount options for rootfs in /etc/fstab if read-only-rootfs is enabled
 ROOTFS_POSTPROCESS_COMMAND += '${@bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs", "read_only_rootfs_hook; ", "",d)}'
@@ -26,7 +26,7 @@ ROOTFS_POSTPROCESS_COMMAND += '${@bb.utils.contains("IMAGE_FEATURES", "read-only
 APPEND_append = '${@bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs", " ro", "", d)}'
 
 # Generates test data file with data store variables expanded in json format
-ROOTFS_POSTPROCESS_COMMAND += "write_image_test_data ; "
+ROOTFS_POSTPROCESS_COMMAND += "write_image_test_data; "
 
 # Write manifest
 IMAGE_MANIFEST = "${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.manifest"
@@ -308,12 +308,16 @@ rootfs_check_host_user_contaminated () {
 	HOST_USER_UID="$(PSEUDO_UNLOAD=1 id -u)"
 	HOST_USER_GID="$(PSEUDO_UNLOAD=1 id -g)"
 
-	find "${IMAGE_ROOTFS}" -wholename "${IMAGE_ROOTFS}/home" -prune \
-	    -user "$HOST_USER_UID" -o -group "$HOST_USER_GID" >"$contaminated"
+	find "${IMAGE_ROOTFS}" -path "${IMAGE_ROOTFS}/home" -prune -o \
+	    -user "$HOST_USER_UID" -print -o -group "$HOST_USER_GID" -print >"$contaminated"
+
+	sed -e "s,${IMAGE_ROOTFS},," $contaminated | while read line; do
+		bbwarn "Path in the rootfs is owned by the same user or group as the user running bitbake:" $line `ls -lan ${IMAGE_ROOTFS}/$line`
+	done
 
 	if [ -s "$contaminated" ]; then
-		echo "WARNING: Paths in the rootfs are owned by the same user or group as the user running bitbake. See the logfile for the specific paths."
-		cat "$contaminated" | sed "s,^,  ,"
+		bbwarn "/etc/passwd:" `cat ${IMAGE_ROOTFS}/etc/passwd`
+		bbwarn "/etc/group:" `cat ${IMAGE_ROOTFS}/etc/group`
 	fi
 }
 

meta/classes/rootfsdebugfiles.bbclass

@@ -28,7 +28,7 @@
 ROOTFS_DEBUG_FILES ?= ""
 ROOTFS_DEBUG_FILES[doc] = "Lists additional files or directories to be installed with 'cp -a' in the format 'source1 target1;source2 target2;...'"
 
-ROOTFS_POSTPROCESS_COMMAND += "rootfs_debug_files ;"
+ROOTFS_POSTPROCESS_COMMAND += "rootfs_debug_files;"
 rootfs_debug_files () {
    #!/bin/sh -e
    echo "${ROOTFS_DEBUG_FILES}" | sed -e 's/;/\n/g' | while read source target mode; do

meta/classes/spdx.bbclass

@@ -1,360 +0,0 @@
-# This class integrates real-time license scanning, generation of SPDX standard
-# output and verifiying license info during the building process.
-# It is a combination of efforts from the OE-Core, SPDX and Fossology projects.
-#
-# For more information on FOSSology:
-#   http://www.fossology.org
-#
-# For more information on FOSSologySPDX commandline:
-#   https://github.com/spdx-tools/fossology-spdx/wiki/Fossology-SPDX-Web-API
-#
-# For more information on SPDX:
-#   http://www.spdx.org
-#
-
-# SPDX file will be output to the path which is defined as[SPDX_MANIFEST_DIR] 
-# in ./meta/conf/licenses.conf.
-
-SPDXSSTATEDIR = "${WORKDIR}/spdx_sstate_dir"
-
-# If ${S} isn't actually the top-level source directory, set SPDX_S to point at
-# the real top-level directory.
-SPDX_S ?= "${S}"
-
-python do_spdx () {
-    import os, sys
-    import json, shutil
-
-    info = {} 
-    info['workdir'] = d.getVar('WORKDIR')
-    info['sourcedir'] = d.getVar('SPDX_S')
-    info['pn'] = d.getVar('PN')
-    info['pv'] = d.getVar('PV')
-    info['spdx_version'] = d.getVar('SPDX_VERSION')
-    info['data_license'] = d.getVar('DATA_LICENSE')
-
-    sstatedir = d.getVar('SPDXSSTATEDIR')
-    sstatefile = os.path.join(sstatedir, info['pn'] + info['pv'] + ".spdx")
-
-    manifest_dir = d.getVar('SPDX_MANIFEST_DIR')
-    info['outfile'] = os.path.join(manifest_dir, info['pn'] + ".spdx" )
-
-    info['spdx_temp_dir'] = d.getVar('SPDX_TEMP_DIR')
-    info['tar_file'] = os.path.join(info['workdir'], info['pn'] + ".tar.gz" )
-
-    # Make sure important dirs exist
-    try:
-        bb.utils.mkdirhier(manifest_dir)
-        bb.utils.mkdirhier(sstatedir)
-        bb.utils.mkdirhier(info['spdx_temp_dir'])
-    except OSError as e:
-        bb.error("SPDX: Could not set up required directories: " + str(e))
-        return
-
-    ## get everything from cache.  use it to decide if 
-    ## something needs to be rerun 
-    cur_ver_code = get_ver_code(info['sourcedir'])
-    cache_cur = False
-    if os.path.exists(sstatefile):
-        ## cache for this package exists. read it in
-        cached_spdx = get_cached_spdx(sstatefile)
-
-        if cached_spdx['PackageVerificationCode'] == cur_ver_code:
-            bb.warn("SPDX: Verification code for " + info['pn']
-                  + "is same as cache's. do nothing")
-            cache_cur = True
-        else:
-            local_file_info = setup_foss_scan(info, True, cached_spdx['Files'])
-    else:
-        local_file_info = setup_foss_scan(info, False, None)
-
-    if cache_cur:
-        spdx_file_info = cached_spdx['Files']
-        foss_package_info = cached_spdx['Package']
-        foss_license_info = cached_spdx['Licenses']
-    else:
-        ## setup fossology command
-        foss_server = d.getVar('FOSS_SERVER')
-        foss_flags = d.getVar('FOSS_WGET_FLAGS')
-        foss_full_spdx = d.getVar('FOSS_FULL_SPDX') == "true" or False
-        foss_command = "wget %s --post-file=%s %s"\
-            % (foss_flags, info['tar_file'], foss_server)
-        
-        foss_result = run_fossology(foss_command, foss_full_spdx)
-        if foss_result is not None:
-            (foss_package_info, foss_file_info, foss_license_info) = foss_result
-            spdx_file_info = create_spdx_doc(local_file_info, foss_file_info)
-            ## write to cache
-            write_cached_spdx(sstatefile, cur_ver_code, foss_package_info,
-                              spdx_file_info, foss_license_info)
-        else:
-            bb.error("SPDX: Could not communicate with FOSSology server. Command was: " + foss_command)
-            return
-    
-    ## Get document and package level information
-    spdx_header_info = get_header_info(info, cur_ver_code, foss_package_info)
-    
-    ## CREATE MANIFEST
-    create_manifest(info, spdx_header_info, spdx_file_info, foss_license_info)
-
-    ## clean up the temp stuff
-    shutil.rmtree(info['spdx_temp_dir'], ignore_errors=True)
-    if os.path.exists(info['tar_file']):
-        remove_file(info['tar_file'])
-}
-addtask spdx after do_patch before do_configure
-
-def create_manifest(info, header, files, licenses):
-    import codecs
-    with codecs.open(info['outfile'], mode='w', encoding='utf-8') as f:
-        # Write header
-        f.write(header + '\n')
-
-        # Write file data
-        for chksum, block in files.iteritems():
-            f.write("FileName: " + block['FileName'] + '\n')
-            for key, value in block.iteritems():
-                if not key == 'FileName':
-                    f.write(key + ": " + value + '\n')
-            f.write('\n')
-
-        # Write license data
-        for id, block in licenses.iteritems():
-            f.write("LicenseID: " + id + '\n')
-            for key, value in block.iteritems():
-                f.write(key + ": " + value + '\n')
-            f.write('\n')
-
-def get_cached_spdx(sstatefile):
-    import json
-    import codecs
-    cached_spdx_info = {}
-    with codecs.open(sstatefile, mode='r', encoding='utf-8') as f:
-        try:
-            cached_spdx_info = json.load(f)
-        except ValueError as e:
-            cached_spdx_info = None
-    return cached_spdx_info
-
-def write_cached_spdx(sstatefile, ver_code, package_info, files, license_info):
-    import json
-    import codecs
-    spdx_doc = {}
-    spdx_doc['PackageVerificationCode'] = ver_code
-    spdx_doc['Files'] = {}
-    spdx_doc['Files'] = files
-    spdx_doc['Package'] = {}
-    spdx_doc['Package'] = package_info
-    spdx_doc['Licenses'] = {}
-    spdx_doc['Licenses'] = license_info
-    with codecs.open(sstatefile, mode='w', encoding='utf-8') as f:
-        f.write(json.dumps(spdx_doc))
-
-def setup_foss_scan(info, cache, cached_files):
-    import errno, shutil
-    import tarfile
-    file_info = {}
-    cache_dict = {}
-
-    for f_dir, f in list_files(info['sourcedir']):
-        full_path = os.path.join(f_dir, f)
-        abs_path = os.path.join(info['sourcedir'], full_path)
-        dest_dir = os.path.join(info['spdx_temp_dir'], f_dir)
-        dest_path = os.path.join(info['spdx_temp_dir'], full_path)
-
-        checksum = hash_file(abs_path)
-        if not checksum is None:
-            file_info[checksum] = {}
-            ## retain cache information if it exists
-            if cache and checksum in cached_files:
-                file_info[checksum] = cached_files[checksum]
-            ## have the file included in what's sent to the FOSSology server
-            else:
-                file_info[checksum]['FileName'] = full_path
-                try:
-                    bb.utils.mkdirhier(dest_dir)
-                    shutil.copyfile(abs_path, dest_path)
-                except OSError as e:
-                    bb.warn("SPDX: mkdirhier failed: " + str(e))
-                except shutil.Error as e:
-                    bb.warn("SPDX: copyfile failed: " + str(e))
-                except IOError as e:
-                    bb.warn("SPDX: copyfile failed: " + str(e))
-        else:
-            bb.warn("SPDX: Could not get checksum for file: " + f)
-    
-    with tarfile.open(info['tar_file'], "w:gz") as tar:
-        tar.add(info['spdx_temp_dir'], arcname=os.path.basename(info['spdx_temp_dir']))
-    
-    return file_info
-
-def remove_file(file_name):
-    try:
-        os.remove(file_name)
-    except OSError as e:
-        pass
-
-def list_files(dir):
-    for root, subFolders, files in os.walk(dir):
-        for f in files:
-            rel_root = os.path.relpath(root, dir)
-            yield rel_root, f
-    return
-
-def hash_file(file_name):
-    from bb.utils import sha1_file
-    return sha1_file(file_name)
-
-def hash_string(data):
-    import hashlib
-    sha1 = hashlib.sha1()
-    sha1.update(data.encode('utf-8'))
-    return sha1.hexdigest()
-
-def run_fossology(foss_command, full_spdx):
-    import string, re
-    import subprocess
-
-    try:
-        foss_output = subprocess.check_output(foss_command.split(),
-                stderr=subprocess.STDOUT).decode('utf-8')
-    except subprocess.CalledProcessError as e:
-        return None
-
-    foss_output = foss_output.replace('\r', '')
-
-    # Package info
-    package_info = {}
-    if full_spdx:
-        # All mandatory, only one occurrence
-        package_info['PackageCopyrightText'] = re.findall('PackageCopyrightText: (.*?</text>)', foss_output, re.S)[0]
-        package_info['PackageLicenseDeclared'] = re.findall('PackageLicenseDeclared: (.*)', foss_output)[0]
-        package_info['PackageLicenseConcluded'] = re.findall('PackageLicenseConcluded: (.*)', foss_output)[0]
-        # These may be more than one
-        package_info['PackageLicenseInfoFromFiles'] = re.findall('PackageLicenseInfoFromFiles: (.*)', foss_output)
-    else:
-        DEFAULT = "NOASSERTION"
-        package_info['PackageCopyrightText'] = "<text>" + DEFAULT + "</text>"
-        package_info['PackageLicenseDeclared'] = DEFAULT
-        package_info['PackageLicenseConcluded'] = DEFAULT
-        package_info['PackageLicenseInfoFromFiles'] = []
-
-    # File info
-    file_info = {}
-    records = []
-    # FileName is also in PackageFileName, so we match on FileType as well.
-    records = re.findall('FileName:.*?FileType:.*?</text>', foss_output, re.S)
-    for rec in records:
-        chksum = re.findall('FileChecksum: SHA1: (.*)\n', rec)[0]
-        file_info[chksum] = {}
-        file_info[chksum]['FileCopyrightText'] = re.findall('FileCopyrightText: '
-            + '(.*?</text>)', rec, re.S )[0]
-        fields = ['FileName', 'FileType', 'LicenseConcluded', 'LicenseInfoInFile']
-        for field in fields:
-            file_info[chksum][field] = re.findall(field + ': (.*)', rec)[0]
-
-    # Licenses
-    license_info = {}
-    licenses = []
-    licenses = re.findall('LicenseID:.*?LicenseName:.*?\n', foss_output, re.S)
-    for lic in licenses:
-        license_id = re.findall('LicenseID: (.*)\n', lic)[0]
-        license_info[license_id] = {}
-        license_info[license_id]['ExtractedText'] = re.findall('ExtractedText: (.*?</text>)', lic, re.S)[0]
-        license_info[license_id]['LicenseName'] = re.findall('LicenseName: (.*)', lic)[0]
-
-    return (package_info, file_info, license_info)
-
-def create_spdx_doc(file_info, scanned_files):
-    import json
-    ## push foss changes back into cache
-    for chksum, lic_info in scanned_files.iteritems():
-        if chksum in file_info:
-            file_info[chksum]['FileType'] = lic_info['FileType']
-            file_info[chksum]['FileChecksum: SHA1'] = chksum
-            file_info[chksum]['LicenseInfoInFile'] = lic_info['LicenseInfoInFile']
-            file_info[chksum]['LicenseConcluded'] = lic_info['LicenseConcluded']
-            file_info[chksum]['FileCopyrightText'] = lic_info['FileCopyrightText']
-        else:
-            bb.warn("SPDX: " + lic_info['FileName'] + " : " + chksum
-                + " : is not in the local file info: "
-                + json.dumps(lic_info, indent=1))
-    return file_info
-
-def get_ver_code(dirname):
-    chksums = []
-    for f_dir, f in list_files(dirname):
-        path = os.path.join(dirname, f_dir, f)
-        hash = hash_file(path)
-        if not hash is None:
-            chksums.append(hash)
-        else:
-            bb.warn("SPDX: Could not hash file: " + path)
-    ver_code_string = ''.join(chksums).lower()
-    ver_code = hash_string(ver_code_string)
-    return ver_code
-
-def get_header_info(info, spdx_verification_code, package_info):
-    """
-        Put together the header SPDX information.
-        Eventually this needs to become a lot less
-        of a hardcoded thing.
-    """
-    from datetime import datetime
-    import os
-    head = []
-    DEFAULT = "NOASSERTION"
-
-    package_checksum = hash_file(info['tar_file'])
-    if package_checksum is None:
-        package_checksum = DEFAULT
-
-    ## document level information
-    head.append("## SPDX Document Information")
-    head.append("SPDXVersion: " + info['spdx_version'])
-    head.append("DataLicense: " + info['data_license'])
-    head.append("DocumentComment: <text>SPDX for "
-        + info['pn'] + " version " + info['pv'] + "</text>")
-    head.append("")
-
-    ## Creator information
-    ## Note that this does not give time in UTC.
-    now = datetime.now().strftime('%Y-%m-%dT%H:%M:%SZ')
-    head.append("## Creation Information")
-    ## Tools are supposed to have a version, but FOSSology+SPDX provides none.
-    head.append("Creator: Tool: FOSSology+SPDX")
-    head.append("Created: " + now)
-    head.append("CreatorComment: <text>UNO</text>")
-    head.append("")
-
-    ## package level information
-    head.append("## Package Information")
-    head.append("PackageName: " + info['pn'])
-    head.append("PackageVersion: " + info['pv'])
-    head.append("PackageFileName: " + os.path.basename(info['tar_file']))
-    head.append("PackageSupplier: Person:" + DEFAULT)
-    head.append("PackageDownloadLocation: " + DEFAULT)
-    head.append("PackageSummary: <text></text>")
-    head.append("PackageOriginator: Person:" + DEFAULT)
-    head.append("PackageChecksum: SHA1: " + package_checksum)
-    head.append("PackageVerificationCode: " + spdx_verification_code)
-    head.append("PackageDescription: <text>" + info['pn']
-        + " version " + info['pv'] + "</text>")
-    head.append("")
-    head.append("PackageCopyrightText: "
-        + package_info['PackageCopyrightText'])
-    head.append("")
-    head.append("PackageLicenseDeclared: "
-        + package_info['PackageLicenseDeclared'])
-    head.append("PackageLicenseConcluded: "
-        + package_info['PackageLicenseConcluded'])
-
-    for licref in package_info['PackageLicenseInfoFromFiles']:
-        head.append("PackageLicenseInfoFromFiles: " + licref)
-    head.append("")
-    
-    ## header for file level
-    head.append("## File Information")
-    head.append("")
-
-    return '\n'.join(head)

meta/classes/uboot-sign.bbclass

@@ -117,15 +117,16 @@ do_install_append() {
 	fi
 }
 
+do_deploy_prepend_pn-${UBOOT_PN}() {
+	if [ "${UBOOT_SIGN_ENABLE}" = "1" -a -n "${UBOOT_DTB_BINARY}" ]; then
+		concat_dtb
+	fi
+}
+
 python () {
     if d.getVar('UBOOT_SIGN_ENABLE') == '1' and d.getVar('PN') == d.getVar('UBOOT_PN') and d.getVar('UBOOT_DTB_BINARY'):
         kernel_pn = d.getVar('PREFERRED_PROVIDER_virtual/kernel')
 
         # Make "bitbake u-boot -cdeploy" deploys the signed u-boot.dtb
         d.appendVarFlag('do_deploy', 'depends', ' %s:do_deploy' % kernel_pn)
-
-        # kernerl's do_deploy is a litle special, so we can't use
-        # do_deploy_append, otherwise it would override
-        # kernel_do_deploy.
-        d.appendVarFlag('do_deploy', 'prefuncs', ' concat_dtb')
 }

meta/classes/uninative.bbclass

@@ -56,12 +56,17 @@ python uninative_event_fetchloader() {
             # Our games with path manipulation of DL_DIR mean standard PREMIRRORS don't work
             # and we can't easily put 'chksum' into the url path from a url parameter with
             # the current fetcher url handling
-            ownmirror = d.getVar('SOURCE_MIRROR_URL')
-            if ownmirror:
-                localdata.appendVar("PREMIRRORS", " ${UNINATIVE_URL}${UNINATIVE_TARBALL} ${SOURCE_MIRROR_URL}/uninative/%s/${UNINATIVE_TARBALL}" % chksum)
+            premirrors = bb.fetch2.mirror_from_string(localdata.getVar("PREMIRRORS"))
+            for line in premirrors:
+                try:
+                    (find, replace) = line
+                except ValueError:
+                    continue
+                if find.startswith("http"):
+                    localdata.appendVar("PREMIRRORS", " ${UNINATIVE_URL}${UNINATIVE_TARBALL} %s/uninative/%s/${UNINATIVE_TARBALL}" % (replace, chksum))
 
             srcuri = d.expand("${UNINATIVE_URL}${UNINATIVE_TARBALL};sha256sum=%s" % chksum)
-            bb.note("Fetching uninative binary shim from %s" % srcuri)
+            bb.note("Fetching uninative binary shim %s (will check PREMIRRORS first)" % srcuri)
 
             fetcher = bb.fetch2.Fetch([srcuri], localdata, cache=False)
             fetcher.download()

meta/conf/bitbake.conf

@@ -252,7 +252,7 @@ This package contains symbolic links, header files, and \
 related items necessary for software development."
 
 SUMMARY_${PN}-staticdev ?= "${SUMMARY} - Development files (Static Libraries)"
-DESCRIPTION_${PN}-staticdev?= "${DESCRIPTION}  \
+DESCRIPTION_${PN}-staticdev ?= "${DESCRIPTION}  \
 This package contains static libraries for software development."
 
 SUMMARY_${PN}-doc ?= "${SUMMARY} - Documentation files"
@@ -267,7 +267,7 @@ HOMEPAGE = ""
 
 # Ensure that -dev packages recommend the corresponding -dev packages of their
 # deps, and the same for -dbg.
-DEPCHAIN_PRE  = ""
+DEPCHAIN_PRE = ""
 DEPCHAIN_POST = "-dev -dbg"
 
 DEPENDS = ""