build-appliance-image: Update to master head revision

(From OE-Core rev: 0f04e81c797d5d337ece4e8638a6b71c75bc0a00)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

bitbake/bin/bitbake-worker

@@ -195,9 +195,6 @@ def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, taskha
             global worker_pipe_lock
             pipein.close()
 
-            signal.signal(signal.SIGTERM, sigterm_handler)
-            # Let SIGHUP exit as SIGTERM
-            signal.signal(signal.SIGHUP, sigterm_handler)
             bb.utils.signal_on_parent_exit("SIGTERM")
 
             # Save out the PID so that the event can include it the
@@ -212,6 +209,11 @@ def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, taskha
             # This ensures signals sent to the controlling terminal like Ctrl+C
             # don't stop the child processes.
             os.setsid()
+
+            signal.signal(signal.SIGTERM, sigterm_handler)
+            # Let SIGHUP exit as SIGTERM
+            signal.signal(signal.SIGHUP, sigterm_handler)
+
             # No stdin
             newsi = os.open(os.devnull, os.O_RDWR)
             os.dup2(newsi, sys.stdin.fileno())

bitbake/contrib/vim/indent/bitbake.vim

@@ -0,0 +1,343 @@
+" Vim indent file
+" Language:             BitBake
+" Copyright:            Copyright (C) 2019 Agilent Technologies, Inc.
+" Maintainer:           Chris Laplante <chris.laplante@agilent.com>
+" License:              You may redistribute this under the same terms as Vim itself
+
+
+if exists("b:did_indent")
+    finish
+endif
+
+if exists("*BitbakeIndent")
+    finish
+endif
+
+runtime! indent/sh.vim
+unlet b:did_indent
+
+setlocal indentexpr=BitbakeIndent(v:lnum)
+setlocal autoindent nolisp
+
+function s:is_bb_python_func_def(lnum)
+    let stack = synstack(a:lnum, 1)
+    if len(stack) == 0
+        return 0
+    endif
+
+    let top = synIDattr(stack[0], "name")
+    echo top
+
+    return synIDattr(stack[0], "name") == "bbPyFuncDef"
+endfunction
+
+"""" begin modified from indent/python.vim, upstream commit 7a9bd7c1e0ce1baf5a02daf36eeae3638aa315c7
+"""" This copied code is licensed the same as Vim itself.
+setlocal indentkeys+=<:>,=elif,=except
+
+let s:keepcpo= &cpo
+set cpo&vim
+
+let s:maxoff = 50	" maximum number of lines to look backwards for ()
+
+function GetPythonIndent(lnum)
+
+  " If this line is explicitly joined: If the previous line was also joined,
+  " line it up with that one, otherwise add two 'shiftwidth'
+  if getline(a:lnum - 1) =~ '\\$'
+    if a:lnum > 1 && getline(a:lnum - 2) =~ '\\$'
+      return indent(a:lnum - 1)
+    endif
+    return indent(a:lnum - 1) + (exists("g:pyindent_continue") ? eval(g:pyindent_continue) : (shiftwidth() * 2))
+  endif
+
+  " If the start of the line is in a string don't change the indent.
+  if has('syntax_items')
+	\ && synIDattr(synID(a:lnum, 1, 1), "name") =~ "String$"
+    return -1
+  endif
+
+  " Search backwards for the previous non-empty line.
+  let plnum = prevnonblank(v:lnum - 1)
+
+  if plnum == 0
+    " This is the first non-empty line, use zero indent.
+    return 0
+  endif
+
+  call cursor(plnum, 1)
+
+  " Identing inside parentheses can be very slow, regardless of the searchpair()
+  " timeout, so let the user disable this feature if he doesn't need it
+  let disable_parentheses_indenting = get(g:, "pyindent_disable_parentheses_indenting", 0)
+
+  if disable_parentheses_indenting == 1
+    let plindent = indent(plnum)
+    let plnumstart = plnum
+  else
+    " searchpair() can be slow sometimes, limit the time to 150 msec or what is
+    " put in g:pyindent_searchpair_timeout
+    let searchpair_stopline = 0
+    let searchpair_timeout = get(g:, 'pyindent_searchpair_timeout', 150)
+
+    " If the previous line is inside parenthesis, use the indent of the starting
+    " line.
+    " Trick: use the non-existing "dummy" variable to break out of the loop when
+    " going too far back.
+    let parlnum = searchpair('(\|{\|\[', '', ')\|}\|\]', 'nbW',
+            \ "line('.') < " . (plnum - s:maxoff) . " ? dummy :"
+            \ . " synIDattr(synID(line('.'), col('.'), 1), 'name')"
+            \ . " =~ '\\(Comment\\|Todo\\|String\\)$'",
+            \ searchpair_stopline, searchpair_timeout)
+    if parlnum > 0
+      " We may have found the opening brace of a BitBake Python task, e.g. 'python do_task {'
+      " If so, ignore it here - it will be handled later.
+      if s:is_bb_python_func_def(parlnum)
+        let parlnum = 0
+        let plindent = indent(plnum)
+        let plnumstart = plnum
+      else
+        let plindent = indent(parlnum)
+        let plnumstart = parlnum
+      endif
+    else
+      let plindent = indent(plnum)
+      let plnumstart = plnum
+    endif
+
+    " When inside parenthesis: If at the first line below the parenthesis add
+    " two 'shiftwidth', otherwise same as previous line.
+    " i = (a
+    "       + b
+    "       + c)
+    call cursor(a:lnum, 1)
+    let p = searchpair('(\|{\|\[', '', ')\|}\|\]', 'bW',
+            \ "line('.') < " . (a:lnum - s:maxoff) . " ? dummy :"
+            \ . " synIDattr(synID(line('.'), col('.'), 1), 'name')"
+            \ . " =~ '\\(Comment\\|Todo\\|String\\)$'",
+            \ searchpair_stopline, searchpair_timeout)
+    if p > 0
+        if s:is_bb_python_func_def(p)
+          " Handle first non-empty line inside a BB Python task
+          if p == plnum
+              return shiftwidth()
+          endif
+
+          " Handle the user actually trying to close a BitBake Python task
+          let line = getline(a:lnum)
+          if line =~ '^\s*}'
+              return -2
+          endif
+
+          " Otherwise ignore the brace
+          let p = 0
+        else
+          if p == plnum
+            " When the start is inside parenthesis, only indent one 'shiftwidth'.
+            let pp = searchpair('(\|{\|\[', '', ')\|}\|\]', 'bW',
+                \ "line('.') < " . (a:lnum - s:maxoff) . " ? dummy :"
+                \ . " synIDattr(synID(line('.'), col('.'), 1), 'name')"
+                \ . " =~ '\\(Comment\\|Todo\\|String\\)$'",
+                \ searchpair_stopline, searchpair_timeout)
+            if pp > 0
+              return indent(plnum) + (exists("g:pyindent_nested_paren") ? eval(g:pyindent_nested_paren) : shiftwidth())
+            endif
+            return indent(plnum) + (exists("g:pyindent_open_paren") ? eval(g:pyindent_open_paren) : (shiftwidth() * 2))
+          endif
+          if plnumstart == p
+            return indent(plnum)
+          endif
+          return plindent
+        endif
+    endif
+
+  endif
+
+
+  " Get the line and remove a trailing comment.
+  " Use syntax highlighting attributes when possible.
+  let pline = getline(plnum)
+  let pline_len = strlen(pline)
+  if has('syntax_items')
+    " If the last character in the line is a comment, do a binary search for
+    " the start of the comment.  synID() is slow, a linear search would take
+    " too long on a long line.
+    if synIDattr(synID(plnum, pline_len, 1), "name") =~ "\\(Comment\\|Todo\\)$"
+      let min = 1
+      let max = pline_len
+      while min < max
+	let col = (min + max) / 2
+	if synIDattr(synID(plnum, col, 1), "name") =~ "\\(Comment\\|Todo\\)$"
+	  let max = col
+	else
+	  let min = col + 1
+	endif
+      endwhile
+      let pline = strpart(pline, 0, min - 1)
+    endif
+  else
+    let col = 0
+    while col < pline_len
+      if pline[col] == '#'
+	let pline = strpart(pline, 0, col)
+	break
+      endif
+      let col = col + 1
+    endwhile
+  endif
+
+  " If the previous line ended with a colon, indent this line
+  if pline =~ ':\s*$'
+    return plindent + shiftwidth()
+  endif
+
+  " If the previous line was a stop-execution statement...
+  " TODO: utilize this logic to deindent when ending a bbPyDefRegion
+  if getline(plnum) =~ '^\s*\(break\|continue\|raise\|return\|pass\|bb\.fatal\)\>'
+    " See if the user has already dedented
+    if indent(a:lnum) > indent(plnum) - shiftwidth()
+      " If not, recommend one dedent
+      return indent(plnum) - shiftwidth()
+    endif
+    " Otherwise, trust the user
+    return -1
+  endif
+
+  " If the current line begins with a keyword that lines up with "try"
+  if getline(a:lnum) =~ '^\s*\(except\|finally\)\>'
+    let lnum = a:lnum - 1
+    while lnum >= 1
+      if getline(lnum) =~ '^\s*\(try\|except\)\>'
+	let ind = indent(lnum)
+	if ind >= indent(a:lnum)
+	  return -1	" indent is already less than this
+	endif
+	return ind	" line up with previous try or except
+      endif
+      let lnum = lnum - 1
+    endwhile
+    return -1		" no matching "try"!
+  endif
+
+  " If the current line begins with a header keyword, dedent
+  if getline(a:lnum) =~ '^\s*\(elif\|else\)\>'
+
+    " Unless the previous line was a one-liner
+    if getline(plnumstart) =~ '^\s*\(for\|if\|try\)\>'
+      return plindent
+    endif
+
+    " Or the user has already dedented
+    if indent(a:lnum) <= plindent - shiftwidth()
+      return -1
+    endif
+
+    return plindent - shiftwidth()
+  endif
+
+  " When after a () construct we probably want to go back to the start line.
+  " a = (b
+  "       + c)
+  " here
+  if parlnum > 0
+    return plindent
+  endif
+
+  return -1
+
+endfunction
+
+let &cpo = s:keepcpo
+unlet s:keepcpo
+
+""" end of stuff from indent/python.vim
+
+
+let b:did_indent = 1
+setlocal indentkeys+=0\"
+
+
+function BitbakeIndent(lnum)
+    if !has('syntax_items')
+        return -1
+    endif
+
+    let stack = synstack(a:lnum, 1)
+    if len(stack) == 0
+        return -1
+    endif
+
+    let name = synIDattr(stack[0], "name")
+
+    " TODO: support different styles of indentation for assignments. For now,
+    " we only support like this:
+    " VAR = " \
+    "     value1 \
+    "     value2 \
+    " "
+    "
+    " i.e. each value indented by shiftwidth(), with the final quote " completely unindented.
+    if name == "bbVarValue"
+        " Quote handling is tricky. kernel.bbclass has this line for instance:
+        "     EXTRA_OEMAKE = " HOSTCC="${BUILD_CC} ${BUILD_CFLAGS} ${BUILD_LDFLAGS}" " HOSTCPP="${BUILD_CPP}""
+        " Instead of trying to handle crazy cases like that, just assume that a
+        " double-quote on a line by itself (following an assignment) means the
+        " user is closing the assignment, and de-dent.
+        if getline(a:lnum) =~ '^\s*"$'
+            return 0
+        endif
+
+        let prevstack = synstack(a:lnum - 1, 1)
+        if len(prevstack) == 0
+            return -1
+        endif
+
+        let prevname = synIDattr(prevstack[0], "name")
+
+        " Only indent if there was actually a continuation character on
+        " the previous line, to avoid misleading indentation.
+        let prevlinelastchar = synIDattr(synID(a:lnum - 1, col([a:lnum - 1, "$"]) - 1, 1), "name")
+        let prev_continued = prevlinelastchar == "bbContinue"
+
+        " Did the previous line introduce an assignment?
+        if index(["bbVarDef", "bbVarFlagDef"], prevname) != -1
+            if prev_continued
+                return shiftwidth()
+            endif
+        endif
+
+        if !prev_continued
+            return 0
+        endif
+
+        " Autoindent can take it from here
+        return -1
+    endif
+
+    if index(["bbPyDefRegion", "bbPyFuncRegion"], name) != -1
+        let ret = GetPythonIndent(a:lnum)
+        " Should normally always be indented by at least one shiftwidth; but allow
+        " return of -1 (defer to autoindent) or -2 (force indent to 0)
+        if ret == 0
+            return shiftwidth()
+        elseif ret == -2
+            return 0
+        endif
+        return ret
+    endif
+
+    " TODO: GetShIndent doesn't detect tasks prepended with 'fakeroot'
+    " Need to submit a patch upstream to Vim to provide an extension point.
+    " Unlike the Python indenter, the Sh indenter is way too large to copy and
+    " modify here.
+    if name == "bbShFuncRegion"
+        return GetShIndent()
+    endif
+
+    " TODO:
+    "   + heuristics for de-denting out of a bbPyDefRegion? e.g. when the user
+    "       types an obvious BB keyword like addhandler or addtask, or starts
+    "       writing a shell task. Maybe too hard to implement...
+
+    return -1
+endfunction

bitbake/lib/bb/build.py

@@ -57,8 +57,9 @@ builtins['os'] = os
 class TaskBase(event.Event):
     """Base class for task events"""
 
-    def __init__(self, t, logfile, d):
+    def __init__(self, t, fn, logfile, d):
         self._task = t
+        self._fn = fn
         self._package = d.getVar("PF")
         self._mc = d.getVar("BB_CURRENT_MC")
         self.taskfile = d.getVar("FILE")
@@ -81,8 +82,8 @@ class TaskBase(event.Event):
 
 class TaskStarted(TaskBase):
     """Task execution started"""
-    def __init__(self, t, logfile, taskflags, d):
-        super(TaskStarted, self).__init__(t, logfile, d)
+    def __init__(self, t, fn, logfile, taskflags, d):
+        super(TaskStarted, self).__init__(t, fn, logfile, d)
         self.taskflags = taskflags
 
 class TaskSucceeded(TaskBase):
@@ -91,9 +92,9 @@ class TaskSucceeded(TaskBase):
 class TaskFailed(TaskBase):
     """Task execution failed"""
 
-    def __init__(self, task, logfile, metadata, errprinted = False):
+    def __init__(self, task, fn, logfile, metadata, errprinted = False):
         self.errprinted = errprinted
-        super(TaskFailed, self).__init__(task, logfile, metadata)
+        super(TaskFailed, self).__init__(task, fn, logfile, metadata)
 
 class TaskFailedSilent(TaskBase):
     """Task execution failed (silently)"""
@@ -103,8 +104,8 @@ class TaskFailedSilent(TaskBase):
 
 class TaskInvalid(TaskBase):
 
-    def __init__(self, task, metadata):
-        super(TaskInvalid, self).__init__(task, None, metadata)
+    def __init__(self, task, fn, metadata):
+        super(TaskInvalid, self).__init__(task, fn, None, metadata)
         self._message = "No such task '%s'" % task
 
 class TaskProgress(event.Event):
@@ -572,7 +573,7 @@ def _exec_task(fn, task, d, quieterr):
 
     try:
         try:
-            event.fire(TaskStarted(task, logfn, flags, localdata), localdata)
+            event.fire(TaskStarted(task, fn, logfn, flags, localdata), localdata)
         except (bb.BBHandledException, SystemExit):
             return 1
 
@@ -583,15 +584,15 @@ def _exec_task(fn, task, d, quieterr):
             for func in (postfuncs or '').split():
                 exec_func(func, localdata)
         except bb.BBHandledException:
-            event.fire(TaskFailed(task, logfn, localdata, True), localdata)
+            event.fire(TaskFailed(task, fn, logfn, localdata, True), localdata)
             return 1
         except Exception as exc:
             if quieterr:
-                event.fire(TaskFailedSilent(task, logfn, localdata), localdata)
+                event.fire(TaskFailedSilent(task, fn, logfn, localdata), localdata)
             else:
                 errprinted = errchk.triggered
                 logger.error(str(exc))
-                event.fire(TaskFailed(task, logfn, localdata, errprinted), localdata)
+                event.fire(TaskFailed(task, fn, logfn, localdata, errprinted), localdata)
             return 1
     finally:
         sys.stdout.flush()
@@ -614,7 +615,7 @@ def _exec_task(fn, task, d, quieterr):
             logger.debug(2, "Zero size logfn %s, removing", logfn)
             bb.utils.remove(logfn)
             bb.utils.remove(loglink)
-    event.fire(TaskSucceeded(task, logfn, localdata), localdata)
+    event.fire(TaskSucceeded(task, fn, logfn, localdata), localdata)
 
     if not localdata.getVarFlag(task, 'nostamp', False) and not localdata.getVarFlag(task, 'selfstamp', False):
         make_stamp(task, localdata)

bitbake/lib/bb/cooker.py

@@ -371,10 +371,6 @@ class BBCooker:
 
         self.data.setVar('BB_CMDLINE', self.ui_cmdline)
 
-        #
-        # Copy of the data store which has been expanded.
-        # Used for firing events and accessing variables where expansion needs to be accounted for
-        #
         if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset:
             self.disableDataTracking()
 

bitbake/lib/bb/fetch2/__init__.py

@@ -1593,7 +1593,7 @@ class Fetch(object):
         fn = d.getVar('FILE')
         mc = d.getVar('__BBMULTICONFIG') or ""
         if cache and fn and mc + fn in urldata_cache:
-            self.ud = urldata_cache[mc + fn]
+            self.ud = urldata_cache[mc + fn + str(id(d))]
 
         for url in urls:
             if url not in self.ud:
@@ -1605,7 +1605,7 @@ class Fetch(object):
                         pass
 
         if fn and cache:
-            urldata_cache[mc + fn] = self.ud
+            urldata_cache[mc + fn + str(id(d))] = self.ud
 
     def localpath(self, url):
         if url not in self.urls:

bitbake/lib/bb/fetch2/clearcase.py

@@ -145,18 +145,18 @@ class ClearCase(FetchMethod):
 
         basecmd = "%s %s" % (ud.basecmd, command)
 
-        if command is 'mkview':
+        if command == 'mkview':
             if not "rcleartool" in ud.basecmd:
                 # Cleartool needs a -snapshot view
                 options.append("-snapshot")
             options.append("-tag %s" % ud.viewname)
             options.append(ud.viewdir)
 
-        elif command is 'rmview':
+        elif command == 'rmview':
             options.append("-force")
             options.append("%s" % ud.viewdir)
 
-        elif command is 'setcs':
+        elif command == 'setcs':
             options.append("-overwrite")
             options.append(ud.configspecfile)
 

bitbake/lib/bb/fetch2/git.py

@@ -292,11 +292,21 @@ class Git(FetchMethod):
     def clonedir_need_update(self, ud, d):
         if not os.path.exists(ud.clonedir):
             return True
+        if ud.shallow and ud.write_shallow_tarballs and self.clonedir_need_shallow_revs(ud, d):
+            return True
         for name in ud.names:
             if not self._contains_ref(ud, d, name, ud.clonedir):
                 return True
         return False
 
+    def clonedir_need_shallow_revs(self, ud, d):
+        for rev in ud.shallow_revs:
+            try:
+                runfetchcmd('%s rev-parse -q --verify %s' % (ud.basecmd, rev), d, quiet=True, workdir=ud.clonedir)
+            except bb.fetch2.FetchError:
+                return rev
+        return None
+
     def shallow_tarball_need_update(self, ud):
         return ud.shallow and ud.write_shallow_tarballs and not os.path.exists(ud.fullshallow)
 
@@ -339,13 +349,7 @@ class Git(FetchMethod):
             runfetchcmd(clone_cmd, d, log=progresshandler)
 
         # Update the checkout if needed
-        needupdate = False
-        for name in ud.names:
-            if not self._contains_ref(ud, d, name, ud.clonedir):
-                needupdate = True
-                break
-
-        if needupdate:
+        if self.clonedir_need_update(ud, d):
             output = runfetchcmd("%s remote" % ud.basecmd, d, quiet=True, workdir=ud.clonedir)
             if "origin" in output:
               runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir)
@@ -369,6 +373,11 @@ class Git(FetchMethod):
             if not self._contains_ref(ud, d, name, ud.clonedir):
                 raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revisions[name], ud.branches[name]))
 
+        if ud.shallow and ud.write_shallow_tarballs:
+            missing_rev = self.clonedir_need_shallow_revs(ud, d)
+            if missing_rev:
+                raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev)
+
     def build_mirror_data(self, ud, d):
         if ud.shallow and ud.write_shallow_tarballs:
             if not os.path.exists(ud.fullshallow):

bitbake/lib/bb/fetch2/hg.py

@@ -54,13 +54,6 @@ class Hg(FetchMethod):
         else:
             ud.proto = "hg"
 
-        ud.setup_revisions(d)
-
-        if 'rev' in ud.parm:
-            ud.revision = ud.parm['rev']
-        elif not ud.revision:
-            ud.revision = self.latest_revision(ud, d)
-
         # Create paths to mercurial checkouts
         hgsrcname = '%s_%s_%s' % (ud.module.replace('/', '.'), \
                             ud.host, ud.path.replace('/', '.'))
@@ -74,6 +67,13 @@ class Hg(FetchMethod):
         ud.localfile = ud.moddir
         ud.basecmd = d.getVar("FETCHCMD_hg") or "/usr/bin/env hg"
 
+        ud.setup_revisions(d)
+
+        if 'rev' in ud.parm:
+            ud.revision = ud.parm['rev']
+        elif not ud.revision:
+            ud.revision = self.latest_revision(ud, d)
+
         ud.write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS")
 
     def need_update(self, ud, d):
@@ -139,7 +139,7 @@ class Hg(FetchMethod):
                 cmd = "%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" pull" % (ud.basecmd, ud.user, ud.pswd, proto)
             else:
                 cmd = "%s pull" % (ud.basecmd)
-        elif command == "update":
+        elif command == "update" or command == "up":
             if ud.user and ud.pswd:
                 cmd = "%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" update -C %s" % (ud.basecmd, ud.user, ud.pswd, proto, " ".join(options))
             else:
@@ -247,12 +247,19 @@ class Hg(FetchMethod):
 
         scmdata = ud.parm.get("scmdata", "")
         if scmdata != "nokeep":
+            proto = ud.parm.get('protocol', 'http')
             if not os.access(os.path.join(codir, '.hg'), os.R_OK):
                 logger.debug(2, "Unpack: creating new hg repository in '" + codir + "'")
                 runfetchcmd("%s init %s" % (ud.basecmd, codir), d)
             logger.debug(2, "Unpack: updating source in '" + codir + "'")
-            runfetchcmd("%s pull %s" % (ud.basecmd, ud.moddir), d, workdir=codir)
-            runfetchcmd("%s up -C %s" % (ud.basecmd, revflag), d, workdir=codir)
+            if ud.user and ud.pswd:
+                runfetchcmd("%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" pull %s" % (ud.basecmd, ud.user, ud.pswd, proto, ud.moddir), d, workdir=codir)
+            else:
+                runfetchcmd("%s pull %s" % (ud.basecmd, ud.moddir), d, workdir=codir)
+            if ud.user and ud.pswd:
+                runfetchcmd("%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" up -C %s" % (ud.basecmd, ud.user, ud.pswd, proto, revflag), d, workdir=codir)
+            else:
+                runfetchcmd("%s up -C %s" % (ud.basecmd, revflag), d, workdir=codir)
         else:
             logger.debug(2, "Unpack: extracting source to '" + codir + "'")
             runfetchcmd("%s archive -t files %s %s" % (ud.basecmd, revflag, codir), d, workdir=ud.moddir)

bitbake/lib/bb/parse/parse_py/ConfHandler.py

@@ -119,30 +119,30 @@ def handle(fn, data, include):
         oldfile = data.getVar('FILE', False)
 
     abs_fn = resolve_file(fn, data)
-    f = open(abs_fn, 'r')
+    with open(abs_fn, 'r') as f:
 
-    statements = ast.StatementGroup()
-    lineno = 0
-    while True:
-        lineno = lineno + 1
-        s = f.readline()
-        if not s:
-            break
-        w = s.strip()
-        # skip empty lines
-        if not w:
-            continue
-        s = s.rstrip()
-        while s[-1] == '\\':
-            s2 = f.readline().rstrip()
+        statements = ast.StatementGroup()
+        lineno = 0
+        while True:
             lineno = lineno + 1
-            if (not s2 or s2 and s2[0] != "#") and s[0] == "#" :
-                bb.fatal("There is a confusing multiline, partially commented expression on line %s of file %s (%s).\nPlease clarify whether this is all a comment or should be parsed." % (lineno, fn, s))
-            s = s[:-1] + s2
-        # skip comments
-        if s[0] == '#':
-            continue
-        feeder(lineno, s, abs_fn, statements)
+            s = f.readline()
+            if not s:
+                break
+            w = s.strip()
+            # skip empty lines
+            if not w:
+                continue
+            s = s.rstrip()
+            while s[-1] == '\\':
+                s2 = f.readline().rstrip()
+                lineno = lineno + 1
+                if (not s2 or s2 and s2[0] != "#") and s[0] == "#" :
+                    bb.fatal("There is a confusing multiline, partially commented expression on line %s of file %s (%s).\nPlease clarify whether this is all a comment or should be parsed." % (lineno, fn, s))
+                s = s[:-1] + s2
+            # skip comments
+            if s[0] == '#':
+                continue
+            feeder(lineno, s, abs_fn, statements)
 
     # DONE WITH PARSING... time to evaluate
     data.setVar('FILE', abs_fn)

bitbake/lib/bb/runqueue.py

@@ -1397,7 +1397,7 @@ class RunQueue:
             cache[tid] = iscurrent
         return iscurrent
 
-    def validate_hashes(self, tocheck, data, currentcount=0, siginfo=False):
+    def validate_hashes(self, tocheck, data, currentcount=0, siginfo=False, summary=True):
         valid = set()
         if self.hashvalidate:
             sq_data = {}
@@ -1410,15 +1410,15 @@ class RunQueue:
                 sq_data['hashfn'][tid] = self.rqdata.dataCaches[mc].hashfn[taskfn]
                 sq_data['unihash'][tid] = self.rqdata.runtaskentries[tid].unihash
 
-            valid = self.validate_hash(sq_data, data, siginfo, currentcount)
+            valid = self.validate_hash(sq_data, data, siginfo, currentcount, summary)
 
         return valid
 
-    def validate_hash(self, sq_data, d, siginfo, currentcount):
-        locs = {"sq_data" : sq_data, "d" : d, "siginfo" : siginfo, "currentcount" : currentcount}
+    def validate_hash(self, sq_data, d, siginfo, currentcount, summary):
+        locs = {"sq_data" : sq_data, "d" : d, "siginfo" : siginfo, "currentcount" : currentcount, "summary" : summary}
 
         # Metadata has **kwargs so args can be added, sq_data can also gain new fields
-        call = self.hashvalidate + "(sq_data, d, siginfo=siginfo, currentcount=currentcount)"
+        call = self.hashvalidate + "(sq_data, d, siginfo=siginfo, currentcount=currentcount, summary=summary)"
 
         return bb.utils.better_eval(call, locs)
 
@@ -1605,7 +1605,7 @@ class RunQueue:
 
             tocheck.add(tid)
 
-        valid_new = self.validate_hashes(tocheck, self.cooker.data, 0, True)
+        valid_new = self.validate_hashes(tocheck, self.cooker.data, 0, True, summary=False)
 
         # Tasks which are both setscene and noexec never care about dependencies
         # We therefore find tasks which are setscene and noexec and mark their
@@ -1986,7 +1986,7 @@ class RunQueueExecute:
                             continue
                         logger.debug(1, "Task %s no longer deferred" % nexttask)
                         del self.sq_deferred[nexttask]
-                        valid = self.rq.validate_hashes(set([nexttask]), self.cooker.data, 0, False)
+                        valid = self.rq.validate_hashes(set([nexttask]), self.cooker.data, 0, False, summary=False)
                         if not valid:
                             logger.debug(1, "%s didn't become valid, skipping setscene" % nexttask)
                             self.sq_task_failoutright(nexttask)
@@ -2283,12 +2283,26 @@ class RunQueueExecute:
                         for dep in self.rqdata.runtaskentries[tid].depends:
                             procdep.append(dep)
                         orighash = self.rqdata.runtaskentries[tid].hash
-                        self.rqdata.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, procdep, self.rqdata.dataCaches[mc_from_tid(tid)])
+                        newhash = bb.parse.siggen.get_taskhash(tid, procdep, self.rqdata.dataCaches[mc_from_tid(tid)])
                         origuni = self.rqdata.runtaskentries[tid].unihash
-                        self.rqdata.runtaskentries[tid].unihash = bb.parse.siggen.get_unihash(tid)
-                        logger.debug(1, "Task %s hash changes: %s->%s %s->%s" % (tid, orighash, self.rqdata.runtaskentries[tid].hash, origuni, self.rqdata.runtaskentries[tid].unihash))
+                        newuni = bb.parse.siggen.get_unihash(tid)
+                        # FIXME, need to check it can come from sstate at all for determinism?
+                        remapped = False
+                        if newuni == origuni:
+                            # Nothing to do, we match, skip code below
+                            remapped = True
+                        elif tid in self.scenequeue_covered or tid in self.sq_live:
+                            # Already ran this setscene task or it running. Report the new taskhash
+                            remapped = bb.parse.siggen.report_unihash_equiv(tid, newhash, origuni, newuni, self.rqdata.dataCaches)
+                            logger.info("Already covered setscene for %s so ignoring rehash (remap)" % (tid))
+
+                        if not remapped:
+                            logger.debug(1, "Task %s hash changes: %s->%s %s->%s" % (tid, orighash, newhash, origuni, newuni))
+                            self.rqdata.runtaskentries[tid].hash = newhash
+                            self.rqdata.runtaskentries[tid].unihash = newuni
+                            changed.add(tid)
+
                         next |= self.rqdata.runtaskentries[tid].revdeps
-                        changed.add(tid)
                         total.remove(tid)
                         next.intersection_update(total)
 
@@ -2303,16 +2317,16 @@ class RunQueueExecute:
         for tid in changed:
             if tid not in self.rqdata.runq_setscene_tids:
                 continue
-            if tid in self.runq_running:
-                continue
-            if tid in self.scenequeue_covered:
-                # Potentially risky, should we report this hash as a match?
-                logger.info("Already covered setscene for %s so ignoring rehash" % (tid))
-                continue
             if tid not in self.pending_migrations:
                 self.pending_migrations.add(tid)
 
+        update_tasks = []
         for tid in self.pending_migrations.copy():
+            if tid in self.runq_running or tid in self.sq_live:
+                # Too late, task already running, not much we can do now
+                self.pending_migrations.remove(tid)
+                continue
+
             valid = True
             # Check no tasks this covers are running
             for dep in self.sqdata.sq_covered_tasks[tid]:
@@ -2337,7 +2351,12 @@ class RunQueueExecute:
                 self.sq_buildable.remove(tid)
             if tid in self.sq_running:
                 self.sq_running.remove(tid)
-            if self.sqdata.sq_revdeps[tid].issubset(self.scenequeue_covered | self.scenequeue_notcovered):
+            harddepfail = False
+            for t in self.sqdata.sq_harddeps:
+                if tid in self.sqdata.sq_harddeps[t] and t in self.scenequeue_notcovered:
+                    harddepfail = True
+                    break
+            if not harddepfail and self.sqdata.sq_revdeps[tid].issubset(self.scenequeue_covered | self.scenequeue_notcovered):
                 if tid not in self.sq_buildable:
                     self.sq_buildable.add(tid)
             if len(self.sqdata.sq_revdeps[tid]) == 0:
@@ -2361,9 +2380,17 @@ class RunQueueExecute:
             if tid in self.build_stamps:
                 del self.build_stamps[tid]
 
-            logger.info("Setscene task %s now valid and being rerun" % tid)
+            update_tasks.append((tid, harddepfail, tid in self.sqdata.valid))
+
+        if update_tasks:
             self.sqdone = False
-            update_scenequeue_data([tid], self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self)
+            update_scenequeue_data([t[0] for t in update_tasks], self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self, summary=False)
+
+        for (tid, harddepfail, origvalid) in update_tasks:
+            if tid in self.sqdata.valid and not origvalid:
+                logger.info("Setscene task %s became valid" % tid)
+            if harddepfail:
+                self.sq_task_failoutright(tid)
 
         if changed:
             self.holdoff_need_update = True
@@ -2500,6 +2527,8 @@ class RunQueueExecute:
                 msg = 'Task %s.%s attempted to execute unexpectedly and should have been setscened' % (pn, taskname)
             else:
                 msg = 'Task %s.%s attempted to execute unexpectedly' % (pn, taskname)
+            for t in self.scenequeue_notcovered:
+                msg = msg + "\nTask %s, unihash %s, taskhash %s" % (t, self.rqdata.runtaskentries[t].unihash, self.rqdata.runtaskentries[t].hash)
             logger.error(msg + '\nThis is usually due to missing setscene tasks. Those missing in this build were: %s' % pprint.pformat(self.scenequeue_notcovered))
             return True
         return False
@@ -2692,9 +2721,9 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
     sqdata.stamppresent = set()
     sqdata.valid = set()
 
-    update_scenequeue_data(sqdata.sq_revdeps, sqdata, rqdata, rq, cooker, stampcache, sqrq)
+    update_scenequeue_data(sqdata.sq_revdeps, sqdata, rqdata, rq, cooker, stampcache, sqrq, summary=True)
 
-def update_scenequeue_data(tids, sqdata, rqdata, rq, cooker, stampcache, sqrq):
+def update_scenequeue_data(tids, sqdata, rqdata, rq, cooker, stampcache, sqrq, summary=True):
 
     tocheck = set()
 
@@ -2728,7 +2757,7 @@ def update_scenequeue_data(tids, sqdata, rqdata, rq, cooker, stampcache, sqrq):
 
         tocheck.add(tid)
 
-    sqdata.valid |= rq.validate_hashes(tocheck, cooker.data, len(sqdata.stamppresent), False)
+    sqdata.valid |= rq.validate_hashes(tocheck, cooker.data, len(sqdata.stamppresent), False, summary=summary)
 
     sqdata.hashes = {}
     for mc in sorted(sqdata.multiconfigs):
@@ -2750,7 +2779,7 @@ def update_scenequeue_data(tids, sqdata, rqdata, rq, cooker, stampcache, sqrq):
                 sqdata.hashes[h] = tid
             else:
                 sqrq.sq_deferred[tid] = sqdata.hashes[h]
-                bb.warn("Deferring %s after %s" % (tid, sqdata.hashes[h]))
+                bb.note("Deferring %s after %s" % (tid, sqdata.hashes[h]))
 
 
 class TaskFailure(Exception):

bitbake/lib/bb/siggen.py

@@ -232,10 +232,14 @@ class SignatureGeneratorBasic(SignatureGenerator):
         taskdep = dataCache.task_deps[fn]
         if 'nostamp' in taskdep and task in taskdep['nostamp']:
             # Nostamp tasks need an implicit taint so that they force any dependent tasks to run
-            import uuid
-            taint = str(uuid.uuid4())
-            data = data + taint
-            self.taints[tid] = "nostamp:" + taint
+            if tid in self.taints and self.taints[tid].startswith("nostamp:"):
+                # Don't reset taint value upon every call
+                data = data + self.taints[tid][8:]
+            else:
+                import uuid
+                taint = str(uuid.uuid4())
+                data = data + taint
+                self.taints[tid] = "nostamp:" + taint
 
         taint = self.read_taint(fn, task, dataCache.stamp[fn])
         if taint:
@@ -508,6 +512,8 @@ class SignatureGeneratorUniHashMixIn(object):
                 if new_unihash != unihash:
                     bb.debug(1, 'Task %s unihash changed %s -> %s by server %s' % (taskhash, unihash, new_unihash, self.server))
                     bb.event.fire(bb.runqueue.taskUniHashUpdate(fn + ':do_' + task, new_unihash), d)
+                    self.set_unihash(tid, new_unihash)
+                    d.setVar('BB_UNIHASH', new_unihash)
                 else:
                     bb.debug(1, 'Reported task %s as unihash %s to %s' % (taskhash, unihash, self.server))
             except hashserv.client.HashConnectionError as e:
@@ -524,6 +530,32 @@ class SignatureGeneratorUniHashMixIn(object):
                 except OSError:
                     pass
 
+    def report_unihash_equiv(self, tid, taskhash, wanted_unihash, current_unihash, datacaches):
+        try:
+            extra_data = {}
+            data = self.client().report_unihash_equiv(taskhash, self.method, wanted_unihash, extra_data)
+            bb.note('Reported task %s as unihash %s to %s (%s)' % (tid, wanted_unihash, self.server, str(data)))
+
+            if data is None:
+                bb.warn("Server unable to handle unihash report")
+                return False
+
+            finalunihash = data['unihash']
+
+            if finalunihash == current_unihash:
+                bb.note('Task %s unihash %s unchanged by server' % (tid, finalunihash))
+            elif finalunihash == wanted_unihash:
+                bb.note('Task %s unihash changed %s -> %s as wanted' % (tid, current_unihash, finalunihash))
+                self.set_unihash(tid, finalunihash)
+                return True
+            else:
+                # TODO: What to do here?
+                bb.note('Task %s unihash reported as unwanted hash %s' % (tid, finalunihash))
+
+        except hashserv.client.HashConnectionError as e:
+            bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))
+
+        return False
 
 #
 # Dummy class used for bitbake-selftest

bitbake/lib/bb/tests/fetch.py

@@ -1863,6 +1863,26 @@ class GitShallowTest(FetcherTest):
         with self.assertRaises(bb.fetch2.FetchError):
             self.fetch()
 
+    def test_shallow_fetch_missing_revs(self):
+        self.add_empty_file('a')
+        self.add_empty_file('b')
+        fetcher, ud = self.fetch(self.d.getVar('SRC_URI'))
+        self.git('tag v0.0 master', cwd=self.srcdir)
+        self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
+        self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
+        self.fetch_shallow()
+
+    def test_shallow_fetch_missing_revs_fails(self):
+        self.add_empty_file('a')
+        self.add_empty_file('b')
+        fetcher, ud = self.fetch(self.d.getVar('SRC_URI'))
+        self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
+        self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
+
+        with self.assertRaises(bb.fetch2.FetchError), self.assertLogs("BitBake.Fetcher", level="ERROR") as cm:
+            self.fetch_shallow()
+        self.assertIn("Unable to find revision v0.0 even from upstream", cm.output[0])
+
     @skipIfNoNetwork()
     def test_bitbake(self):
         self.git('remote add --mirror=fetch origin git://github.com/openembedded/bitbake', cwd=self.srcdir)

bitbake/lib/bb/tests/runqueue.py

@@ -255,7 +255,7 @@ class RunQueueTests(unittest.TestCase):
             cmd = ["bitbake", "a1", "b1"]
             tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
             expected = ['a1:populate_sysroot', 'a1:package', 'a1:package_write_rpm_setscene', 'a1:packagedata_setscene',
-                        'a1:package_write_ipk_setscene', 'a1:package_qa_setscene']
+                        'a1:package_write_ipk_setscene', 'a1:package_qa_setscene', 'a1:build']
             self.assertEqual(set(tasks), set(expected))
 
             self.shutdown(tempdir)

bitbake/lib/bb/tests/utils.py

@@ -103,6 +103,32 @@ class Path(unittest.TestCase):
             result = bb.utils._check_unsafe_delete_path(arg1)
             self.assertEqual(result, correctresult, '_check_unsafe_delete_path("%s") != %s' % (arg1, correctresult))
 
+class Checksum(unittest.TestCase):
+    filler = b"Shiver me timbers square-rigged spike Gold Road galleon bilge water boatswain wherry jack pirate. Mizzenmast rum lad Privateer jack salmagundi hang the jib piracy Pieces of Eight Corsair. Parrel marooned black spot yawl provost quarterdeck cable no prey, no pay spirits lateen sail."
+
+    def test_md5(self):
+        import hashlib
+        with tempfile.NamedTemporaryFile() as f:
+            f.write(self.filler)
+            f.flush()
+            checksum = bb.utils.md5_file(f.name)
+            self.assertEqual(checksum, "bd572cd5de30a785f4efcb6eaf5089e3")
+
+    def test_sha1(self):
+        import hashlib
+        with tempfile.NamedTemporaryFile() as f:
+            f.write(self.filler)
+            f.flush()
+            checksum = bb.utils.sha1_file(f.name)
+            self.assertEqual(checksum, "249eb8fd654732ea836d5e702d7aa567898eca71")
+
+    def test_sha256(self):
+        import hashlib
+        with tempfile.NamedTemporaryFile() as f:
+            f.write(self.filler)
+            f.flush()
+            checksum = bb.utils.sha256_file(f.name)
+            self.assertEqual(checksum, "fcfbae8bf6b721dbb9d2dc6a9334a58f2031a9a9b302999243f99da4d7f12d0f")
 
 class EditMetadataFile(unittest.TestCase):
     _origfile = """

bitbake/lib/bb/ui/knotty.py

@@ -255,19 +255,19 @@ class TerminalFilter(object):
                 start_time = activetasks[t].get("starttime", None)
                 if not pbar or pbar.bouncing != (progress < 0):
                     if progress < 0:
-                        pbar = BBProgress("0: %s (pid %s) " % (activetasks[t]["title"], t), 100, widgets=[progressbar.BouncingSlider(), ''], extrapos=2, resize_handler=self.sigwinch_handle)
+                        pbar = BBProgress("0: %s (pid %s) " % (activetasks[t]["title"], activetasks[t]["pid"]), 100, widgets=[progressbar.BouncingSlider(), ''], extrapos=2, resize_handler=self.sigwinch_handle)
                         pbar.bouncing = True
                     else:
-                        pbar = BBProgress("0: %s (pid %s) " % (activetasks[t]["title"], t), 100, widgets=[progressbar.Percentage(), ' ', progressbar.Bar(), ''], extrapos=4, resize_handler=self.sigwinch_handle)
+                        pbar = BBProgress("0: %s (pid %s) " % (activetasks[t]["title"], activetasks[t]["pid"]), 100, widgets=[progressbar.Percentage(), ' ', progressbar.Bar(), ''], extrapos=4, resize_handler=self.sigwinch_handle)
                         pbar.bouncing = False
                     activetasks[t]["progressbar"] = pbar
                 tasks.append((pbar, progress, rate, start_time))
             else:
                 start_time = activetasks[t].get("starttime", None)
                 if start_time:
-                    tasks.append("%s - %s (pid %s)" % (activetasks[t]["title"], self.elapsed(currenttime - start_time), t))
+                    tasks.append("%s - %s (pid %s)" % (activetasks[t]["title"], self.elapsed(currenttime - start_time), activetasks[t]["pid"]))
                 else:
-                    tasks.append("%s (pid %s)" % (activetasks[t]["title"], t))
+                    tasks.append("%s (pid %s)" % (activetasks[t]["title"], activetasks[t]["pid"]))
 
         if self.main.shutdown:
             content = "Waiting for %s running tasks to finish:" % len(activetasks)
@@ -517,8 +517,8 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                         continue
 
                     # Prefix task messages with recipe/task
-                    if event.taskpid in helper.running_tasks and event.levelno != format.PLAIN:
-                        taskinfo = helper.running_tasks[event.taskpid]
+                    if event.taskpid in helper.pidmap and event.levelno != format.PLAIN:
+                        taskinfo = helper.running_tasks[helper.pidmap[event.taskpid]]
                         event.msg = taskinfo['title'] + ': ' + event.msg
                 if hasattr(event, 'fn'):
                     event.msg = event.fn + ': ' + event.msg

bitbake/lib/bb/ui/uihelper.py

@@ -15,39 +15,48 @@ class BBUIHelper:
         # Running PIDs preserves the order tasks were executed in
         self.running_pids = []
         self.failed_tasks = []
+        self.pidmap = {}
         self.tasknumber_current = 0
         self.tasknumber_total = 0
 
     def eventHandler(self, event):
+        # PIDs are a bad idea as they can be reused before we process all UI events.
+        # We maintain a 'fuzzy' match for TaskProgress since there is no other way to match
+        def removetid(pid, tid):
+            self.running_pids.remove(tid)
+            del self.running_tasks[tid]
+            if self.pidmap[pid] == tid:
+                del self.pidmap[pid]
+            self.needUpdate = True
+
         if isinstance(event, bb.build.TaskStarted):
+            tid = event._fn + ":" + event._task
             if event._mc != "default":
-                self.running_tasks[event.pid] = { 'title' : "mc:%s:%s %s" % (event._mc, event._package, event._task), 'starttime' : time.time() }
+                self.running_tasks[tid] = { 'title' : "mc:%s:%s %s" % (event._mc, event._package, event._task), 'starttime' : time.time(), 'pid' : event.pid }
             else:
-                self.running_tasks[event.pid] = { 'title' : "%s %s" % (event._package, event._task), 'starttime' : time.time() }
-            self.running_pids.append(event.pid)
+                self.running_tasks[tid] = { 'title' : "%s %s" % (event._package, event._task), 'starttime' : time.time(), 'pid' : event.pid }
+            self.running_pids.append(tid)
+            self.pidmap[event.pid] = tid
             self.needUpdate = True
         elif isinstance(event, bb.build.TaskSucceeded):
-            del self.running_tasks[event.pid]
-            self.running_pids.remove(event.pid)
-            self.needUpdate = True
+            tid = event._fn + ":" + event._task
+            removetid(event.pid, tid)
         elif isinstance(event, bb.build.TaskFailedSilent):
-            del self.running_tasks[event.pid]
-            self.running_pids.remove(event.pid)
+            tid = event._fn + ":" + event._task
+            removetid(event.pid, tid)
             # Don't add to the failed tasks list since this is e.g. a setscene task failure
-            self.needUpdate = True
         elif isinstance(event, bb.build.TaskFailed):
-            del self.running_tasks[event.pid]
-            self.running_pids.remove(event.pid)
+            tid = event._fn + ":" + event._task
+            removetid(event.pid, tid)
             self.failed_tasks.append( { 'title' : "%s %s" % (event._package, event._task)})
-            self.needUpdate = True
         elif isinstance(event, bb.runqueue.runQueueTaskStarted):
             self.tasknumber_current = event.stats.completed + event.stats.active + event.stats.failed + 1
             self.tasknumber_total = event.stats.total
             self.needUpdate = True
         elif isinstance(event, bb.build.TaskProgress):
-            if event.pid > 0:
-                self.running_tasks[event.pid]['progress'] = event.progress
-                self.running_tasks[event.pid]['rate'] = event.rate
+            if event.pid > 0 and event.pid in self.pidmap:
+                self.running_tasks[self.pidmap[event.pid]]['progress'] = event.progress
+                self.running_tasks[self.pidmap[event.pid]]['rate'] = event.rate
                 self.needUpdate = True
         else:
             return False

bitbake/lib/bb/utils.py

@@ -520,22 +520,26 @@ def unlockfile(lf):
     fcntl.flock(lf.fileno(), fcntl.LOCK_UN)
     lf.close()
 
+def _hasher(method, filename):
+    import mmap
+
+    with open(filename, "rb") as f:
+        try:
+            with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm:
+                for chunk in iter(lambda: mm.read(8192), b''):
+                    method.update(chunk)
+        except ValueError:
+            # You can't mmap() an empty file so silence this exception
+            pass
+    return method.hexdigest()
+
+
 def md5_file(filename):
     """
     Return the hex string representation of the MD5 checksum of filename.
     """
-    import hashlib, mmap
-
-    with open(filename, "rb") as f:
-        m = hashlib.md5()
-        try:
-            with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm:
-                for chunk in iter(lambda: mm.read(8192), b''):
-                    m.update(chunk)
-        except ValueError:
-            # You can't mmap() an empty file so silence this exception
-            pass
-    return m.hexdigest()
+    import hashlib
+    return _hasher(hashlib.md5(), filename)
 
 def sha256_file(filename):
     """
@@ -543,24 +547,14 @@ def sha256_file(filename):
     filename.
     """
     import hashlib
-
-    s = hashlib.sha256()
-    with open(filename, "rb") as f:
-        for line in f:
-            s.update(line)
-    return s.hexdigest()
+    return _hasher(hashlib.sha256(), filename)
 
 def sha1_file(filename):
     """
     Return the hex string representation of the SHA1 checksum of the filename
     """
     import hashlib
-
-    s = hashlib.sha1()
-    with open(filename, "rb") as f:
-        for line in f:
-            s.update(line)
-    return s.hexdigest()
+    return _hasher(hashlib.sha1(), filename)
 
 def preserved_envvars_exported():
     """Variables which are taken from the environment and placed in and exported

bitbake/lib/hashserv/client.py

@@ -148,6 +148,14 @@ class Client(object):
         m['unihash'] = unihash
         return self.send_message({'report': m})
 
+    def report_unihash_equiv(self, taskhash, method, unihash, extra={}):
+        self._set_mode(self.MODE_NORMAL)
+        m = extra.copy()
+        m['taskhash'] = taskhash
+        m['method'] = method
+        m['unihash'] = unihash
+        return self.send_message({'report-equiv': m})
+
     def get_stats(self):
         self._set_mode(self.MODE_NORMAL)
         return self.send_message({'get-stats': None})

bitbake/lib/hashserv/server.py

@@ -143,6 +143,7 @@ class ServerClient(object):
             handlers = {
                 'get': self.handle_get,
                 'report': self.handle_report,
+                'report-equiv': self.handle_equivreport,
                 'get-stream': self.handle_get_stream,
                 'get-stats': self.handle_get_stats,
                 'reset-stats': self.handle_reset_stats,
@@ -303,6 +304,41 @@ class ServerClient(object):
 
         self.write_message(d)
 
+    async def handle_equivreport(self, data):
+        with closing(self.db.cursor()) as cursor:
+            insert_data = {
+                'method': data['method'],
+                'outhash': "",
+                'taskhash': data['taskhash'],
+                'unihash': data['unihash'],
+                'created': datetime.now()
+            }
+
+            for k in ('owner', 'PN', 'PV', 'PR', 'task', 'outhash_siginfo'):
+                if k in data:
+                    insert_data[k] = data[k]
+
+            cursor.execute('''INSERT OR IGNORE INTO tasks_v2 (%s) VALUES (%s)''' % (
+                ', '.join(sorted(insert_data.keys())),
+                ', '.join(':' + k for k in sorted(insert_data.keys()))),
+                insert_data)
+
+            self.db.commit()
+
+            # Fetch the unihash that will be reported for the taskhash. If the
+            # unihash matches, it means this row was inserted (or the mapping
+            # was already valid)
+            row = self.query_equivalent(data['method'], data['taskhash'])
+
+            if row['unihash'] == data['unihash']:
+                logger.info('Adding taskhash equivalence for %s with unihash %s',
+                                data['taskhash'], row['unihash'])
+
+            d = {k: row[k] for k in ('taskhash', 'method', 'unihash')}
+
+        self.write_message(d)
+
+
     async def handle_get_stats(self, request):
         d = {
             'requests': self.request_stats.todict(),

bitbake/lib/prserv/serv.py

@@ -243,6 +243,7 @@ class PRServer(SimpleXMLRPCServer):
         try:
             pid = os.fork()
             if pid > 0:
+                self.socket.close() # avoid ResourceWarning in parent
                 return pid
         except OSError as e:
             raise Exception("%s [%d]" % (e.strerror, e.errno))
@@ -379,9 +380,8 @@ def stop_daemon(host, port):
     ip = socket.gethostbyname(host)
     pidfile = PIDPREFIX % (ip, port)
     try:
-        pf = open(pidfile,'r')
-        pid = int(pf.readline().strip())
-        pf.close()
+        with open(pidfile) as pf:
+            pid = int(pf.readline().strip())
     except IOError:
         pid = None
 
@@ -449,29 +449,35 @@ class PRServiceConfigError(Exception):
 def auto_start(d):
     global singleton
 
-    # Shutdown any existing PR Server
-    auto_shutdown()
-
     host_params = list(filter(None, (d.getVar('PRSERV_HOST') or '').split(':')))
     if not host_params:
+        # Shutdown any existing PR Server
+        auto_shutdown()
         return None
 
     if len(host_params) != 2:
+        # Shutdown any existing PR Server
+        auto_shutdown()
         logger.critical('\n'.join(['PRSERV_HOST: incorrect format',
                 'Usage: PRSERV_HOST = "<hostname>:<port>"']))
         raise PRServiceConfigError
 
-    if is_local_special(host_params[0], int(host_params[1])) and not singleton:
+    if is_local_special(host_params[0], int(host_params[1])):
         import bb.utils
         cachedir = (d.getVar("PERSISTENT_DIR") or d.getVar("CACHE"))
         if not cachedir:
             logger.critical("Please set the 'PERSISTENT_DIR' or 'CACHE' variable")
             raise PRServiceConfigError
-        bb.utils.mkdirhier(cachedir)
         dbfile = os.path.join(cachedir, "prserv.sqlite3")
         logfile = os.path.join(cachedir, "prserv.log")
-        singleton = PRServSingleton(os.path.abspath(dbfile), os.path.abspath(logfile), ("localhost",0))
-        singleton.start()
+        if singleton:
+            if singleton.dbfile != dbfile:
+               # Shutdown any existing PR Server as doesn't match config
+               auto_shutdown()
+        if not singleton:
+            bb.utils.mkdirhier(cachedir)
+            singleton = PRServSingleton(os.path.abspath(dbfile), os.path.abspath(logfile), ("localhost",0))
+            singleton.start()
     if singleton:
         host, port = singleton.getinfo()
     else:

documentation/bsp-guide/bsp-guide.xml

@@ -132,9 +132,14 @@
                 <revremark>Released with the Yocto Project 2.7 Release.</revremark>
             </revision>
             <revision>
-                <revnumber>2.8</revnumber>
+                <revnumber>3.0</revnumber>
+                <date>October</date>
+                <revremark>Released with the Yocto Project 3.0 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>3.1</revnumber>
                 <date>&REL_MONTH_YEAR;</date>
-                <revremark>Released with the Yocto Project 2.8 Release.</revremark>
+                <revremark>Released with the Yocto Project 3.1 Release.</revremark>
             </revision>
         </revhistory>
 

documentation/dev-manual/dev-manual-common-tasks.xml

@@ -2349,7 +2349,7 @@
                 Most software provides some means of setting build-time
                 configuration options before compilation.
                 Typically, setting these options is accomplished by running a
-                configure script with some options, or by modifying a build
+                configure script with options, or by modifying a build
                 configuration file.
                 <note>
                     As of Yocto Project Release 1.7, some of the core recipes
@@ -2389,6 +2389,7 @@
                         software is built using Autotools.
                         If this is the case, you just need to worry about
                         modifying the configuration.</para>
+
                         <para>When using Autotools, your recipe needs to inherit
                         the
                         <ulink url='&YOCTO_DOCS_REF_URL;#ref-classes-autotools'><filename>autotools</filename></ulink>
@@ -2401,13 +2402,15 @@
                         or
                         <ulink url='&YOCTO_DOCS_REF_URL;#var-PACKAGECONFIG_CONFARGS'><filename>PACKAGECONFIG_CONFARGS</filename></ulink>
                         to pass any needed configure options that are specific
-                        to the recipe.</para></listitem>
+                        to the recipe.
+                        </para></listitem>
                     <listitem><para><emphasis>CMake:</emphasis>
                         If your source files have a
                         <filename>CMakeLists.txt</filename> file, then your
                         software is built using CMake.
                         If this is the case, you just need to worry about
                         modifying the configuration.</para>
+
                         <para>When you use CMake, your recipe needs to inherit
                         the
                         <ulink url='&YOCTO_DOCS_REF_URL;#ref-classes-cmake'><filename>cmake</filename></ulink>
@@ -2417,7 +2420,16 @@
                         You can make some adjustments by setting
                         <ulink url='&YOCTO_DOCS_REF_URL;#var-EXTRA_OECMAKE'><filename>EXTRA_OECMAKE</filename></ulink>
                         to pass any needed configure options that are specific
-                        to the recipe.</para></listitem>
+                        to the recipe.
+                        <note>
+                            If you need to install one or more custom CMake
+                            toolchain files that are supplied by the
+                            application you are building, install the files to
+                            <filename>${D}${datadir}/cmake/</filename> Modules
+                            during
+                            <ulink url='&YOCTO_DOCS_REF_URL;#ref-tasks-install'><filename>do_install</filename></ulink>.
+                        </note>
+                        </para></listitem>
                     <listitem><para><emphasis>Other:</emphasis>
                         If your source files do not have a
                         <filename>configure.ac</filename> or
@@ -2780,6 +2792,14 @@
                         <ulink url='&YOCTO_DOCS_REF_URL;#var-PARALLEL_MAKEINST'><filename>PARALLEL_MAKEINST</filename></ulink>
                         for additional information.
                         </para></listitem>
+                    <listitem><para>
+                        If you need to install one or more custom CMake
+                        toolchain files that are supplied by the
+                        application you are building, install the files to
+                        <filename>${D}${datadir}/cmake/</filename> Modules
+                        during
+                        <ulink url='&YOCTO_DOCS_REF_URL;#ref-tasks-install'><filename>do_install</filename></ulink>.
+                        </para></listitem>
                 </itemizedlist>
             </note>
         </section>
@@ -5420,12 +5440,16 @@
                             <literallayout class='monospaced'>
      BBMULTICONFIG = "x86 arm"
                             </literallayout>
-                          </para>
-
-                          <para>Please note, that a "default" configuration already exists by definition,
-                          this configuration is named: "" (empty string) and is defined by the variables
-                          coming from your local.conf file. So, the previous example actually adds two
-                          additional configurations to your build "arm" and "x86" along with "".
+                            <note>
+                                A "default" configuration already exists by
+                                definition.
+                                This configuration is named: "" (i.e. empty
+                                string) and is defined by the variables coming
+                                from your <filename>local.conf</filename> file.
+                                Consequently, the previous example actually
+                                adds two additional configurations to your
+                                build: "arm" and "x86" along with "".
+                            </note>
                             </para></listitem>
                         <listitem><para>
                             <emphasis>Launch BitBake</emphasis>:
@@ -5445,9 +5469,10 @@
                             <filename>x86.conf</filename> configuration file,
                             a <filename>core-image-sato</filename>
                             image that is configured through the
-                            <filename>arm.conf</filename> configuration file and a
-                            <filename>core-image-base</filename> that is configured
-                            through your <filename>local.conf</filename> configuration file.
+                            <filename>arm.conf</filename> configuration file
+                            and a <filename>core-image-base</filename> that is
+                            configured through your
+                            <filename>local.conf</filename> configuration file.
                             </para></listitem>
                     </itemizedlist>
                     <note>
@@ -10819,6 +10844,38 @@
             features that are used by many distributions.
         </para>
 
+        <para>
+            Within the system, SysVinit treats system components as services.
+            These services are maintained as shell scripts stored in the
+            <filename>/etc/init.d/</filename> directory.
+            Services organize into different run levels.
+            This organization is maintained by putting links to the services
+            in the <filename>/etc/rcN.d/</filename> directories, where
+            <replaceable>N/</replaceable> is one of the following options:
+            "S", "0", "1", "2", "3", "4", "5", or "6".
+            <note>
+                Each runlevel has a dependency on the previous runlevel.
+                This dependency allows the services to work properly.
+            </note>
+        </para>
+
+        <para>
+            In comparison, systemd treats components as units.
+            Using units is a broader concept as compared to using a service.
+            A unit includes several different types of entities.
+            Service is one of the types of entities.
+            The runlevel concept in SysVinit corresponds to the concept of a
+            target in systemd, where target is also a type of supported unit.
+        </para>
+
+        <para>
+            In a SysVinit-based system, services load sequentially (i.e. one
+            by one) during and parallelization is not supported.
+            With systemd, services start in parallel.
+            Needless to say, the method can have an impact on system startup
+            performance.
+        </para>
+
         <para>
             If you want to use SysVinit, you do
             not have to do anything.

documentation/dev-manual/dev-manual.xml

@@ -117,9 +117,14 @@
                 <revremark>Released with the Yocto Project 2.7 Release.</revremark>
             </revision>
             <revision>
-                <revnumber>2.8</revnumber>
+                <revnumber>3.0</revnumber>
+                <date>October</date>
+                <revremark>Released with the Yocto Project 3.0 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>3.1</revnumber>
                 <date>&REL_MONTH_YEAR;</date>
-                <revremark>Released with the Yocto Project 2.8 Release.</revremark>
+                <revremark>Released with the Yocto Project 3.1 Release.</revremark>
             </revision>
         </revhistory>
 

documentation/kernel-dev/kernel-dev-common.xml

@@ -549,9 +549,9 @@
                     <literallayout class='monospaced'>
      FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"
 
-     SRC_URI_append += "file://<replaceable>patch-file-one</replaceable>"
-     SRC_URI_append += "file://<replaceable>patch-file-two</replaceable>"
-     SRC_URI_append += "file://<replaceable>patch-file-three</replaceable>"
+     SRC_URI_append = " file://<replaceable>patch-file-one</replaceable>"
+     SRC_URI_append = " file://<replaceable>patch-file-two</replaceable>"
+     SRC_URI_append = " file://<replaceable>patch-file-three</replaceable>"
                     </literallayout>
                     The
                     <ulink url='&YOCTO_DOCS_REF_URL;#var-FILESEXTRAPATHS'><filename>FILESEXTRAPATHS</filename></ulink>

documentation/kernel-dev/kernel-dev.xml

@@ -102,9 +102,14 @@
                 <revremark>Released with the Yocto Project 2.7 Release.</revremark>
             </revision>
             <revision>
-                <revnumber>2.8</revnumber>
+                <revnumber>3.0</revnumber>
+                <date>October</date>
+                <revremark>Released with the Yocto Project 3.0 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>3.1</revnumber>
                 <date>&REL_MONTH_YEAR;</date>
-                <revremark>Released with the Yocto Project 2.8 Release.</revremark>
+                <revremark>Released with the Yocto Project 3.1 Release.</revremark>
             </revision>
         </revhistory>
 

documentation/mega-manual/mega-manual.xml

@@ -88,9 +88,14 @@
                 <revremark>Released with the Yocto Project 2.7 Release.</revremark>
             </revision>
             <revision>
-                <revnumber>2.8</revnumber>
+                <revnumber>3.0</revnumber>
+                <date>October</date>
+                <revremark>Released with the Yocto Project 3.0 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>3.1</revnumber>
                 <date>&REL_MONTH_YEAR;</date>
-                <revremark>Released with the Yocto Project 2.8 Release.</revremark>
+                <revremark>Released with the Yocto Project 3.1 Release.</revremark>
             </revision>
        </revhistory>
 
@@ -332,22 +337,22 @@
     </para>
 
     <xi:include
-        xmlns:xi="http://www.w3.org/2003/XInclude" href="../../../bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.xml"/>
+        xmlns:xi="http://www.w3.org/2003/XInclude" href="../../bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.xml"/>
 
     <xi:include
-        xmlns:xi="http://www.w3.org/2003/XInclude" href="../../../bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.xml"/>
+        xmlns:xi="http://www.w3.org/2003/XInclude" href="../../bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.xml"/>
 
     <xi:include
-        xmlns:xi="http://www.w3.org/2003/XInclude" href="../../../bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml"/>
+        xmlns:xi="http://www.w3.org/2003/XInclude" href="../../bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml"/>
 
     <xi:include
-        xmlns:xi="http://www.w3.org/2003/XInclude" href="../../../bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml"/>
+        xmlns:xi="http://www.w3.org/2003/XInclude" href="../../bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml"/>
 
     <xi:include
-        xmlns:xi="http://www.w3.org/2003/XInclude" href="../../../bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.xml"/>
+        xmlns:xi="http://www.w3.org/2003/XInclude" href="../../bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.xml"/>
 
     <xi:include
-        xmlns:xi="http://www.w3.org/2003/XInclude" href="../../../bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.xml"/>
+        xmlns:xi="http://www.w3.org/2003/XInclude" href="../../bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.xml"/>
 
 </book>
 

documentation/overview-manual/overview-manual.xml

@@ -47,9 +47,14 @@
                 <revremark>Released with the Yocto Project 2.7 Release.</revremark>
             </revision>
             <revision>
-                <revnumber>2.8</revnumber>
+                <revnumber>3.0</revnumber>
+                <date>October</date>
+                <revremark>Released with the Yocto Project 3.0 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>3.1</revnumber>
                 <date>&REL_MONTH_YEAR;</date>
-                <revremark>Released with the Yocto Project 2.8 Release.</revremark>
+                <revremark>Released with the Yocto Project 3.1 Release.</revremark>
             </revision>
         </revhistory>
 

documentation/poky.ent

@@ -1,19 +1,19 @@
-<!ENTITY DISTRO "2.8">
-<!ENTITY DISTRO_COMPRESSED "28">
+<!ENTITY DISTRO "3.1">
+<!ENTITY DISTRO_COMPRESSED "31">
 <!ENTITY DISTRO_NAME_NO_CAP "zeus">
 <!ENTITY DISTRO_NAME "Zeus">
 <!ENTITY DISTRO_NAME_NO_CAP_MINUS_ONE "warrior">
 <!ENTITY DISTRO_NAME_MINUS_ONE "Warrior">
-<!ENTITY YOCTO_DOC_VERSION "2.8">
-<!ENTITY YOCTO_DOC_VERSION_MINUS_ONE "2.7">
-<!ENTITY DISTRO_REL_TAG "yocto-2.8">
-<!ENTITY METAINTELVERSION "9.0">
-<!ENTITY REL_MONTH_YEAR "TBD">
+<!ENTITY YOCTO_DOC_VERSION "3.1">
+<!ENTITY YOCTO_DOC_VERSION_MINUS_ONE "2.7.2">
+<!ENTITY DISTRO_REL_TAG "yocto-3.1">
+<!ENTITY METAINTELVERSION "12.0">
+<!ENTITY REL_MONTH_YEAR "TBD 2020">
 <!ENTITY META_INTEL_REL_TAG "&METAINTELVERSION;-&DISTRO_NAME_NO_CAP;-&YOCTO_DOC_VERSION;">
-<!ENTITY POKYVERSION "22.0.0">
-<!ENTITY POKYVERSION_COMPRESSED "2200">
+<!ENTITY POKYVERSION "23.0.0">
+<!ENTITY POKYVERSION_COMPRESSED "2300">
 <!ENTITY YOCTO_POKY "poky-&DISTRO_NAME_NO_CAP;-&POKYVERSION;">
-<!ENTITY COPYRIGHT_YEAR "2010-2019">
+<!ENTITY COPYRIGHT_YEAR "2010-2020">
 <!ENTITY YOCTO_DL_URL "http://downloads.yoctoproject.org">
 <!ENTITY YOCTO_HOME_URL "http://www.yoctoproject.org">
 <!ENTITY YOCTO_LISTS_URL "http://lists.yoctoproject.org">

documentation/profile-manual/profile-manual.xml

@@ -102,9 +102,14 @@
                 <revremark>Released with the Yocto Project 2.7 Release.</revremark>
             </revision>
             <revision>
-                <revnumber>2.8</revnumber>
+                <revnumber>3.0</revnumber>
+                <date>October</date>
+                <revremark>Released with the Yocto Project 3.0 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>3.1</revnumber>
                 <date>&REL_MONTH_YEAR;</date>
-                <revremark>Released with the Yocto Project 2.8 Release.</revremark>
+                <revremark>Released with the Yocto Project 3.1 Release.</revremark>
             </revision>
         </revhistory>
 

documentation/ref-manual/migration.xml

@@ -2103,10 +2103,9 @@
         </para>
 
         <para>
-            Additionally, a
-            <link linkend='ref-classes-bluetooth'><filename>bluetooth</filename></link>
-            class has been added to make selection of the appropriate bluetooth
-            support within a recipe a little easier.
+            Additionally, a <filename>bluetooth</filename> class has been added
+            to make selection of the appropriate bluetooth support within a
+            recipe a little easier.
             If you wish to make use of this class in a recipe, add something
             such as the following:
             <literallayout class='monospaced'>
@@ -4740,7 +4739,7 @@ id=f4d4f99cfbc2396e49c1613a7d237b9e57f06f81'>commit message</ulink>.
 
         <para>
             This section provides information about packaging changes that have
-            ocurred:
+            occurred:
             <itemizedlist>
                 <listitem><para>
                     <emphasis><filename>python3</filename> Changes:</emphasis>
@@ -6597,6 +6596,505 @@ id=f4d4f99cfbc2396e49c1613a7d237b9e57f06f81'>commit message</ulink>.
         </para>
     </section>
 </section>
+
+<section id='moving-to-the-yocto-project-3.0-release'>
+    <title>Moving to the Yocto Project 3.0 Release</title>
+
+    <para>
+        This section provides migration information for moving to the
+        Yocto Project 3.0 Release from the prior release.
+    </para>
+
+    <section id='migration-3.0-init-system-selection'>
+        <title>Init System Selection</title>
+
+        <para>
+            Changing the init system manager previously required setting a
+            number of different variables.
+            You can now change the manager by setting the
+            <filename>INIT_MANAGER</filename> variable and the corresponding
+            include files
+            (i.e. <filename>conf/distro/include/init-manager-*.conf</filename>).
+            Include files are provided for four values: "none", "sysvinit",
+            "systemd", and "mdev-busybox".
+            The default value, "none", for <filename>INIT_MANAGER</filename>
+            should allow your current settings to continue working.
+            However, it is advisable to explicitly set
+            <filename>INIT_MANAGER</filename>.
+        </para>
+    </section>
+
+    <section id='migration-3.0-lsb-support-removed'>
+        <title>LSB Support Removed</title>
+
+        <para>
+            Linux Standard Base (LSB) as a standard is not current, and
+            is not well suited for embedded applications.
+            Support can be continued in a separate layer if needed.
+            However, presently LSB support has been removed from the core.
+        </para>
+
+        <para>
+            As a result of this change, the <filename>poky-lsb</filename>
+            derivative distribution configuration that was also used for
+            testing alternative configurations has been replaced with a
+            <filename>poky-altcfg</filename> distribution that has LSB
+            parts removed.
+        </para>
+    </section>
+
+    <section id='migration-3.0-removed-recipes'>
+        <title>Removed Recipes</title>
+
+        <para>
+            The following recipes have been removed.
+            <itemizedlist>
+                <listitem><para>
+                    <filename>core-image-lsb-dev</filename>: Part of removed
+                    LSB support.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>core-image-lsb</filename>: Part of removed
+                    LSB support.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>core-image-lsb-sdk</filename>: Part of removed
+                    LSB support.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>cve-check-tool</filename>: Functionally replaced
+                    by the <filename>cve-update-db</filename> recipe and
+                    <filename>cve-check</filename> class.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>eglinfo</filename>: No longer maintained.
+                    <filename>eglinfo</filename> from
+                    <filename>mesa-demos</filename> is an adequate and
+                    maintained alternative.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>gcc-8.3</filename>: Version 8.3 removed.
+                    Replaced by 9.2.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>gnome-themes-standard</filename>: Only needed
+                    by gtk+ 2.x, which has been removed.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>gtk+</filename>: GTK+ 2 is obsolete and has been
+                    replaced by gtk+3.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>irda-utils</filename>: Has become obsolete.
+                    IrDA support has been removed from the Linux kernel in
+                    version 4.17 and later.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>libnewt-python</filename>:
+                    <filename>libnewt</filename> Python support merged into
+                    main <filename>libnewt</filename> recipe.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>libsdl</filename>: Replaced by newer
+                    <filename>libsdl2</filename>.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>libx11-diet</filename>: Became obsolete.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>libxx86dga</filename>: Removed obsolete client
+                    library.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>libxx86misc</filename>: Removed. Library is
+                    redundant.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>linux-yocto</filename>: Version 5.0 removed,
+                    which is now redundant (5.2 / 4.19 present).
+                    </para></listitem>
+                <listitem><para>
+                    <filename>lsbinitscripts</filename>: Part of removed LSB
+                    support.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>lsb</filename>: Part of removed LSB support.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>lsbtest</filename>: Part of removed LSB support.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>openssl10</filename>: Replaced by newer
+                    <filename>openssl</filename> version 1.1.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>packagegroup-core-lsb</filename>: Part of removed
+                    LSB support.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>python-nose</filename>: Removed the Python 2.x
+                    version of the recipe.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>python-numpy</filename>: Removed the Python 2.x
+                    version of the recipe.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>python-scons</filename>: Removed the Python 2.x
+                    version of the recipe.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>source-highlight</filename>: No longer needed.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>stress</filename>: Replaced by
+                    <filename>stress-ng</filename>.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>vulkan</filename>: Split into
+                    <filename>vulkan-loader</filename>,
+                    <filename>vulkan-headers</filename>, and
+                    <filename>vulkan-tools</filename>.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>weston-conf</filename>: Functionality moved to
+                    <filename>weston-init</filename>.
+                    </para></listitem>
+            </itemizedlist>
+        </para>
+    </section>
+
+    <section id='migration-3.0-packaging-changes'>
+        <title>Packaging Changes</title>
+
+        <para>
+            The following packaging changes have occurred.
+            <itemizedlist>
+                <listitem><para>
+                    The
+                    <ulink url='https://en.wikipedia.org/wiki/GNOME_Web'>Epiphany</ulink>
+                    browser has been dropped from
+                    <filename>packagegroup-self-hosted</filename> as it has
+                    not been needed inside
+                    <filename>build-appliance-image</filename> for
+                    quite some time and was causing resource problems.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>libcap-ng</filename> Python support has been
+                    moved to a separate <filename>libcap-ng-python</filename>
+                    recipe to streamline the build process when the Python
+                    bindings are not needed.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>libdrm</filename> now packages the file
+                    <filename>amdgpu.ids</filename> into a separate
+                    <filename>libdrm-amdgpu</filename> package.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>python3</filename>: The
+                    <filename>runpy</filename> module is now in the
+                    <filename>python3-core</filename> package as it is
+                    required to support the common "python3 -m" command usage.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>distcc</filename> now provides separate
+                    <filename>distcc-client</filename> and
+                    <filename>distcc-server</filename> packages as typically
+                    one or the other are needed, rather than both.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>python*-setuptools</filename> recipes now
+                    separately package the <filename>pkg_resources</filename>
+                    module in a <filename>python-pkg-resources</filename> /
+                    <filename>python3-pkg-resources</filename> package as
+                    the module is useful independent of the rest of the
+                    setuptools package.
+                    The main <filename>python-setuptools</filename> /
+                    <filename>python3-setuptools</filename> package depends
+                    on this new package so you should only need to update
+                    dependencies unless you want to take advantage of the
+                    increased granularity.
+                    </para></listitem>
+            </itemizedlist>
+        </para>
+    </section>
+
+    <section id='migration-3.0-cve-checking'>
+        <title>CVE Checking</title>
+
+        <para>
+            <filename>cve-check-tool</filename> has been functionally replaced
+            by a new <filename>cve-update-db</filename> recipe and
+            functionality built into the <filename>cve-check</filename> class.
+            The result uses NVD JSON data feeds rather than the deprecated
+            XML feeds that <filename>cve-check-tool</filename> was using,
+            supports CVSSv3 scoring, and makes other improvements.
+        </para>
+
+        <para>
+            Additionally, the <filename>CVE_CHECK_CVE_WHITELIST</filename>
+            variable has been replaced by
+            <filename>CVE_CHECK_WHITELIST</filename>.
+        </para>
+    </section>
+
+    <section id='migration-3.0-bitbake-changes'>
+        <title>Bitbake Changes</title>
+
+        <para>
+            The following BitBake changes have occurred.
+            <itemizedlist>
+                <listitem><para>
+                    <filename>addtask</filename> statements now properly
+                    validate dependent tasks.
+                    Previously, an invalid task was silently ignored.
+                    With this change, the invalid task generates a warning.
+                    </para></listitem>
+                <listitem><para>
+                    Other invalid <filename>addtask</filename> and
+                    <filename>deltask</filename> usages now trigger these
+                    warnings: "multiple target tasks arguments with
+                    addtask / deltask", and "multiple before/after clauses".
+                    </para></listitem>
+                <listitem><para>
+                    The "multiconfig" prefix is now shortened to "mc".
+                    "multiconfig" will continue to work, however it may be
+                    removed in a future release.
+                    </para></listitem>
+                <listitem><para>
+                    The <filename>bitbake -g</filename> command no longer
+                    generates a <filename>recipe-depends.dot</filename> file
+                    as the contents (i.e. a reprocessed version of
+                    <filename>task-depends.dot</filename>) were confusing.
+                    </para></listitem>
+                <listitem><para>
+                    The <filename>bb.build.FuncFailed</filename> exception,
+                    previously raised by
+                    <filename>bb.build.exec_func()</filename> when certain
+                    other exceptions have occurred, has been removed.
+                    The real underlying exceptions will be raised instead.
+                    If you have calls to
+                    <filename>bb.build.exec_func()</filename> in custom classes
+                    or <filename>tinfoil-using</filename> scripts, any
+                    references to <filename>bb.build.FuncFailed</filename>
+                    should be cleaned up.
+                    </para></listitem>
+                <listitem><para>
+                    Additionally, the
+                    <filename>bb.build.exec_func()</filename> no longer accepts
+                    the "pythonexception" parameter.
+                    The function now always raises exceptions.
+                    Remove this argument in any calls to
+                    <filename>bb.build.exec_func()</filename> in custom classes
+                    or scripts.
+                    </para></listitem>
+                <listitem><para>
+                    The
+                    <ulink url='&YOCTO_DOCS_BB_URL;#var-bb-BB_SETSCENE_VERIFY_FUNCTION2'><filename>BB_SETSCENE_VERIFY_FUNCTION2</filename></ulink>
+                    is no longer used.
+                    In the unlikely event that you have any references to it,
+                    they should be removed.
+                    </para></listitem>
+                <listitem><para>
+                    The <filename>RunQueueExecuteScenequeue</filename> and
+                    <filename>RunQueueExecuteTasks</filename> events have been
+                    removed since setscene tasks are now executed as part of
+                    the normal runqueue.
+                    Any event handling code in custom classes or scripts that
+                    handles these two events need to be updated.
+                    </para></listitem>
+                <listitem><para>
+                    The arguments passed to functions used with
+                    <ulink url='&YOCTO_DOCS_BB_URL;#var-bb-BB_HASHCHECK_FUNCTION'><filename>BB_HASHCHECK_FUNCTION</filename></ulink>
+                    have changed.
+                    If you are using your own custom hash check function, see
+                    <ulink url='http://git.yoctoproject.org/cgit/cgit.cgi/poky/commit/?id=40a5e193c4ba45c928fccd899415ea56b5417725'></ulink>
+                    for details.
+                    </para></listitem>
+                <listitem><para>
+                    Task specifications in <filename>BB_TASKDEPDATA</filename>
+                    and class implementations used in signature generator
+                    classes now use "&lt;fn&gt;:&lt;task&gt;" everywhere rather than
+                    the "." delimiter that was being used in some places.
+                    This change makes it consistent with all areas in the code.
+                    Custom signature generator classes and code that reads
+                    <filename>BB_TASKDEPDATA</filename> need to be updated to
+                    use ':' as a separator rather than '.'.
+                    </para></listitem>
+            </itemizedlist>
+        </para>
+    </section>
+
+    <section id='migration-3.0-sanity-checks'>
+        <title>Sanity Checks</title>
+
+        <para>
+            The following sanity check changes occurred.
+            <itemizedlist>
+                <listitem><para>
+                    <link linkend='var-SRC_URI'><filename>SRC_URI</filename></link>
+                    is now checked for usage of two problematic items:
+                    <itemizedlist>
+                        <listitem><para>
+                            "${PN}" prefix/suffix use - Warnings always appear
+                            if ${PN} is used.
+                            You must fix the issue regardless of whether
+                            multiconfig or anything else that would cause
+                            prefixing/suffixing to happen.
+                            </para></listitem>
+                        <listitem><para>
+                            Github archive tarballs - these are not guaranteed
+                            to be stable.
+                            Consequently, it is likely that the tarballs will
+                            be refreshed and thus the SRC_URI checksums
+                            will fail to apply.
+                            It is recommended that you fetch either an official
+                            release tarball or a specific revision from the
+                            actual Git repository instead.
+                            </para></listitem>
+                    </itemizedlist>
+                    Either one of these items now trigger a warning by default.
+                    If you wish to disable this check, remove
+                    <filename>src-uri-bad</filename> from
+                    <link linkend='var-WARN_QA'><filename>WARN_QA</filename></link>.
+                    </para></listitem>
+                <listitem><para>
+                    The <filename>file-rdeps</filename> runtime dependency
+                    check no longer expands
+                    <link linkend='var-RDEPENDS'><filename>RDEPENDS</filename></link>
+                    recursively as there is no mechanism to ensure they can be
+                    fully computed, and thus races sometimes result in errors
+                    either showing up or not.
+                    Thus, you might now see errors for missing runtime
+                    dependencies that were previously satisfied recursively.
+                    Here is an example: package A contains a shell script
+                    starting with <filename>#!/bin/bash</filename> but has no
+                    dependency on bash.
+                    However, package A depends on package B, which does depend
+                    on bash.
+                    You need to add the missing dependency or dependencies to
+                    resolve the warning.
+                    </para></listitem>
+                <listitem><para>
+                    Setting <filename>DEPENDS_${PN}</filename> anywhere
+                    (i.e. typically in a recipe) now triggers an error.
+                    The error is triggered because
+                    <link linkend='var-DEPENDS'><filename>DEPENDS</filename></link>
+                    is not a package-specific variable unlike RDEPENDS.
+                    You should set <filename>DEPENDS</filename> instead.
+                    </para></listitem>
+                <listitem><para>
+                    systemd currently does not work well with the musl C
+                    library because only upstream officially supports linking
+                    the library with glibc.
+                    Thus, a warning is shown when building systemd in
+                    conjunction with musl.
+                    </para></listitem>
+            </itemizedlist>
+        </para>
+    </section>
+
+    <section id='migration-3.0-miscellaneous-changes'>
+        <title>Miscellaneous Changes</title>
+
+        <para>
+            The following miscellaneous changes have occurred.
+            <itemizedlist>
+                <listitem><para>
+                    The <filename>gnome</filename>
+                    class has been removed because it now does very little.
+                    You should update recipes that previously inherited this
+                    class to do the following:
+                    <literallayout class='monospaced'>
+     inherit gnomebase gtk-icon-cache gconf mime
+                    </literallayout>
+                    </para></listitem>
+                <listitem><para>
+                    The
+                    <filename>meta/recipes-kernel/linux/linux-dtb.inc</filename>
+                    file has been removed.
+                    This file was previously deprecated in favor of setting
+                    <link linkend='var-KERNEL_DEVICETREE'><filename>KERNEL_DEVICETREE</filename></link>
+                    in any kernel recipe and only produced a warning.
+                    Remove any <filename>include</filename> or
+                    <filename>require</filename> statements pointing to this
+                    file.
+                    </para></listitem>
+                <listitem><para>
+                    <link linkend='var-TARGET_CFLAGS'><filename>TARGET_CFLAGS</filename></link>,
+                    <link linkend='var-TARGET_CPPFLAGS'><filename>TARGET_CPPFLAGS</filename></link>,
+                    <link linkend='var-TARGET_CXXFLAGS'><filename>TARGET_CXXFLAGS</filename></link>,
+                    and
+                    <link linkend='var-TARGET_LDFLAGS'><filename>TARGET_LDFLAGS</filename></link>
+                    are no longer exported to the external environment.
+                    This change did not require any changes to core recipes,
+                    which is a good indicator that no changes will be
+                    required.
+                    However, if for some reason the software being built by one
+                    of your recipes is expecting these variables to be set,
+                    then building the recipe will fail.
+                    In such cases, you must either export the variable or
+                    variables in the recipe or change the scripts so that
+                    exporting is not necessary.
+                    </para></listitem>
+                <listitem><para>
+                    You must change the host distro identifier used in
+                    <link linkend='var-NATIVELSBSTRING'><filename>NATIVELSBSTRING</filename></link>
+                    to use all lowercase characters even if it does not contain
+                    a version number.
+                    This change is necessary only if you are not using
+                    <filename>uninative</filename> and
+                    <link linkend='var-SANITY_TESTED_DISTROS'><filename>SANITY_TESTED_DISTROS</filename></link>.
+                    </para></listitem>
+                <listitem><para>
+                    In the <filename>base-files</filename> recipe, writing the
+                    hostname into <filename>/etc/hosts</filename> and
+                    <filename>/etc/hostname</filename> is now done within the
+                    main
+                    <link linkend='ref-tasks-install'><filename>do_install</filename></link>
+                    function rather than in the
+                    <filename>do_install_basefilesissue</filename> function.
+                    The reason for the change is because
+                    <filename>do_install_basefilesissue</filename> is more
+                    easily overridden without having to duplicate the hostname
+                    functionality.
+                    If you have done the latter (e.g. in a
+                    <filename>base-files</filename> bbappend), then you should
+                    remove it from your customized
+                    <filename>do_install_basefilesissue</filename> function.
+                    </para></listitem>
+                <listitem><para>
+                    The <filename>wic --expand</filename> command now uses
+                    commas to separate "key:value" pairs rather than hyphens.
+                    <note>
+                        The wic command-line help is not updated.
+                    </note>
+                    You must update any scripts or commands where you use
+                    <filename>wic --expand</filename> with multiple
+                    "key:value" pairs.
+                    </para></listitem>
+                <listitem><para>
+                    UEFI image variable settings have been moved from various
+                    places to a central
+                    <filename>conf/image-uefi.conf</filename>.
+                    This change should not influence any existing configuration
+                    as the <filename>meta/conf/image-uefi.conf</filename>
+                    in the core metadata sets defaults that can be overridden
+                    in the same manner as before.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>conf/distro/include/world-broken.inc</filename>
+                    has been removed.
+                    For cases where certain recipes need to be disabled when
+                    using the musl C library, these recipes now have
+                    <filename>COMPATIBLE_HOST_libc-musl</filename> set with a
+                    comment that explains why.
+                    </para></listitem>
+            </itemizedlist>
+        </para>
+    </section>
+</section>
+
 </chapter>
 <!--
 vim: expandtab tw=80 ts=4

documentation/ref-manual/ref-classes.xml

@@ -428,6 +428,14 @@
         variable to specify additional configuration options to be passed
         using the <filename>cmake</filename> command line.
     </para>
+
+    <para>
+        On the occasion that you would be installing custom CMake toolchain
+        files supplied by the application being built, you should install them
+        to the preferred CMake Module directory:
+        <filename>${D}${datadir}/cmake/</filename> Modules during
+        <ulink url='&YOCTO_DOCS_REF_URL;#ref-tasks-install'><filename>do_install</filename></ulink>.
+    </para>
 </section>
 
 <section id='ref-classes-cml1'>

documentation/ref-manual/ref-manual.xml

@@ -133,9 +133,14 @@
                 <revremark>Released with the Yocto Project 2.7 Release.</revremark>
             </revision>
             <revision>
-                <revnumber>2.8</revnumber>
+                <revnumber>3.0</revnumber>
+                <date>October</date>
+                <revremark>Released with the Yocto Project 3.0 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>3.1</revnumber>
                 <date>&REL_MONTH_YEAR;</date>
-                <revremark>Released with the Yocto Project 2.8 Release.</revremark>
+                <revremark>Released with the Yocto Project 3.1 Release.</revremark>
             </revision>
         </revhistory>
 

documentation/ref-manual/ref-variables.xml

@@ -1349,8 +1349,8 @@
             <glossdef>
                 <para role="glossdeffirst">
 <!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
-                    Specifies each additional separate configuration when you are
-                    building targets with multiple configurations.
+                    Specifies each additional separate configuration when you
+                    are building targets with multiple configurations.
                     Use this variable in your
                     <filename>conf/local.conf</filename> configuration file.
                     Specify a <replaceable>multiconfigname</replaceable> for
@@ -2182,9 +2182,10 @@
 
                 <para>
                     The <filename>BUSYBOX_SPLIT_SUID</filename> variable
-                    defaults to "1", which results in a single output
+                    defaults to "1", which results in splitting the output
                     executable file.
-                    Set the variable to "0" to split the output file.
+                    Set the variable to "0" to get a single output executable
+                    file.
                 </para>
             </glossdef>
         </glossentry>

documentation/sdk-manual/sdk-manual.xml

@@ -67,9 +67,14 @@
                 <revremark>Released with the Yocto Project 2.7 Release.</revremark>
             </revision>
             <revision>
-                <revnumber>2.8</revnumber>
+                <revnumber>3.0</revnumber>
+                <date>October</date>
+                <revremark>Released with the Yocto Project 3.0 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>3.1</revnumber>
                 <date>&REL_MONTH_YEAR;</date>
-                <revremark>Released with the Yocto Project 2.8 Release.</revremark>
+                <revremark>Released with the Yocto Project 3.1 Release.</revremark>
             </revision>
        </revhistory>
 

documentation/toaster-manual/toaster-manual.xml

@@ -77,9 +77,14 @@
                 <revremark>Released with the Yocto Project 2.7 Release.</revremark>
             </revision>
             <revision>
-                <revnumber>2.8</revnumber>
+                <revnumber>3.0</revnumber>
+                <date>October</date>
+                <revremark>Released with the Yocto Project 3.0 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>3.1</revnumber>
                 <date>&REL_MONTH_YEAR;</date>
-                <revremark>Released with the Yocto Project 2.8 Release.</revremark>
+                <revremark>Released with the Yocto Project 3.1 Release.</revremark>
             </revision>
        </revhistory>
 

documentation/tools/mega-manual.sed

@@ -1,36 +1,36 @@
 # Processes bitbake-user-manual (<word>-<word>-<word> style).
 # This style is for manual three-word folders, which currently is only the BitBake User Manual.
 # We used to have the "yocto-project-qs" and "poky-ref-manual" folders but no longer do.
-# s@"ulink" href="http://www.yoctoproject.org/docs/2.8/[a-z]*-[a-z]*-[a-z]*/[a-z]*-[a-z]*-[a-z]*.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.8/bitbake-user-manual/bitbake-user-manual.html#@"link" href="#@g
+# s@"ulink" href="http://www.yoctoproject.org/docs/3.1/[a-z]*-[a-z]*-[a-z]*/[a-z]*-[a-z]*-[a-z]*.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/3.1/bitbake-user-manual/bitbake-user-manual.html#@"link" href="#@g
 
 # Processes all other manuals (<word>-<word> style).
 # This style is for manual folders that use two word, which is the standard now (e.g. "ref-manual").
 # Here is the one-liner:
-# s@"ulink" href="http://www.yoctoproject.org/docs/2.8/[a-z]*-[a-z]*/[a-z]*-[a-z]*.html#@"link" href="#@g
+# s@"ulink" href="http://www.yoctoproject.org/docs/3.1/[a-z]*-[a-z]*/[a-z]*-[a-z]*.html#@"link" href="#@g
 
-s@"ulink" href="http://www.yoctoproject.org/docs/2.8/sdk-manual/sdk-manual.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.8/bsp-guide/bsp-guide.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.8/dev-manual/dev-manual.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.8/overview-manual/overview-manual.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.8/brief-yoctoprojectqs/brief-yoctoprojectqs.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.8/kernel-dev/kernel-dev.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.8/profile-manual/profile-manual.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.8/ref-manual/ref-manual.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.8/toaster-manual/toaster-manual.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/3.1/sdk-manual/sdk-manual.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/3.1/bsp-guide/bsp-guide.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/3.1/dev-manual/dev-manual.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/3.1/overview-manual/overview-manual.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/3.1/brief-yoctoprojectqs/brief-yoctoprojectqs.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/3.1/kernel-dev/kernel-dev.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/3.1/profile-manual/profile-manual.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/3.1/ref-manual/ref-manual.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/3.1/toaster-manual/toaster-manual.html#@"link" href="#@g
 
 # Process cases where just an external manual is referenced without an id anchor
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.8/brief-yoctoprojectqs/brief-yoctoprojectqs.html" target="_top">Yocto Project Quick Build</a>@Yocto Project Quick Build@g
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.8/bitbake-user-manual/bitbake-user-manual.html" target="_top">BitBake User Manual</a>@BitBake User Manual@g
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.8/dev-manual/dev-manual.html" target="_top">Yocto Project Development Tasks Manual</a>@Yocto Project Development Tasks Manual@g
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.8/overview-manual/overview-manual.html" target="_top">Yocto Project Overview and Concepts Manual</a>@Yocto project Overview and Concepts Manual@g
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.8/sdk-manual/sdk-manual.html" target="_top">Yocto Project Application Development and the Extensible Software Development Kit (eSDK)</a>@Yocto Project Application Development and the Extensible Software Development Kit (eSDK)@g
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.8/bsp-guide/bsp-guide.html" target="_top">Yocto Project Board Support Package (BSP) Developer's Guide</a>@Yocto Project Board Support Package (BSP) Developer's Guide@g
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.8/profile-manual/profile-manual.html" target="_top">Yocto Project Profiling and Tracing Manual</a>@Yocto Project Profiling and Tracing Manual@g
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.8/kernel-dev/kernel-dev.html" target="_top">Yocto Project Linux Kernel Development Manual</a>@Yocto Project Linux Kernel Development Manual@g
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.8/ref-manual/ref-manual.html" target="_top">Yocto Project Reference Manual</a>@Yocto Project Reference Manual@g
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.8/toaster-manual/toaster-manual.html" target="_top">Toaster User Manual</a>@Toaster User Manual@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/3.1/brief-yoctoprojectqs/brief-yoctoprojectqs.html" target="_top">Yocto Project Quick Build</a>@Yocto Project Quick Build@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/3.1/bitbake-user-manual/bitbake-user-manual.html" target="_top">BitBake User Manual</a>@BitBake User Manual@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/3.1/dev-manual/dev-manual.html" target="_top">Yocto Project Development Tasks Manual</a>@Yocto Project Development Tasks Manual@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/3.1/overview-manual/overview-manual.html" target="_top">Yocto Project Overview and Concepts Manual</a>@Yocto project Overview and Concepts Manual@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/3.1/sdk-manual/sdk-manual.html" target="_top">Yocto Project Application Development and the Extensible Software Development Kit (eSDK)</a>@Yocto Project Application Development and the Extensible Software Development Kit (eSDK)@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/3.1/bsp-guide/bsp-guide.html" target="_top">Yocto Project Board Support Package (BSP) Developer's Guide</a>@Yocto Project Board Support Package (BSP) Developer's Guide@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/3.1/profile-manual/profile-manual.html" target="_top">Yocto Project Profiling and Tracing Manual</a>@Yocto Project Profiling and Tracing Manual@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/3.1/kernel-dev/kernel-dev.html" target="_top">Yocto Project Linux Kernel Development Manual</a>@Yocto Project Linux Kernel Development Manual@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/3.1/ref-manual/ref-manual.html" target="_top">Yocto Project Reference Manual</a>@Yocto Project Reference Manual@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/3.1/toaster-manual/toaster-manual.html" target="_top">Toaster User Manual</a>@Toaster User Manual@g
 
 # Process a single, rouge occurrence of a linked reference to the Mega-Manual.
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.8/mega-manual/mega-manual.html" target="_top">Yocto Project Mega-Manual</a>@Yocto Project Mega-Manual@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/3.1/mega-manual/mega-manual.html" target="_top">Yocto Project Mega-Manual</a>@Yocto Project Mega-Manual@g
 

meta-poky/conf/distro/poky.conf

@@ -87,3 +87,8 @@ require conf/distro/include/no-static-libs.inc
 require conf/distro/include/yocto-uninative.inc
 require conf/distro/include/security_flags.inc
 INHERIT += "uninative"
+
+INHERIT += "reproducible_build"
+
+BB_SIGNATURE_HANDLER ?= "OEEquivHash"
+BB_HASHSERVE ??= "auto"

meta-selftest/lib/oeqa/runtime/cases/virgl.py

@@ -13,11 +13,6 @@ class VirglTest(OERuntimeTestCase):
 
     @OETestDepends(['virgl.VirglTest.test_kernel_driver'])
     def test_kmscube(self):
-
-        distro = oe.lsb.distro_identifier()
-        if distro and distro == 'centos-7':
-            self.skipTest('kmscube is not working when centos 7 is the host OS')
-
         status, output = self.target.run('kmscube', timeout=30)
         self.assertEqual(status, 0, "kmscube exited with non-zero status %d and output:\n%s" %(status, output))
         self.assertIn('renderer: "virgl"', output, "kmscube does not seem to use virgl:\n%s" %(output))

meta-selftest/recipes-test/devtool/devtool-upgrade-test1-1.5.3/backported.patch

@@ -0,0 +1,37 @@
+commit ced2ec32b657a7f52604b2e16e5d5881041c517a
+Author: OpenEmbedded <oe.patch@oe>
+Date:   Mon Nov 18 18:43:15 2019 +0100
+
+    Backport of the NEWS file from version 1.6.0
+
+diff --git a/doc/NEWS b/doc/NEWS
+index 69793fc..fd49b1c 100644
+--- a/doc/NEWS
++++ b/doc/NEWS
+@@ -1,3 +1,26 @@
++1.6.0 - 15 March 2015
++  - fix lstat64 support when unavailable - separate patches supplied by
++    Ganael Laplanche and Peter Korsgaard
++  - (#1506) new option "-D" / "--delay-start" to only show bar after N
++    seconds (Damon Harper)
++  - new option "--fineta" / "-I" to show ETA as time of day rather than time
++    remaining - patch supplied by Erkki Seppälä (r147)
++  - (#1509) change ETA (--eta / -e) so that days are given if the hours
++    remaining are 24 or more (Jacek Wielemborek)
++  - (#1499) repeat read and write attempts on partial buffer fill/empty to
++    work around post-signal transfer rate drop reported by Ralf Ramsauer
++  - (#1507) do not try to calculate total size in line mode, due to bug
++    reported by Jacek Wielemborek and Michiel Van Herwegen
++  - cleanup: removed defunct RATS comments and unnecessary copyright notices
++  - clean up displayed lines when using --watchfd PID, when PID exits
++  - output errors on a new line to avoid overwriting transfer bar
++
++1.5.7 - 26 August 2014
++  - show KiB instead of incorrect kiB (Debian bug #706175)
++  - (#1284) do not gzip man page, for non-Linux OSes (Bob Friesenhahn)
++  - work around "awk" bug in tests/016-numeric-timer in decimal "," locales
++  - fix "make rpm" and "make srpm", extend "make release" to sign releases
++
+ 1.5.3 - 4 May 2014
+   - remove SPLICE_F_NONBLOCK to fix problem with slow splice() (Jan Seda)
+ 

meta-selftest/recipes-test/devtool/devtool-upgrade-test1_1.5.3.bb

@@ -3,7 +3,9 @@ LICENSE = "Artistic-2.0"
 LIC_FILES_CHKSUM = "file://doc/COPYING;md5=9c50db2589ee3ef10a9b7b2e50ce1d02"
 
 SRC_URI = "http://www.ivarch.com/programs/sources/pv-${PV}.tar.gz \
-           file://0001-Add-a-note-line-to-the-quick-reference.patch"
+           file://0001-Add-a-note-line-to-the-quick-reference.patch \
+           file://backported.patch \
+          "
 UPSTREAM_CHECK_URI = "http://www.ivarch.com/programs/pv.shtml"
 RECIPE_NO_UPDATE_REASON = "This recipe is used to test devtool upgrade feature"
 

meta-selftest/recipes-test/devtool/devtool-upgrade-test1_1.5.3.bb.upgraded

@@ -3,7 +3,9 @@ LICENSE = "Artistic-2.0"
 LIC_FILES_CHKSUM = "file://doc/COPYING;md5=9c50db2589ee3ef10a9b7b2e50ce1d02"
 
 SRC_URI = "http://www.ivarch.com/programs/sources/pv-${PV}.tar.gz \
-           file://0001-Add-a-note-line-to-the-quick-reference.patch"
+           file://0001-Add-a-note-line-to-the-quick-reference.patch \
+           file://backported.patch \
+          "
 UPSTREAM_CHECK_URI = "http://www.ivarch.com/programs/pv.shtml"
 RECIPE_NO_UPDATE_REASON = "This recipe is used to test devtool upgrade feature"
 

meta-selftest/recipes-test/recipeutils/recipeutils-test_1.2.bb

@@ -4,7 +4,7 @@ require recipeutils-test.inc
 
 LICENSE = "Proprietary"
 LIC_FILES_CHKSUM = "file://${WORKDIR}/somefile;md5=d41d8cd98f00b204e9800998ecf8427e"
-DEPENDS += "virtual/libx11"
+DEPENDS += "zlib"
 
 BBCLASSEXTEND = "native nativesdk"
 

meta/classes/archiver.bbclass

@@ -441,9 +441,10 @@ python do_ar_recipe () {
             incfile = include_re.match(line).group(1)
         if incfile:
             incfile = d.expand(incfile)
+        if incfile:
             incfile = bb.utils.which(bbpath, incfile)
-            if incfile:
-                shutil.copy(incfile, outdir)
+        if incfile:
+            shutil.copy(incfile, outdir)
 
     create_tarball(d, outdir, 'recipe', d.getVar('ARCHIVER_OUTDIR'))
     bb.utils.remove(outdir, recurse=True)

meta/classes/base.bbclass

@@ -138,11 +138,6 @@ def setup_hosttools_dir(dest, toolsvar, d, fatal=True):
                 os.symlink(srctool, desttool)
             else:
                 notfound.append(tool)
-    # Force "python" -> "python2"
-    desttool = os.path.join(dest, "python")
-    if not os.path.exists(desttool):
-        srctool = "python2"
-        os.symlink(srctool, desttool)
 
     if notfound and fatal:
         bb.fatal("The following required tools (as specified by HOSTTOOLS) appear to be unavailable in PATH, please install them in order to proceed:\n  %s" % " ".join(notfound))
@@ -482,6 +477,7 @@ python () {
     # If we're building a target package we need to use fakeroot (pseudo)
     # in order to capture permissions, owners, groups and special files
     if not bb.data.inherits_class('native', d) and not bb.data.inherits_class('cross', d):
+        d.appendVarFlag('do_prepare_recipe_sysroot', 'depends', ' virtual/fakeroot-native:do_populate_sysroot')
         d.setVarFlag('do_unpack', 'umask', '022')
         d.setVarFlag('do_configure', 'umask', '022')
         d.setVarFlag('do_compile', 'umask', '022')
@@ -605,6 +601,7 @@ python () {
         # Mercurial packages should DEPEND on mercurial-native
         elif scheme == "hg":
             needsrcrev = True
+            d.appendVar("EXTRANATIVEPATH", ' python3-native ')
             d.appendVarFlag('do_fetch', 'depends', ' mercurial-native:do_populate_sysroot')
 
         # Perforce packages support SRCREV = "${AUTOREV}"

meta/classes/buildhistory.bbclass

@@ -280,7 +280,7 @@ python buildhistory_emit_pkghistory() {
             last_pkgr = lastversion.pkgr
             r = bb.utils.vercmp((pkge, pkgv, pkgr), (last_pkge, last_pkgv, last_pkgr))
             if r < 0:
-                msg = "Package version for package %s went backwards which would break package feeds from (%s:%s-%s to %s:%s-%s)" % (pkg, last_pkge, last_pkgv, last_pkgr, pkge, pkgv, pkgr)
+                msg = "Package version for package %s went backwards which would break package feeds (from %s:%s-%s to %s:%s-%s)" % (pkg, last_pkge, last_pkgv, last_pkgr, pkge, pkgv, pkgr)
                 package_qa_handle_error("version-going-backwards", msg, d)
 
         pkginfo = PackageInfo(pkg)

meta/classes/cmake.bbclass

@@ -106,11 +106,12 @@ set( CMAKE_CXX_LINK_FLAGS "${OECMAKE_CXX_LINK_FLAGS}" CACHE STRING "LDFLAGS" )
 
 # only search in the paths provided so cmake doesnt pick
 # up libraries and tools from the native build machine
-set( CMAKE_FIND_ROOT_PATH ${STAGING_DIR_HOST} ${STAGING_DIR_NATIVE} ${CROSS_DIR} ${OECMAKE_PERLNATIVE_DIR} ${OECMAKE_EXTRA_ROOT_PATH} ${EXTERNAL_TOOLCHAIN})
+set( CMAKE_FIND_ROOT_PATH ${STAGING_DIR_HOST} ${STAGING_DIR_NATIVE} ${CROSS_DIR} ${OECMAKE_PERLNATIVE_DIR} ${OECMAKE_EXTRA_ROOT_PATH} ${EXTERNAL_TOOLCHAIN} ${HOSTTOOLS_DIR})
 set( CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY )
 set( CMAKE_FIND_ROOT_PATH_MODE_PROGRAM ${OECMAKE_FIND_ROOT_PATH_MODE_PROGRAM} )
 set( CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY )
 set( CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY )
+set( CMAKE_PROGRAM_PATH "/" )
 
 # Use qt.conf settings
 set( ENV{QT_CONF_PATH} ${WORKDIR}/qt.conf )
@@ -119,6 +120,9 @@ set( ENV{QT_CONF_PATH} ${WORKDIR}/qt.conf )
 # directory as rpath by default
 set( CMAKE_INSTALL_RPATH ${OECMAKE_RPATH} )
 
+# Use RPATHs relative to build directory for reproducibility
+set( CMAKE_BUILD_RPATH_USE_ORIGIN ON )
+
 # Use our cmake modules
 list(APPEND CMAKE_MODULE_PATH "${STAGING_DATADIR}/cmake/Modules/")
 

meta/classes/cross.bbclass

@@ -97,3 +97,4 @@ python do_addto_recipe_sysroot () {
     bb.build.exec_func("extend_recipe_sysroot", d)
 }
 addtask addto_recipe_sysroot after do_populate_sysroot
+do_addto_recipe_sysroot[deptask] = "do_populate_sysroot"

meta/classes/cve-check.bbclass

@@ -26,7 +26,7 @@ CVE_PRODUCT ??= "${BPN}"
 CVE_VERSION ??= "${PV}"
 
 CVE_CHECK_DB_DIR ?= "${DL_DIR}/CVE_CHECK"
-CVE_CHECK_DB_FILE ?= "${CVE_CHECK_DB_DIR}/nvdcve_1.0.db"
+CVE_CHECK_DB_FILE ?= "${CVE_CHECK_DB_DIR}/nvdcve_1.1.db"
 
 CVE_CHECK_LOG ?= "${T}/cve.log"
 CVE_CHECK_TMP_FILE ?= "${TMPDIR}/cve_check"
@@ -62,7 +62,7 @@ python do_cve_check () {
 
 }
 
-addtask cve_check after do_unpack before do_build
+addtask cve_check before do_build
 do_cve_check[depends] = "cve-update-db-native:do_populate_cve_db"
 do_cve_check[nostamp] = "1"
 
@@ -70,7 +70,6 @@ python cve_check_cleanup () {
     """
     Delete the file used to gather all the CVE information.
     """
-
     bb.utils.remove(e.data.getVar("CVE_CHECK_TMP_FILE"))
 }
 
@@ -166,7 +165,6 @@ def check_cves(d, patched_cves):
     """
     Connect to the NVD database and find unpatched cves.
     """
-    import ast, csv, tempfile, subprocess, io
     from distutils.version import LooseVersion
 
     cves_unpatched = []
@@ -188,63 +186,74 @@ def check_cves(d, patched_cves):
     cve_whitelist = d.getVar("CVE_CHECK_WHITELIST").split()
 
     import sqlite3
-    db_file = d.getVar("CVE_CHECK_DB_FILE")
-    conn = sqlite3.connect(db_file)
+    db_file = d.expand("file:${CVE_CHECK_DB_FILE}?mode=ro")
+    conn = sqlite3.connect(db_file, uri=True)
 
+    # For each of the known product names (e.g. curl has CPEs using curl and libcurl)...
     for product in products:
-        c = conn.cursor()
         if ":" in product:
             vendor, product = product.split(":", 1)
-            c.execute("SELECT * FROM PRODUCTS WHERE PRODUCT IS ? AND VENDOR IS ?", (product, vendor))
         else:
-            c.execute("SELECT * FROM PRODUCTS WHERE PRODUCT IS ?", (product,))
+            vendor = "%"
 
-        for row in c:
-            cve = row[0]
-            version_start = row[3]
-            operator_start = row[4]
-            version_end = row[5]
-            operator_end = row[6]
+        # Find all relevant CVE IDs.
+        for cverow in conn.execute("SELECT DISTINCT ID FROM PRODUCTS WHERE PRODUCT IS ? AND VENDOR LIKE ?", (product, vendor)):
+            cve = cverow[0]
 
             if cve in cve_whitelist:
                 bb.note("%s-%s has been whitelisted for %s" % (product, pv, cve))
+                # TODO: this should be in the report as 'whitelisted'
+                patched_cves.add(cve)
+                continue
             elif cve in patched_cves:
                 bb.note("%s has been patched" % (cve))
-            else:
-                to_append = False
+                continue
+
+            vulnerable = False
+            for row in conn.execute("SELECT * FROM PRODUCTS WHERE ID IS ? AND PRODUCT IS ? AND VENDOR LIKE ?", (cve, product, vendor)):
+                (_, _, _, version_start, operator_start, version_end, operator_end) = row
+                #bb.debug(2, "Evaluating row " + str(row))
+
                 if (operator_start == '=' and pv == version_start):
-                    cves_unpatched.append(cve)
+                    vulnerable = True
                 else:
                     if operator_start:
                         try:
-                            to_append_start =  (operator_start == '>=' and LooseVersion(pv) >= LooseVersion(version_start))
-                            to_append_start |= (operator_start == '>' and LooseVersion(pv) > LooseVersion(version_start))
+                            vulnerable_start =  (operator_start == '>=' and LooseVersion(pv) >= LooseVersion(version_start))
+                            vulnerable_start |= (operator_start == '>' and LooseVersion(pv) > LooseVersion(version_start))
                         except:
-                            bb.note("%s: Failed to compare %s %s %s for %s" %
+                            bb.warn("%s: Failed to compare %s %s %s for %s" %
                                     (product, pv, operator_start, version_start, cve))
-                            to_append_start = False
+                            vulnerable_start = False
                     else:
-                        to_append_start = False
+                        vulnerable_start = False
 
                     if operator_end:
                         try:
-                            to_append_end  = (operator_end == '<=' and LooseVersion(pv) <= LooseVersion(version_end))
-                            to_append_end |= (operator_end == '<' and LooseVersion(pv) < LooseVersion(version_end))
+                            vulnerable_end  = (operator_end == '<=' and LooseVersion(pv) <= LooseVersion(version_end))
+                            vulnerable_end |= (operator_end == '<' and LooseVersion(pv) < LooseVersion(version_end))
                         except:
-                            bb.note("%s: Failed to compare %s %s %s for %s" %
+                            bb.warn("%s: Failed to compare %s %s %s for %s" %
                                     (product, pv, operator_end, version_end, cve))
-                            to_append_end = False
+                            vulnerable_end = False
                     else:
-                        to_append_end = False
+                        vulnerable_end = False
 
                     if operator_start and operator_end:
-                        to_append = to_append_start and to_append_end
+                        vulnerable = vulnerable_start and vulnerable_end
                     else:
-                        to_append = to_append_start or to_append_end
+                        vulnerable = vulnerable_start or vulnerable_end
 
-                if to_append:
+                if vulnerable:
+                    bb.note("%s-%s is vulnerable to %s" % (product, pv, cve))
                     cves_unpatched.append(cve)
-                bb.debug(2, "%s-%s is not patched for %s" % (product, pv, cve))
+                    break
+
+            if not vulnerable:
+                bb.note("%s-%s is not vulnerable to %s" % (product, pv, cve))
+                # TODO: not patched but not vulnerable
+                patched_cves.add(cve)
+
     conn.close()
 
     return (list(patched_cves), cves_unpatched)
@@ -252,31 +261,23 @@ def check_cves(d, patched_cves):
 def get_cve_info(d, cves):
     """
     Get CVE information from the database.
-
-    Unfortunately the only way to get CVE info is set the output to
-    html (hard to parse) or query directly the database.
     """
 
-    try:
-        import sqlite3
-    except ImportError:
-        from pysqlite2 import dbapi2 as sqlite3
+    import sqlite3
 
     cve_data = {}
-    db_file = d.getVar("CVE_CHECK_DB_FILE")
-    placeholder = ",".join("?" * len(cves))
-    query = "SELECT * FROM NVD WHERE id IN (%s)" % placeholder
-    conn = sqlite3.connect(db_file)
-    cur = conn.cursor()
-    for row in cur.execute(query, tuple(cves)):
-        cve_data[row[0]] = {}
-        cve_data[row[0]]["summary"] = row[1]
-        cve_data[row[0]]["scorev2"] = row[2]
-        cve_data[row[0]]["scorev3"] = row[3]
-        cve_data[row[0]]["modified"] = row[4]
-        cve_data[row[0]]["vector"] = row[5]
-    conn.close()
+    conn = sqlite3.connect(d.getVar("CVE_CHECK_DB_FILE"))
 
+    for cve in cves:
+        for row in conn.execute("SELECT * FROM NVD WHERE ID IS ?", (cve,)):
+            cve_data[row[0]] = {}
+            cve_data[row[0]]["summary"] = row[1]
+            cve_data[row[0]]["scorev2"] = row[2]
+            cve_data[row[0]]["scorev3"] = row[3]
+            cve_data[row[0]]["modified"] = row[4]
+            cve_data[row[0]]["vector"] = row[5]
+
+    conn.close()
     return cve_data
 
 def cve_write_data(d, patched, unpatched, cve_data):

meta/classes/devtool-source.bbclass

@@ -97,17 +97,15 @@ python devtool_post_unpack() {
     local_files = oe.recipeutils.get_recipe_local_files(d)
 
     if is_kernel_yocto:
-        for key in local_files.copy():
-            if key.endswith('scc'):
-                sccfile = open(local_files[key], 'r')
+        for key in [f for f in local_files if f.endswith('scc')]:
+            with open(local_files[key], 'r') as sccfile:
                 for l in sccfile:
                     line = l.split()
                     if line and line[0] in ('kconf', 'patch'):
                         cfg = os.path.join(os.path.dirname(local_files[key]), line[-1])
-                        if not cfg in local_files.values():
+                        if cfg not in local_files.values():
                             local_files[line[-1]] = cfg
                             shutil.copy2(cfg, workdir)
-                sccfile.close()
 
     # Ignore local files with subdir={BP}
     srcabspath = os.path.abspath(srcsubdir)

meta/classes/distro_features_check.bbclass

@@ -1,32 +1,7 @@
-# Allow checking of required and conflicting DISTRO_FEATURES
-#
-# ANY_OF_DISTRO_FEATURES:   ensure at least one item on this list is included
-#                           in DISTRO_FEATURES.
-# REQUIRED_DISTRO_FEATURES: ensure every item on this list is included
-#                           in DISTRO_FEATURES.
-# CONFLICT_DISTRO_FEATURES: ensure no item in this list is included in
-#                           DISTRO_FEATURES.
-#
-# Copyright 2013 (C) O.S. Systems Software LTDA.
+# Temporarily provide fallback to the old name of the class
 
-python () {
-    # Assume at least one var is set.
-    distro_features = set((d.getVar('DISTRO_FEATURES') or '').split())
-
-    any_of_distro_features = set((d.getVar('ANY_OF_DISTRO_FEATURES') or '').split())
-    if any_of_distro_features:
-        if set.isdisjoint(any_of_distro_features, distro_features):
-            raise bb.parse.SkipRecipe("one of '%s' needs to be in DISTRO_FEATURES" % ' '.join(any_of_distro_features))
-
-    required_distro_features = set((d.getVar('REQUIRED_DISTRO_FEATURES') or '').split())
-    if required_distro_features:
-        missing = set.difference(required_distro_features, distro_features)
-        if missing:
-            raise bb.parse.SkipRecipe("missing required distro feature%s '%s' (not in DISTRO_FEATURES)" % ('s' if len(missing) > 1 else '', ' '.join(missing)))
-
-    conflict_distro_features = set((d.getVar('CONFLICT_DISTRO_FEATURES') or '').split())
-    if conflict_distro_features:
-        conflicts = set.intersection(conflict_distro_features, distro_features)
-        if conflicts:
-            raise bb.parse.SkipRecipe("conflicting distro feature%s '%s' (in DISTRO_FEATURES)" % ('s' if len(conflicts) > 1 else '', ' '.join(conflicts)))
+python __anonymous() {
+    bb.warn("distro_features_check.bbclass is deprecated, please use features_check.bbclass instead")
 }
+
+inherit features_check

meta/classes/features_check.bbclass

@@ -0,0 +1,85 @@
+# Allow checking of required and conflicting DISTRO_FEATURES
+#
+# ANY_OF_DISTRO_FEATURES:     ensure at least one item on this list is included
+#                             in DISTRO_FEATURES.
+# REQUIRED_DISTRO_FEATURES:   ensure every item on this list is included
+#                             in DISTRO_FEATURES.
+# CONFLICT_DISTRO_FEATURES:   ensure no item in this list is included in
+#                             DISTRO_FEATURES.
+# ANY_OF_MACHINE_FEATURES:    ensure at least one item on this list is included
+#                             in MACHINE_FEATURES.
+# REQUIRED_MACHINE_FEATURES:  ensure every item on this list is included
+#                             in MACHINE_FEATURES.
+# CONFLICT_MACHINE_FEATURES:  ensure no item in this list is included in
+#                             MACHINE_FEATURES.
+# ANY_OF_COMBINED_FEATURES:   ensure at least one item on this list is included
+#                             in COMBINED_FEATURES.
+# REQUIRED_COMBINED_FEATURES: ensure every item on this list is included
+#                             in COMBINED_FEATURES.
+# CONFLICT_COMBINED_FEATURES: ensure no item in this list is included in
+#                             COMBINED_FEATURES.
+#
+# Copyright 2019 (C) Texas Instruments Inc.
+# Copyright 2013 (C) O.S. Systems Software LTDA.
+
+python () {
+    # Assume at least one var is set.
+    distro_features = set((d.getVar('DISTRO_FEATURES') or '').split())
+
+    any_of_distro_features = set((d.getVar('ANY_OF_DISTRO_FEATURES') or '').split())
+    if any_of_distro_features:
+        if set.isdisjoint(any_of_distro_features, distro_features):
+            raise bb.parse.SkipRecipe("one of '%s' needs to be in DISTRO_FEATURES" % ' '.join(any_of_distro_features))
+
+    required_distro_features = set((d.getVar('REQUIRED_DISTRO_FEATURES') or '').split())
+    if required_distro_features:
+        missing = set.difference(required_distro_features, distro_features)
+        if missing:
+            raise bb.parse.SkipRecipe("missing required distro feature%s '%s' (not in DISTRO_FEATURES)" % ('s' if len(missing) > 1 else '', ' '.join(missing)))
+
+    conflict_distro_features = set((d.getVar('CONFLICT_DISTRO_FEATURES') or '').split())
+    if conflict_distro_features:
+        conflicts = set.intersection(conflict_distro_features, distro_features)
+        if conflicts:
+            raise bb.parse.SkipRecipe("conflicting distro feature%s '%s' (in DISTRO_FEATURES)" % ('s' if len(conflicts) > 1 else '', ' '.join(conflicts)))
+
+    # Assume at least one var is set.
+    machine_features = set((d.getVar('MACHINE_FEATURES') or '').split())
+
+    any_of_machine_features = set((d.getVar('ANY_OF_MACHINE_FEATURES') or '').split())
+    if any_of_machine_features:
+        if set.isdisjoint(any_of_machine_features, machine_features):
+            raise bb.parse.SkipRecipe("one of '%s' needs to be in MACHINE_FEATURES" % ' '.join(any_of_machine_features))
+
+    required_machine_features = set((d.getVar('REQUIRED_MACHINE_FEATURES') or '').split())
+    if required_machine_features:
+        missing = set.difference(required_machine_features, machine_features)
+        if missing:
+            raise bb.parse.SkipRecipe("missing required machine feature%s '%s' (not in MACHINE_FEATURES)" % ('s' if len(missing) > 1 else '', ' '.join(missing)))
+
+    conflict_machine_features = set((d.getVar('CONFLICT_MACHINE_FEATURES') or '').split())
+    if conflict_machine_features:
+        conflicts = set.intersection(conflict_machine_features, machine_features)
+        if conflicts:
+            raise bb.parse.SkipRecipe("conflicting machine feature%s '%s' (in MACHINE_FEATURES)" % ('s' if len(conflicts) > 1 else '', ' '.join(conflicts)))
+
+    # Assume at least one var is set.
+    combined_features = set((d.getVar('COMBINED_FEATURES') or '').split())
+
+    any_of_combined_features = set((d.getVar('ANY_OF_COMBINED_FEATURES') or '').split())
+    if any_of_combined_features:
+        if set.isdisjoint(any_of_combined_features, combined_features):
+            raise bb.parse.SkipRecipe("one of '%s' needs to be in COMBINED_FEATURES" % ' '.join(any_of_combined_features))
+
+    required_combined_features = set((d.getVar('REQUIRED_COMBINED_FEATURES') or '').split())
+    if required_combined_features:
+        missing = set.difference(required_combined_features, combined_features)
+        if missing:
+            raise bb.parse.SkipRecipe("missing required machine feature%s '%s' (not in COMBINED_FEATURES)" % ('s' if len(missing) > 1 else '', ' '.join(missing)))
+
+    conflict_combined_features = set((d.getVar('CONFLICT_COMBINED_FEATURES') or '').split())
+    if conflict_combined_features:
+        conflicts = set.intersection(conflict_combined_features, combined_features)
+        if conflicts:
+            raise bb.parse.SkipRecipe("conflicting machine feature%s '%s' (in COMBINED_FEATURES)" % ('s' if len(conflicts) > 1 else '', ' '.join(conflicts)))
+}

meta/classes/icecc.bbclass

@@ -356,17 +356,6 @@ set_icecc_env() {
         return
     fi
 
-    # Create symlinks to icecc in the recipe-sysroot directory
-    mkdir -p ${ICE_PATH}
-    if [ -n "${KERNEL_CC}" ]; then
-        compilers="${@get_cross_kernel_cc(bb,d)}"
-    else
-        compilers="${HOST_PREFIX}gcc ${HOST_PREFIX}g++"
-    fi
-    for compiler in $compilers; do
-        ln -sf ${ICECC_BIN} ${ICE_PATH}/$compiler
-    done
-
     ICECC_CC="${@icecc_get_and_check_tool(bb, d, "gcc")}"
     ICECC_CXX="${@icecc_get_and_check_tool(bb, d, "g++")}"
     # cannot use icecc_get_and_check_tool here because it assumes as without target_sys prefix
@@ -385,6 +374,26 @@ set_icecc_env() {
         return
     fi
 
+    # Create symlinks to icecc and wrapper-scripts in the recipe-sysroot directory
+    mkdir -p $ICE_PATH/symlinks
+    if [ -n "${KERNEL_CC}" ]; then
+        compilers="${@get_cross_kernel_cc(bb,d)}"
+    else
+        compilers="${HOST_PREFIX}gcc ${HOST_PREFIX}g++"
+    fi
+    for compiler in $compilers; do
+        ln -sf $ICECC_BIN $ICE_PATH/symlinks/$compiler
+        rm -f $ICE_PATH/$compiler
+        cat <<-__EOF__ > $ICE_PATH/$compiler
+		#!/bin/sh -e
+		export ICECC_VERSION=$ICECC_VERSION
+		export ICECC_CC=$ICECC_CC
+		export ICECC_CXX=$ICECC_CXX
+		$ICE_PATH/symlinks/$compiler "\$@"
+		__EOF__
+        chmod 775 $ICE_PATH/$compiler
+    done
+
     ICECC_AS="`${ICECC_CC} -print-prog-name=as`"
     # for target recipes should return something like:
     # /OE/tmp-eglibc/sysroots/x86_64-linux/usr/libexec/arm920tt-oe-linux-gnueabi/gcc/arm-oe-linux-gnueabi/4.8.2/as
@@ -417,7 +426,6 @@ set_icecc_env() {
     export CCACHE_PATH="$PATH"
     export CCACHE_DISABLE="1"
 
-    export ICECC_VERSION ICECC_CC ICECC_CXX
     export PATH="$ICE_PATH:$PATH"
 
     bbnote "Using icecc path: $ICE_PATH"

meta/classes/image.bbclass

@@ -124,7 +124,7 @@ python () {
 def rootfs_variables(d):
     from oe.rootfs import variable_depends
     variables = ['IMAGE_DEVICE_TABLE','IMAGE_DEVICE_TABLES','BUILD_IMAGES_FROM_FEEDS','IMAGE_TYPES_MASKED','IMAGE_ROOTFS_ALIGNMENT','IMAGE_OVERHEAD_FACTOR','IMAGE_ROOTFS_SIZE','IMAGE_ROOTFS_EXTRA_SPACE',
-                 'IMAGE_ROOTFS_MAXSIZE','IMAGE_NAME','IMAGE_LINK_NAME','IMAGE_MANIFEST','DEPLOY_DIR_IMAGE','IMAGE_FSTYPES','IMAGE_INSTALL_COMPLEMENTARY','IMAGE_LINGUAS',
+                 'IMAGE_ROOTFS_MAXSIZE','IMAGE_NAME','IMAGE_LINK_NAME','IMAGE_MANIFEST','DEPLOY_DIR_IMAGE','IMAGE_FSTYPES','IMAGE_INSTALL_COMPLEMENTARY','IMAGE_LINGUAS', 'IMAGE_LINGUAS_COMPLEMENTARY',
                  'MULTILIBRE_ALLOW_REP','MULTILIB_TEMP_ROOTFS','MULTILIB_VARIANTS','MULTILIBS','ALL_MULTILIB_PACKAGE_ARCHS','MULTILIB_GLOBAL_VARIANTS','BAD_RECOMMENDATIONS','NO_RECOMMENDATIONS',
                  'PACKAGE_ARCHS','PACKAGE_CLASSES','TARGET_VENDOR','TARGET_ARCH','TARGET_OS','OVERRIDES','BBEXTENDVARIANT','FEED_DEPLOYDIR_BASE_URI','INTERCEPT_DIR','USE_DEVFS',
                  'CONVERSIONTYPES', 'IMAGE_GEN_DEBUGFS', 'ROOTFS_RO_UNNEEDED', 'IMGDEPLOYDIR', 'PACKAGE_EXCLUDE_COMPLEMENTARY', 'REPRODUCIBLE_TIMESTAMP_ROOTFS', 'IMAGE_INSTALL_DEBUGFS']

meta/classes/image_types.bbclass

@@ -59,6 +59,8 @@ XZ_INTEGRITY_CHECK ?= "crc32"
 
 ZIP_COMPRESSION_LEVEL ?= "-9"
 
+ZSTD_COMPRESSION_LEVEL ?= "-3"
+
 JFFS2_SUM_EXTRA_ARGS ?= ""
 IMAGE_CMD_jffs2 = "mkfs.jffs2 --root=${IMAGE_ROOTFS} --faketime --output=${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.jffs2 ${EXTRA_IMAGECMD}"
 
@@ -128,7 +130,7 @@ IMAGE_CMD_tar = "${IMAGE_CMD_TAR} --numeric-owner -cf ${IMGDEPLOYDIR}/${IMAGE_NA
 
 do_image_cpio[cleandirs] += "${WORKDIR}/cpio_append"
 IMAGE_CMD_cpio () {
-	(cd ${IMAGE_ROOTFS} && find . | cpio -o -H newc >${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.cpio)
+	(cd ${IMAGE_ROOTFS} && find . | sort | cpio --reproducible -o -H newc >${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.cpio)
 	# We only need the /init symlink if we're building the real
 	# image. The -dbg image doesn't need it! By being clever
 	# about this we also avoid 'touch' below failing, as it
@@ -269,7 +271,7 @@ IMAGE_TYPES = " \
     hddimg \
     squashfs squashfs-xz squashfs-lzo squashfs-lz4 \
     ubi ubifs multiubi \
-    tar tar.gz tar.bz2 tar.xz tar.lz4 \
+    tar tar.gz tar.bz2 tar.xz tar.lz4 tar.zst \
     cpio cpio.gz cpio.xz cpio.lzma cpio.lz4 \
     wic wic.gz wic.bz2 wic.lzma \
     container \
@@ -282,7 +284,7 @@ IMAGE_TYPES = " \
 # CONVERSION_CMD/DEPENDS.
 COMPRESSIONTYPES ?= ""
 
-CONVERSIONTYPES = "gz bz2 lzma xz lz4 lzo zip sum md5sum sha1sum sha224sum sha256sum sha384sum sha512sum bmap u-boot vmdk vdi qcow2 base64 ${COMPRESSIONTYPES}"
+CONVERSIONTYPES = "gz bz2 lzma xz lz4 lzo zip zst sum md5sum sha1sum sha224sum sha256sum sha384sum sha512sum bmap u-boot vmdk vdi qcow2 base64 ${COMPRESSIONTYPES}"
 CONVERSION_CMD_lzma = "lzma -k -f -7 ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"
 CONVERSION_CMD_gz = "gzip -f -9 -n -c --rsyncable ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.gz"
 CONVERSION_CMD_bz2 = "pbzip2 -f -k ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"
@@ -290,6 +292,7 @@ CONVERSION_CMD_xz = "xz -f -k -c ${XZ_COMPRESSION_LEVEL} ${XZ_DEFAULTS} --check=
 CONVERSION_CMD_lz4 = "lz4 -9 -z -l ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.lz4"
 CONVERSION_CMD_lzo = "lzop -9 ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"
 CONVERSION_CMD_zip = "zip ${ZIP_COMPRESSION_LEVEL} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.zip ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"
+CONVERSION_CMD_zst = "zstd -f -k -T0 -c ${ZSTD_COMPRESSION_LEVEL} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.zst"
 CONVERSION_CMD_sum = "sumtool -i ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} -o ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.sum ${JFFS2_SUM_EXTRA_ARGS}"
 CONVERSION_CMD_md5sum = "md5sum ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.md5sum"
 CONVERSION_CMD_sha1sum = "sha1sum ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.sha1sum"
@@ -310,6 +313,7 @@ CONVERSION_DEPENDS_xz = "xz-native"
 CONVERSION_DEPENDS_lz4 = "lz4-native"
 CONVERSION_DEPENDS_lzo = "lzop-native"
 CONVERSION_DEPENDS_zip = "zip-native"
+CONVERSION_DEPENDS_zst = "zstd-native"
 CONVERSION_DEPENDS_sum = "mtd-utils-native"
 CONVERSION_DEPENDS_bmap = "bmap-tools-native"
 CONVERSION_DEPENDS_u-boot = "u-boot-tools-native"

meta/classes/insane.bbclass

@@ -340,9 +340,11 @@ def package_qa_textrel(path, name, d, elf, messages):
     for line in phdrs.split("\n"):
         if textrel_re.match(line):
             sane = False
+            break
 
     if not sane:
-        package_qa_add_message(messages, "textrel", "ELF binary '%s' has relocations in .text" % path)
+        path = package_qa_clean_path(path, d, name)
+        package_qa_add_message(messages, "textrel", "%s: ELF binary %s has relocations in .text" % (name, path))
 
 QAPATHTEST[ldflags] = "package_qa_hash_style"
 def package_qa_hash_style(path, name, d, elf, messages):
@@ -891,9 +893,9 @@ def package_qa_check_src_uri(pn, d, messages):
     if "${PN}" in d.getVar("SRC_URI", False):
         package_qa_handle_error("src-uri-bad", "%s: SRC_URI uses PN not BPN" % pn, d)
 
-    pn = d.getVar("SRC_URI")
-    if re.search(r"github\.com/.+/.+/archive/.+", pn):
-        package_qa_handle_error("src-uri-bad", "%s: SRC_URI uses unstable GitHub archives" % pn, d)
+    for url in d.getVar("SRC_URI").split():
+        if re.search(r"github\.com/.+/.+/archive/.+", url):
+            package_qa_handle_error("src-uri-bad", "%s: SRC_URI uses unstable GitHub archives" % pn, d)
 
 
 # The PACKAGE FUNC to scan each package

meta/classes/kernel-devicetree.bbclass

@@ -71,23 +71,23 @@ do_deploy_append() {
 		dtb=`normalize_dtb "$dtbf"`
 		dtb_ext=${dtb##*.}
 		dtb_base_name=`basename $dtb .$dtb_ext`
-		install -d ${DEPLOYDIR}
-		install -m 0644 ${D}/${KERNEL_IMAGEDEST}/$dtb_base_name.$dtb_ext ${DEPLOYDIR}/$dtb_base_name-${KERNEL_DTB_NAME}.$dtb_ext
-		ln -sf $dtb_base_name-${KERNEL_DTB_NAME}.$dtb_ext ${DEPLOYDIR}/$dtb_base_name.$dtb_ext
-		ln -sf $dtb_base_name-${KERNEL_DTB_NAME}.$dtb_ext ${DEPLOYDIR}/$dtb_base_name-${KERNEL_DTB_LINK_NAME}.$dtb_ext
+		install -d $deployDir
+		install -m 0644 ${D}/${KERNEL_IMAGEDEST}/$dtb_base_name.$dtb_ext $deployDir/$dtb_base_name-${KERNEL_DTB_NAME}.$dtb_ext
+		ln -sf $dtb_base_name-${KERNEL_DTB_NAME}.$dtb_ext $deployDir/$dtb_base_name.$dtb_ext
+		ln -sf $dtb_base_name-${KERNEL_DTB_NAME}.$dtb_ext $deployDir/$dtb_base_name-${KERNEL_DTB_LINK_NAME}.$dtb_ext
 		for type in ${KERNEL_IMAGETYPE_FOR_MAKE}; do
 			if [ "$type" = "zImage" ] && [ "${KERNEL_DEVICETREE_BUNDLE}" = "1" ]; then
 				cat ${D}/${KERNEL_IMAGEDEST}/$type \
-					${DEPLOYDIR}/$dtb_base_name-${KERNEL_DTB_NAME}.$dtb_ext \
-					> ${DEPLOYDIR}/$type-$dtb_base_name-${KERNEL_DTB_NAME}.$dtb_ext.bin
+					$deployDir/$dtb_base_name-${KERNEL_DTB_NAME}.$dtb_ext \
+					> $deployDir/$type-$dtb_base_name-${KERNEL_DTB_NAME}.$dtb_ext.bin
 				ln -sf $type-$dtb_base_name-${KERNEL_DTB_NAME}.$dtb_ext.bin \
-					${DEPLOYDIR}/$type-$dtb_base_name-${KERNEL_DTB_LINK_NAME}.$dtb_ext.bin
+					$deployDir/$type-$dtb_base_name-${KERNEL_DTB_LINK_NAME}.$dtb_ext.bin
 				if [ -e "${KERNEL_OUTPUT_DIR}/${type}.initramfs" ]; then
 					cat ${KERNEL_OUTPUT_DIR}/${type}.initramfs \
-						${DEPLOYDIR}/$dtb_base_name-${KERNEL_DTB_NAME}.$dtb_ext \
-						>  ${DEPLOYDIR}/${type}-${INITRAMFS_NAME}-$dtb_base_name-${KERNEL_DTB_NAME}.$dtb_ext.bin
+						$deployDir/$dtb_base_name-${KERNEL_DTB_NAME}.$dtb_ext \
+						>  $deployDir/${type}-${INITRAMFS_NAME}-$dtb_base_name-${KERNEL_DTB_NAME}.$dtb_ext.bin
 					ln -sf ${type}-${INITRAMFS_NAME}-$dtb_base_name-${KERNEL_DTB_NAME}.$dtb_ext.bin \
-						${DEPLOYDIR}/${type}-${INITRAMFS_NAME}-$dtb_base_name-${KERNEL_DTB_LINK_NAME}.$dtb_ext.bin
+						$deployDir/${type}-${INITRAMFS_NAME}-$dtb_base_name-${KERNEL_DTB_LINK_NAME}.$dtb_ext.bin
 				fi
 			fi
 		done

meta/classes/kernel-fitimage.bbclass

@@ -53,6 +53,9 @@ UBOOT_MKIMAGE_DTCOPTS ??= ""
 # fitImage Hash Algo
 FIT_HASH_ALG ?= "sha256"
 
+# fitImage Signature Algo
+FIT_SIGN_ALG ?= "rsa2048"
+
 #
 # Emit the fitImage ITS header
 #
@@ -246,6 +249,7 @@ EOF
 fitimage_emit_section_config() {
 
 	conf_csum="${FIT_HASH_ALG}"
+	conf_sign_algo="${FIT_SIGN_ALG}"
 	if [ -n "${UBOOT_SIGN_ENABLE}" ] ; then
 		conf_sign_keyname="${UBOOT_SIGN_KEYNAME}"
 	fi
@@ -327,7 +331,7 @@ EOF
 
 		cat << EOF >> ${1}
                         signature@1 {
-                                algo = "${conf_csum},rsa2048";
+                                algo = "${conf_csum},${conf_sign_algo}";
                                 key-name-hint = "${conf_sign_keyname}";
 				${sign_line}
                         };
@@ -500,27 +504,27 @@ kernel_do_deploy_append() {
 	# Update deploy directory
 	if echo ${KERNEL_IMAGETYPES} | grep -wq "fitImage"; then
 		echo "Copying fit-image.its source file..."
-		install -m 0644 ${B}/fit-image.its ${DEPLOYDIR}/fitImage-its-${KERNEL_FIT_NAME}.its
-		ln -snf fitImage-its-${KERNEL_FIT_NAME}.its ${DEPLOYDIR}/fitImage-its-${KERNEL_FIT_LINK_NAME}
+		install -m 0644 ${B}/fit-image.its "$deployDir/fitImage-its-${KERNEL_FIT_NAME}.its"
+		ln -snf fitImage-its-${KERNEL_FIT_NAME}.its "$deployDir/fitImage-its-${KERNEL_FIT_LINK_NAME}"
 
 		echo "Copying linux.bin file..."
-		install -m 0644 ${B}/linux.bin ${DEPLOYDIR}/fitImage-linux.bin-${KERNEL_FIT_NAME}.bin
-		ln -snf fitImage-linux.bin-${KERNEL_FIT_NAME}.bin ${DEPLOYDIR}/fitImage-linux.bin-${KERNEL_FIT_LINK_NAME}
+		install -m 0644 ${B}/linux.bin $deployDir/fitImage-linux.bin-${KERNEL_FIT_NAME}.bin
+		ln -snf fitImage-linux.bin-${KERNEL_FIT_NAME}.bin "$deployDir/fitImage-linux.bin-${KERNEL_FIT_LINK_NAME}"
 
 		if [ -n "${INITRAMFS_IMAGE}" ]; then
 			echo "Copying fit-image-${INITRAMFS_IMAGE}.its source file..."
-			install -m 0644 ${B}/fit-image-${INITRAMFS_IMAGE}.its ${DEPLOYDIR}/fitImage-its-${INITRAMFS_IMAGE_NAME}-${KERNEL_FIT_NAME}.its
-			ln -snf fitImage-its-${INITRAMFS_IMAGE_NAME}-${KERNEL_FIT_NAME}.its ${DEPLOYDIR}/fitImage-its-${INITRAMFS_IMAGE_NAME}-${KERNEL_FIT_LINK_NAME}
+			install -m 0644 ${B}/fit-image-${INITRAMFS_IMAGE}.its "$deployDir/fitImage-its-${INITRAMFS_IMAGE_NAME}-${KERNEL_FIT_NAME}.its"
+			ln -snf fitImage-its-${INITRAMFS_IMAGE_NAME}-${KERNEL_FIT_NAME}.its "$deployDir/fitImage-its-${INITRAMFS_IMAGE_NAME}-${KERNEL_FIT_LINK_NAME}"
 
 			echo "Copying fitImage-${INITRAMFS_IMAGE} file..."
-			install -m 0644 ${B}/arch/${ARCH}/boot/fitImage-${INITRAMFS_IMAGE} ${DEPLOYDIR}/fitImage-${INITRAMFS_IMAGE_NAME}-${KERNEL_FIT_NAME}.bin
-			ln -snf fitImage-${INITRAMFS_IMAGE_NAME}-${KERNEL_FIT_NAME}.bin ${DEPLOYDIR}/fitImage-${INITRAMFS_IMAGE_NAME}-${KERNEL_FIT_LINK_NAME}
+			install -m 0644 ${B}/arch/${ARCH}/boot/fitImage-${INITRAMFS_IMAGE} "$deployDir/fitImage-${INITRAMFS_IMAGE_NAME}-${KERNEL_FIT_NAME}.bin"
+			ln -snf fitImage-${INITRAMFS_IMAGE_NAME}-${KERNEL_FIT_NAME}.bin "$deployDir/fitImage-${INITRAMFS_IMAGE_NAME}-${KERNEL_FIT_LINK_NAME}"
 		fi
 		if [ "${UBOOT_SIGN_ENABLE}" = "1" -a -n "${UBOOT_DTB_BINARY}" ] ; then
 			# UBOOT_DTB_IMAGE is a realfile, but we can't use
 			# ${UBOOT_DTB_IMAGE} since it contains ${PV} which is aimed
 			# for u-boot, but we are in kernel env now.
-			install -m 0644 ${B}/u-boot-${MACHINE}*.dtb ${DEPLOYDIR}/
+			install -m 0644 ${B}/u-boot-${MACHINE}*.dtb "$deployDir/"
 		fi
 	fi
 }

meta/classes/libc-package.bbclass

@@ -346,14 +346,13 @@ python package_do_split_gconvs () {
 
     if use_bin == "compile":
         makefile = oe.path.join(d.getVar("WORKDIR"), "locale-tree", "Makefile")
-        m = open(makefile, "w")
-        m.write("all: %s\n\n" % " ".join(commands.keys()))
-        total = len(commands)
-        for i, cmd in enumerate(commands):
-            m.write(cmd + ":\n")
-            m.write("\t@echo 'Progress %d/%d'\n" % (i, total))
-            m.write("\t" + commands[cmd] + "\n\n")
-        m.close()
+        with open(makefile, "w") as m:
+            m.write("all: %s\n\n" % " ".join(commands.keys()))
+            total = len(commands)
+            for i, (maketarget, makerecipe) in enumerate(commands.items()):
+                m.write(maketarget + ":\n")
+                m.write("\t@echo 'Progress %d/%d'\n" % (i, total))
+                m.write("\t" + makerecipe + "\n\n")
         d.setVar("EXTRA_OEMAKE", "-C %s ${PARALLEL_MAKE}" % (os.path.dirname(makefile)))
         d.setVarFlag("oe_runmake", "progress", "outof:Progress\s(\d+)/(\d+)")
         bb.note("Executing binary locale generation makefile")

meta/classes/license.bbclass

@@ -287,17 +287,7 @@ def incompatible_license_contains(license, truevalue, falsevalue, d):
     bad_licenses = expand_wildcard_licenses(d, bad_licenses)
     return truevalue if license in bad_licenses else falsevalue
 
-def incompatible_license(d, dont_want_licenses, package=None):
-    """
-    This function checks if a recipe has only incompatible licenses. It also
-    take into consideration 'or' operand.  dont_want_licenses should be passed
-    as canonical (SPDX) names.
-    """
-    import oe.license
-    license = d.getVar("LICENSE_%s" % package) if package else None
-    if not license:
-        license = d.getVar('LICENSE')
-
+def incompatible_pkg_license(d, dont_want_licenses, license):
     # Handles an "or" or two license sets provided by
     # flattened_licenses(), pick one that works if possible.
     def choose_lic_set(a, b):
@@ -311,6 +301,19 @@ def incompatible_license(d, dont_want_licenses, package=None):
     return any(not oe.license.license_ok(canonical_license(d, l), \
                dont_want_licenses) for l in licenses)
 
+def incompatible_license(d, dont_want_licenses, package=None):
+    """
+    This function checks if a recipe has only incompatible licenses. It also
+    take into consideration 'or' operand.  dont_want_licenses should be passed
+    as canonical (SPDX) names.
+    """
+    import oe.license
+    license = d.getVar("LICENSE_%s" % package) if package else None
+    if not license:
+        license = d.getVar('LICENSE')
+
+    return incompatible_pkg_license(d, dont_want_licenses, license)
+
 def check_license_flags(d):
     """
     This function checks if a recipe has any LICENSE_FLAGS that

meta/classes/license_image.bbclass

@@ -43,10 +43,16 @@ def write_license_files(d, license_manifest, pkg_dic, rootfs=True):
     bad_licenses = [canonical_license(d, l) for l in bad_licenses]
     bad_licenses = expand_wildcard_licenses(d, bad_licenses)
 
+    whitelist = []
+    for lic in bad_licenses:
+        whitelist.extend((d.getVar("WHITELIST_" + lic) or "").split())
+
     with open(license_manifest, "w") as license_file:
         for pkg in sorted(pkg_dic):
-            if bad_licenses:
+            if bad_licenses and pkg not in whitelist:
                 try:
+                    if incompatible_pkg_license(d, bad_licenses, pkg_dic[pkg]["LICENSE"]):
+                        bb.fatal("Package %s has an incompatible license %s and cannot be installed into the image." %(pkg, pkg_dic[pkg]["LICENSE"]))
                     (pkg_dic[pkg]["LICENSE"], pkg_dic[pkg]["LICENSES"]) = \
                         oe.license.manifest_licenses(pkg_dic[pkg]["LICENSE"],
                         bad_licenses, canonical_license, d)
@@ -56,6 +62,8 @@ def write_license_files(d, license_manifest, pkg_dic, rootfs=True):
                 pkg_dic[pkg]["LICENSES"] = re.sub(r'[|&()*]', ' ', pkg_dic[pkg]["LICENSE"])
                 pkg_dic[pkg]["LICENSES"] = re.sub(r'  *', ' ', pkg_dic[pkg]["LICENSES"])
                 pkg_dic[pkg]["LICENSES"] = pkg_dic[pkg]["LICENSES"].split()
+                if pkg in whitelist:
+                    bb.warn("Including %s with an incompatible license %s into the image, because it has been whitelisted." %(pkg, pkg_dic[pkg]["LICENSE"]))
 
             if not "IMAGE_MANIFEST" in pkg_dic[pkg]:
                 # Rootfs manifest

meta/classes/meson.bbclass

@@ -24,7 +24,8 @@ MESONOPTS = " --prefix ${prefix} \
               --infodir ${@noprefix('infodir', d)} \
               --sysconfdir ${sysconfdir} \
               --localstatedir ${localstatedir} \
-              --sharedstatedir ${sharedstatedir} "
+              --sharedstatedir ${sharedstatedir} \
+              --wrap-mode nodownload"
 
 EXTRA_OEMESON_append = " ${PACKAGECONFIG_CONFARGS}"
 

meta/classes/multilib.bbclass

@@ -184,11 +184,12 @@ python do_package_qa_multilib() {
         for i in values:
             if i.startswith('virtual/'):
                 i = i[len('virtual/'):]
-            if (not i.startswith('kernel-module')) and (not i.startswith(mlprefix)) and \
-                (not 'cross-canadian' in i) and (not i.startswith("nativesdk-")) and \
-                (not i.startswith("rtld")) and (not i.startswith('kernel-vmlinux')) \
-                and (not i.startswith("kernel-image")) and (not i.startswith("/")):
+
+            if (not (i.startswith(mlprefix) or i.startswith("kernel-") \
+                    or ('cross-canadian' in i) or i.startswith("nativesdk-") \
+                    or i.startswith("rtld") or i.startswith("/"))):
                 candidates.append(i)
+
         if len(candidates) > 0:
             msg = "%s package %s - suspicious values '%s' in %s" \
                    % (d.getVar('PN'), pkg, ' '.join(candidates), var)

meta/classes/native.bbclass

@@ -186,6 +186,7 @@ python do_addto_recipe_sysroot () {
     bb.build.exec_func("extend_recipe_sysroot", d)
 }
 addtask addto_recipe_sysroot after do_populate_sysroot
+do_addto_recipe_sysroot[deptask] = "do_populate_sysroot"
 
 inherit nopackages
 

meta/classes/package.bbclass

@@ -826,8 +826,9 @@ python fixup_perms () {
 
     # Now we actually load from the configuration files
     for conf in get_fs_perms_list(d).split():
-        if os.path.exists(conf):
-            f = open(conf)
+        if not os.path.exists(conf):
+            continue
+        with open(conf) as f:
             for line in f:
                 if line.startswith('#'):
                     continue
@@ -848,7 +849,6 @@ python fixup_perms () {
                         fs_perms_table[entry.path] = entry
                         if entry.path in fs_link_table:
                             fs_link_table.pop(entry.path)
-            f.close()
 
     # Debug -- list out in-memory table
     #for dir in fs_perms_table:
@@ -1424,10 +1424,9 @@ fi
     pkgdest = d.getVar('PKGDEST')
     pkgdatadir = d.getVar('PKGDESTWORK')
 
-    data_file = pkgdatadir + d.expand("/${PN}" )
-    f = open(data_file, 'w')
-    f.write("PACKAGES: %s\n" % packages)
-    f.close()
+    data_file = pkgdatadir + d.expand("/${PN}")
+    with open(data_file, 'w') as fd:
+        fd.write("PACKAGES: %s\n" % packages)
 
     pn = d.getVar('PN')
     global_variants = (d.getVar('MULTILIB_GLOBAL_VARIANTS') or "").split()
@@ -1732,8 +1731,6 @@ python package_do_shlibs() {
     else:
         snap_symlinks = False
 
-    use_ldconfig = bb.utils.contains('DISTRO_FEATURES', 'ldconfig', True, False, d)
-
     needed = {}
 
     shlib_provider = oe.package.read_shlib_providers(d)
@@ -1778,22 +1775,21 @@ python package_do_shlibs() {
             bb.note("Renaming %s to %s" % (old, new))
             os.rename(old, new)
             pkgfiles[pkg].remove(old)
-	    
+
         shlibs_file = os.path.join(shlibswork_dir, pkg + ".list")
         if len(sonames):
-            fd = open(shlibs_file, 'w')
-            for s in sonames:
-                if s[0] in shlib_provider and s[1] in shlib_provider[s[0]]:
-                    (old_pkg, old_pkgver) = shlib_provider[s[0]][s[1]]
-                    if old_pkg != pkg:
-                        bb.warn('%s-%s was registered as shlib provider for %s, changing it to %s-%s because it was built later' % (old_pkg, old_pkgver, s[0], pkg, pkgver))
-                bb.debug(1, 'registering %s-%s as shlib provider for %s' % (pkg, pkgver, s[0]))
-                fd.write(s[0] + ':' + s[1] + ':' + s[2] + '\n')
-                if s[0] not in shlib_provider:
-                    shlib_provider[s[0]] = {}
-                shlib_provider[s[0]][s[1]] = (pkg, pkgver)
-            fd.close()
-        if needs_ldconfig and use_ldconfig:
+            with open(shlibs_file, 'w') as fd:
+                for s in sonames:
+                    if s[0] in shlib_provider and s[1] in shlib_provider[s[0]]:
+                        (old_pkg, old_pkgver) = shlib_provider[s[0]][s[1]]
+                        if old_pkg != pkg:
+                            bb.warn('%s-%s was registered as shlib provider for %s, changing it to %s-%s because it was built later' % (old_pkg, old_pkgver, s[0], pkg, pkgver))
+                    bb.debug(1, 'registering %s-%s as shlib provider for %s' % (pkg, pkgver, s[0]))
+                    fd.write(s[0] + ':' + s[1] + ':' + s[2] + '\n')
+                    if s[0] not in shlib_provider:
+                        shlib_provider[s[0]] = {}
+                    shlib_provider[s[0]][s[1]] = (pkg, pkgver)
+        if needs_ldconfig:
             bb.debug(1, 'adding ldconfig call to postinst for %s' % pkg)
             postinst = d.getVar('pkg_postinst_%s' % pkg)
             if not postinst:
@@ -1864,11 +1860,10 @@ python package_do_shlibs() {
         deps_file = os.path.join(pkgdest, pkg + ".shlibdeps")
         if os.path.exists(deps_file):
             os.remove(deps_file)
-        if len(deps):
-            fd = open(deps_file, 'w')
-            for dep in sorted(deps):
-                fd.write(dep + '\n')
-            fd.close()
+        if deps:
+            with open(deps_file, 'w') as fd:
+                for dep in sorted(deps):
+                    fd.write(dep + '\n')
 }
 
 python package_do_pkgconfig () {
@@ -1898,9 +1893,8 @@ python package_do_pkgconfig () {
                     pkgconfig_provided[pkg].append(name)
                     if not os.access(file, os.R_OK):
                         continue
-                    f = open(file, 'r')
-                    lines = f.readlines()
-                    f.close()
+                    with open(file, 'r') as f:
+                        lines = f.readlines()
                     for l in lines:
                         m = var_re.match(l)
                         if m:
@@ -1918,10 +1912,9 @@ python package_do_pkgconfig () {
     for pkg in packages.split():
         pkgs_file = os.path.join(shlibswork_dir, pkg + ".pclist")
         if pkgconfig_provided[pkg] != []:
-            f = open(pkgs_file, 'w')
-            for p in pkgconfig_provided[pkg]:
-                f.write('%s\n' % p)
-            f.close()
+            with open(pkgs_file, 'w') as f:
+                for p in pkgconfig_provided[pkg]:
+                    f.write('%s\n' % p)
 
     # Go from least to most specific since the last one found wins
     for dir in reversed(shlibs_dirs):
@@ -1931,9 +1924,8 @@ python package_do_pkgconfig () {
             m = re.match(r'^(.*)\.pclist$', file)
             if m:
                 pkg = m.group(1)
-                fd = open(os.path.join(dir, file))
-                lines = fd.readlines()
-                fd.close()
+                with open(os.path.join(dir, file)) as fd:
+                    lines = fd.readlines()
                 pkgconfig_provided[pkg] = []
                 for l in lines:
                     pkgconfig_provided[pkg].append(l.rstrip())
@@ -1951,10 +1943,9 @@ python package_do_pkgconfig () {
                 bb.note("couldn't find pkgconfig module '%s' in any package" % n)
         deps_file = os.path.join(pkgdest, pkg + ".pcdeps")
         if len(deps):
-            fd = open(deps_file, 'w')
-            for dep in deps:
-                fd.write(dep + '\n')
-            fd.close()
+            with open(deps_file, 'w') as fd:
+                for dep in deps:
+                    fd.write(dep + '\n')
 }
 
 def read_libdep_files(d):
@@ -1965,9 +1956,8 @@ def read_libdep_files(d):
         for extension in ".shlibdeps", ".pcdeps", ".clilibdeps":
             depsfile = d.expand("${PKGDEST}/" + pkg + extension)
             if os.access(depsfile, os.R_OK):
-                fd = open(depsfile)
-                lines = fd.readlines()
-                fd.close()
+                with open(depsfile) as fd:
+                    lines = fd.readlines()
                 for l in lines:
                     l.rstrip()
                     deps = bb.utils.explode_dep_versions2(l)

meta/classes/package_ipk.bbclass

@@ -154,7 +154,6 @@ def ipk_write_pkg(pkg, d):
                     ctrlfile.write('%s\n' % textwrap.fill(description, width=74, initial_indent=' ', subsequent_indent=' '))
             else:
                 ctrlfile.write(c % tuple(pullData(fs, localdata)))
-        # more fields
 
         custom_fields_chunk = get_package_additional_metadata("ipk", localdata)
         if custom_fields_chunk is not None:

meta/classes/populate_sdk_ext.bbclass

@@ -21,6 +21,7 @@ SDK_EXT_TYPE ?= "full"
 SDK_INCLUDE_PKGDATA ?= "0"
 SDK_INCLUDE_TOOLCHAIN ?= "${@'1' if d.getVar('SDK_EXT_TYPE') == 'full' else '0'}"
 SDK_INCLUDE_NATIVESDK ?= "0"
+SDK_INCLUDE_BUILDTOOLS ?= '1'
 
 SDK_RECRDEP_TASKS ?= ""
 
@@ -94,6 +95,7 @@ python write_target_sdk_ext_manifest () {
     real_target_multimach = d.getVar('REAL_MULTIMACH_TARGET_SYS')
 
     pkgs = {}
+    os.makedirs(os.path.dirname(d.getVar('SDK_EXT_TARGET_MANIFEST')), exist_ok=True)
     with open(d.getVar('SDK_EXT_TARGET_MANIFEST'), 'w') as f:
         for fn in extra_info['filesizes']:
             info = fn.split(':')
@@ -535,8 +537,12 @@ def get_sdk_required_utilities(buildtools_fn, d):
     sanity_required_utilities = (d.getVar('SANITY_REQUIRED_UTILITIES') or '').split()
     sanity_required_utilities.append(d.expand('${BUILD_PREFIX}gcc'))
     sanity_required_utilities.append(d.expand('${BUILD_PREFIX}g++'))
-    buildtools_installer = os.path.join(d.getVar('SDK_DEPLOY'), buildtools_fn)
-    filelist, _ = bb.process.run('%s -l' % buildtools_installer)
+    if buildtools_fn:
+        buildtools_installer = os.path.join(d.getVar('SDK_DEPLOY'), buildtools_fn)
+        filelist, _ = bb.process.run('%s -l' % buildtools_installer)
+    else:
+        buildtools_installer = None
+        filelist = ""
     localdata = bb.data.createCopy(d)
     localdata.setVar('SDKPATH', '.')
     sdkpathnative = localdata.getVar('SDKPATHNATIVE')
@@ -579,7 +585,9 @@ install_tools() {
 	touch ${SDK_OUTPUT}/${SDKPATH}/.devtoolbase
 
 	# find latest buildtools-tarball and install it
-	install ${SDK_DEPLOY}/${SDK_BUILDTOOLS_INSTALLER} ${SDK_OUTPUT}/${SDKPATH}
+	if [ -n "${SDK_BUILDTOOLS_INSTALLER}" ]; then
+		install ${SDK_DEPLOY}/${SDK_BUILDTOOLS_INSTALLER} ${SDK_OUTPUT}/${SDKPATH}
+	fi
 
 	install -m 0644 ${COREBASE}/meta/files/ext-sdk-prepare.py ${SDK_OUTPUT}/${SDKPATH}
 }
@@ -629,16 +637,18 @@ sdk_ext_postinst() {
 	printf "\nExtracting buildtools...\n"
 	cd $target_sdk_dir
 	env_setup_script="$target_sdk_dir/environment-setup-${REAL_MULTIMACH_TARGET_SYS}"
-	printf "buildtools\ny" | ./${SDK_BUILDTOOLS_INSTALLER} > buildtools.log || { printf 'ERROR: buildtools installation failed:\n' ; cat buildtools.log ; echo "printf 'ERROR: this SDK was not fully installed and needs reinstalling\n'" >> $env_setup_script ; exit 1 ; }
+        if [ -n "${SDK_BUILDTOOLS_INSTALLER}" ]; then
+		printf "buildtools\ny" | ./${SDK_BUILDTOOLS_INSTALLER} > buildtools.log || { printf 'ERROR: buildtools installation failed:\n' ; cat buildtools.log ; echo "printf 'ERROR: this SDK was not fully installed and needs reinstalling\n'" >> $env_setup_script ; exit 1 ; }
 
-	# Delete the buildtools tar file since it won't be used again
-	rm -f ./${SDK_BUILDTOOLS_INSTALLER}
-	# We don't need the log either since it succeeded
-	rm -f buildtools.log
+		# Delete the buildtools tar file since it won't be used again
+		rm -f ./${SDK_BUILDTOOLS_INSTALLER}
+		# We don't need the log either since it succeeded
+		rm -f buildtools.log
 
-	# Make sure when the user sets up the environment, they also get
-	# the buildtools-tarball tools in their path.
-	echo ". $target_sdk_dir/buildtools/environment-setup*" >> $env_setup_script
+		# Make sure when the user sets up the environment, they also get
+		# the buildtools-tarball tools in their path.
+		echo ". $target_sdk_dir/buildtools/environment-setup*" >> $env_setup_script
+	fi
 
 	# Allow bitbake environment setup to be ran as part of this sdk.
 	echo "export OE_SKIP_SDK_CHECK=1" >> $env_setup_script
@@ -654,7 +664,7 @@ sdk_ext_postinst() {
 	# Warn if trying to use external bitbake and the ext SDK together
 	echo "(which bitbake > /dev/null 2>&1 && echo 'WARNING: attempting to use the extensible SDK in an environment set up to run bitbake - this may lead to unexpected results. Please source this script in a new shell session instead.') || true" >> $env_setup_script
 
-	if [ "$prepare_buildsystem" != "no" ]; then
+	if [ "$prepare_buildsystem" != "no" -a -n "${SDK_BUILDTOOLS_INSTALLER}" ]; then
 		printf "Preparing build system...\n"
 		# dash which is /bin/sh on Ubuntu will not preserve the
 		# current working directory when first ran, nor will it set $1 when
@@ -680,7 +690,10 @@ fakeroot python do_populate_sdk_ext() {
         bb.fatal('The extensible SDK can currently only be built for the same architecture as the machine being built on - SDK_ARCH is set to %s (likely via setting SDKMACHINE) which is different from the architecture of the build machine (%s). Unable to continue.' % (d.getVar('SDK_ARCH'), d.getVar('BUILD_ARCH')))
 
     d.setVar('SDK_INSTALL_TARGETS', get_sdk_install_targets(d))
-    buildtools_fn = get_current_buildtools(d)
+    if d.getVar('SDK_INCLUDE_BUILDTOOLS') == '1':
+        buildtools_fn = get_current_buildtools(d)
+    else:
+        buildtools_fn = None
     d.setVar('SDK_REQUIRED_UTILITIES', get_sdk_required_utilities(buildtools_fn, d))
     d.setVar('SDK_BUILDTOOLS_INSTALLER', buildtools_fn)
     d.setVar('SDKDEPLOYDIR', '${SDKEXTDEPLOYDIR}')
@@ -731,7 +744,7 @@ def get_sdk_ext_rdepends(d):
 do_populate_sdk_ext[dirs] = "${@d.getVarFlag('do_populate_sdk', 'dirs', False)}"
 
 do_populate_sdk_ext[depends] = "${@d.getVarFlag('do_populate_sdk', 'depends', False)} \
-                                buildtools-tarball:do_populate_sdk \
+                                ${@'buildtools-tarball:do_populate_sdk' if d.getVar('SDK_INCLUDE_BUILDTOOLS') == '1' else ''} \
                                 ${@'meta-world-pkgdata:do_collect_packagedata' if d.getVar('SDK_INCLUDE_PKGDATA') == '1' else ''} \
                                 ${@'meta-extsdk-toolchain:do_locked_sigs' if d.getVar('SDK_INCLUDE_TOOLCHAIN') == '1' else ''}"
 

meta/classes/python3native.bbclass

@@ -1,6 +1,8 @@
 inherit python3-dir
 
 PYTHON="${STAGING_BINDIR_NATIVE}/python3-native/python3"
+# PYTHON_EXECUTABLE is used by cmake
+PYTHON_EXECUTABLE="${PYTHON}"
 EXTRANATIVEPATH += "python3-native"
 DEPENDS_append = " python3-native "
 

meta/classes/report-error.bbclass

@@ -78,19 +78,15 @@ python errorreport_handler () {
             taskdata['task'] = task
             if log:
                 try:
-                    logFile = codecs.open(log, 'r', 'utf-8')
-                    logdata = logFile.read()
-
+                    with codecs.open(log, encoding='utf-8') as logFile:
+                        logdata = logFile.read()
                     # Replace host-specific paths so the logs are cleaner
                     for d in ("TOPDIR", "TMPDIR"):
                         s = e.data.getVar(d)
                         if s:
                             logdata = logdata.replace(s, d)
-
-                    logFile.close()
                 except:
                     logdata = "Unable to read log file"
-
             else:
                 logdata = "No Log"
 

meta/classes/rm_work.bbclass

@@ -47,30 +47,26 @@ do_rm_work () {
     cd `dirname ${STAMP}`
     for i in `basename ${STAMP}`*
     do
-        # By default we'll delete the stamp, unless $i is changed by the inner loop
-        # (i=dummy does this)
-
         case $i in
         *sigdata*|*sigbasedata*)
             # Save/skip anything that looks like a signature data file.
-            i=dummy
             ;;
-        *do_image_complete_setscene*)
-            # Ensure we don't 'stack' setscene extensions to this stamp with the section below
-            i=dummy
+        *do_image_complete_setscene*|*do_image_qa_setscene*)
+            # Ensure we don't 'stack' setscene extensions to these stamps with the sections below
             ;;
         *do_image_complete*)
             # Promote do_image_complete stamps to setscene versions (ahead of *do_image* below)
             mv $i `echo $i | sed -e "s#do_image_complete#do_image_complete_setscene#"`
-            i=dummy
+            ;;
+        *do_image_qa*)
+            # Promote do_image_qa stamps to setscene versions (ahead of *do_image* below)
+            mv $i `echo $i | sed -e "s#do_image_qa#do_image_qa_setscene#"`
             ;;
         *do_package_write*|*do_rootfs*|*do_image*|*do_bootimg*|*do_write_qemuboot_conf*|*do_build*)
-            i=dummy
             ;;
         *do_addto_recipe_sysroot*)
             # Preserve recipe-sysroot-native if do_addto_recipe_sysroot has been used
             excludes="$excludes recipe-sysroot-native"
-            i=dummy
             ;;
         *do_package|*do_package.*|*do_package_setscene.*)
             # We remove do_package entirely, including any
@@ -78,30 +74,24 @@ do_rm_work () {
             # such as 'packages' and 'packages-split' and these can be large. No end
             # of chain tasks depend directly on do_package anymore.
             rm -f $i;
-            i=dummy
             ;;
         *_setscene*)
             # Skip stamps which are already setscene versions
-            i=dummy
             ;;
+        *)
+            # For everything else: if suitable, promote the stamp to a setscene
+            # version, otherwise remove it
+            for j in ${SSTATETASKS} do_shared_workdir
+            do
+                case $i in
+                *$j|*$j.*)
+                    mv $i `echo $i | sed -e "s#${j}#${j}_setscene#"`
+                    break
+                    ;;
+                esac
+            done
+            rm -f $i
         esac
-
-        for j in ${SSTATETASKS} do_shared_workdir
-        do
-            case $i in
-            dummy)
-                break
-                ;;
-            *$j|*$j.*)
-                # Promote the stamp to a setscene version
-                mv $i `echo $i | sed -e "s#${j}#${j}_setscene#"`
-                i=dummy
-                break
-                ;;
-            esac
-        done
-
-        rm -f $i
     done
 
     cd ${WORKDIR}

meta/classes/sanity.bbclass

@@ -523,6 +523,7 @@ def check_wsl(d):
 
 # Tar version 1.24 and onwards handle overwriting symlinks correctly
 # but earlier versions do not; this needs to work properly for sstate
+# Version 1.28 is needed so opkg-build works correctly when reproducibile builds are enabled
 def check_tar_version(sanity_data):
     from distutils.version import LooseVersion
     import subprocess
@@ -532,7 +533,9 @@ def check_tar_version(sanity_data):
         return "Unable to execute tar --version, exit code %d\n%s\n" % (e.returncode, e.output)
     version = result.split()[3]
     if LooseVersion(version) < LooseVersion("1.24"):
-        return "Your version of tar is older than 1.24 and has bugs which will break builds. Please install a newer version of tar.\n"
+        return "Your version of tar is older than 1.24 and has bugs which will break builds. Please install a newer version of tar (1.28+).\n"
+    if LooseVersion(version) < LooseVersion("1.28"):
+        return "Your version of tar is older than 1.28 and does not have the support needed to enable reproducible builds. Please install a newer version of tar (you could use the projects buildtools-tarball from our last release).\n"
     return None
 
 # We use git parameters and functionality only found in 1.7.8 or later
@@ -622,13 +625,14 @@ def check_sanity_version_change(status, d):
     # In other words, these tests run once in a given build directory and then 
     # never again until the sanity version or host distrubution id/version changes.
 
-    # Check the python install is complete. glib-2.0-natives requries
-    # xml.parsers.expat
+    # Check the python install is complete. Examples that are often removed in
+    # minimal installations: glib-2.0-natives requries # xml.parsers.expat and icu
+    # requires distutils.sysconfig.
     try:
         import xml.parsers.expat
-    except ImportError:
-        status.addresult('Your python is not a full install. Please install the module xml.parsers.expat (python-xml on openSUSE and SUSE Linux).\n')
-    import stat
+        import distutils.sysconfig
+    except ImportError as e:
+        status.addresult('Your Python 3 is not a full install. Please install the module %s (see the Getting Started guide for further information).\n' % e.name)
 
     status.addresult(check_make_version(d))
     status.addresult(check_patch_version(d))
@@ -664,6 +668,7 @@ def check_sanity_version_change(status, d):
         status.addresult('Please use ASSUME_PROVIDED +=, not ASSUME_PROVIDED = in your local.conf\n')
 
     # Check that TMPDIR isn't on a filesystem with limited filename length (eg. eCryptFS)
+    import stat
     tmpdir = d.getVar('TMPDIR')
     status.addresult(check_create_long_filename(tmpdir, "TMPDIR"))
     tmpdirmode = os.stat(tmpdir).st_mode
@@ -798,6 +803,11 @@ def check_sanity_everybuild(status, d):
         elif d.getVar('SDK_ARCH', False) == "${BUILD_ARCH}":
             status.addresult('SDKMACHINE is set, but SDK_ARCH has not been changed as a result - SDKMACHINE may have been set too late (e.g. in the distro configuration)\n')
 
+    # If SDK_VENDOR looks like "-my-sdk" then the triples are badly formed so fail early
+    sdkvendor = d.getVar("SDK_VENDOR")
+    if not (sdkvendor.startswith("-") and sdkvendor.count("-") == 1):
+        status.addresult("SDK_VENDOR should be of the form '-foosdk' with a single dash\n")
+
     check_supported_distro(d)
 
     omask = os.umask(0o022)

meta/classes/sstate.bbclass

@@ -318,7 +318,8 @@ def sstate_installpkg(ss, d):
 
     sstateinst = d.expand("${WORKDIR}/sstate-install-%s/" % ss['task'])
     sstatefetch = d.getVar('SSTATE_PKGNAME') + '_' + ss['task'] + ".tgz"
-    sstatepkg = d.getVar('SSTATE_PKG') + '_' + ss['task'] + ".tgz"
+    d.appendVar('SSTATE_PKG', '_'+ ss['task'] + ".tgz")
+    sstatepkg = d.getVar('SSTATE_PKG')
 
     if not os.path.exists(sstatepkg):
         pstaging_fetch(sstatefetch, d)
@@ -330,7 +331,6 @@ def sstate_installpkg(ss, d):
     sstate_clean(ss, d)
 
     d.setVar('SSTATE_INSTDIR', sstateinst)
-    d.setVar('SSTATE_PKG', sstatepkg)
 
     if bb.utils.to_boolean(d.getVar("SSTATE_VERIFY_SIG"), False):
         signer = get_signer(d, 'local')
@@ -612,10 +612,9 @@ def sstate_package(ss, d):
     tmpdir = d.getVar('TMPDIR')
 
     sstatebuild = d.expand("${WORKDIR}/sstate-build-%s/" % ss['task'])
-    sstatepkg = d.getVar('SSTATE_PKG') + '_'+ ss['task'] + ".tgz"
+    d.appendVar('SSTATE_PKG', '_'+ ss['task'] + ".tgz")
     bb.utils.remove(sstatebuild, recurse=True)
     bb.utils.mkdirhier(sstatebuild)
-    bb.utils.mkdirhier(os.path.dirname(sstatepkg))
     for state in ss['dirs']:
         if not os.path.exists(state[1]):
             continue
@@ -648,7 +647,6 @@ def sstate_package(ss, d):
         os.rename(plain, pdir)
 
     d.setVar('SSTATE_BUILDDIR', sstatebuild)
-    d.setVar('SSTATE_PKG', sstatepkg)
     d.setVar('SSTATE_INSTDIR', sstatebuild)
 
     if d.getVar('SSTATE_SKIP_CREATION') == '1':
@@ -664,7 +662,8 @@ def sstate_package(ss, d):
         # All hooks should run in SSTATE_BUILDDIR.
         bb.build.exec_func(f, d, (sstatebuild,))
 
-    bb.siggen.dump_this_task(sstatepkg + ".siginfo", d)
+    # SSTATE_PKG may have been changed by sstate_report_unihash
+    bb.siggen.dump_this_task(d.getVar('SSTATE_PKG') + ".siginfo", d)
 
     return
 
@@ -748,13 +747,14 @@ sstate_task_postfunc[dirs] = "${WORKDIR}"
 # set as SSTATE_BUILDDIR. Will be run from within SSTATE_BUILDDIR.
 #
 sstate_create_package () {
-	TFILE=`mktemp ${SSTATE_PKG}.XXXXXXXX`
-
-	# Exit earlu if it already exists
+	# Exit early if it already exists
 	if [ -e ${SSTATE_PKG} ]; then
 		return
 	fi
 
+        mkdir -p `dirname ${SSTATE_PKG}`
+	TFILE=`mktemp ${SSTATE_PKG}.XXXXXXXX`
+
         # Use pigz if available
         OPT="-czS"
         if [ -x "$(command -v pigz)" ]; then
@@ -818,7 +818,7 @@ sstate_unpack_package () {
 
 BB_HASHCHECK_FUNCTION = "sstate_checkhashes"
 
-def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, **kwargs):
+def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, **kwargs):
     found = set()
     missed = set()
     extension = ".tgz"
@@ -951,16 +951,17 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, **kwargs):
             evdata['found'].append((bb.runqueue.fn_from_tid(tid), bb.runqueue.taskname_from_tid(tid), gethash(tid), sstatefile ) )
         bb.event.fire(bb.event.MetadataEvent("MissedSstate", evdata), d)
 
-    # Print some summary statistics about the current task completion and how much sstate
-    # reuse there was. Avoid divide by zero errors.
-    total = len(sq_data['hash'])
-    complete = 0
-    if currentcount:
-        complete = (len(found) + currentcount) / (total + currentcount) * 100
-    match = 0
-    if total:
-        match = len(found) / total * 100
-    bb.plain("Sstate summary: Wanted %d Found %d Missed %d Current %d (%d%% match, %d%% complete)" % (total, len(found), len(missed), currentcount, match, complete))
+    if summary:
+        # Print some summary statistics about the current task completion and how much sstate
+        # reuse there was. Avoid divide by zero errors.
+        total = len(sq_data['hash'])
+        complete = 0
+        if currentcount:
+            complete = (len(found) + currentcount) / (total + currentcount) * 100
+        match = 0
+        if total:
+            match = len(found) / total * 100
+        bb.plain("Sstate summary: Wanted %d Found %d Missed %d Current %d (%d%% match, %d%% complete)" % (total, len(found), len(missed), currentcount, match, complete))
 
     if hasattr(bb.parse.siggen, "checkhashes"):
         bb.parse.siggen.checkhashes(sq_data, missed, found, d)

meta/classes/staging.bbclass

@@ -75,8 +75,8 @@ python sysroot_strip () {
 
     dstdir = d.getVar('SYSROOT_DESTDIR')
     pn = d.getVar('PN')
-    libdir = os.path.abspath(dstdir + os.sep + d.getVar("libdir"))
-    base_libdir = os.path.abspath(dstdir + os.sep + d.getVar("base_libdir"))
+    libdir = d.getVar("libdir")
+    base_libdir = d.getVar("base_libdir")
     qa_already_stripped = 'already-stripped' in (d.getVar('INSANE_SKIP_' + pn) or "").split()
     strip_cmd = d.getVar("STRIP")
 
@@ -449,6 +449,28 @@ python extend_recipe_sysroot() {
     msg_exists = []
     msg_adding = []
 
+    # Handle all removals first since files may move between recipes
+    for dep in configuredeps:
+        c = setscenedeps[dep][0]
+        if c not in installed:
+            continue
+        taskhash = setscenedeps[dep][5]
+        taskmanifest = depdir + "/" + c + "." + taskhash
+
+        if os.path.exists(depdir + "/" + c):
+            lnk = os.readlink(depdir + "/" + c)
+            if lnk == c + "." + taskhash and os.path.exists(depdir + "/" + c + ".complete"):
+                continue
+            else:
+                bb.note("%s exists in sysroot, but is stale (%s vs. %s), removing." % (c, lnk, c + "." + taskhash))
+                sstate_clean_manifest(depdir + "/" + lnk, d, workdir)
+                os.unlink(depdir + "/" + c)
+                if os.path.lexists(depdir + "/" + c + ".complete"):
+                    os.unlink(depdir + "/" + c + ".complete")
+        elif os.path.lexists(depdir + "/" + c):
+            os.unlink(depdir + "/" + c)
+
+    # Now handle installs
     for dep in configuredeps:
         c = setscenedeps[dep][0]
         if c not in installed:
@@ -461,14 +483,6 @@ python extend_recipe_sysroot() {
             if lnk == c + "." + taskhash and os.path.exists(depdir + "/" + c + ".complete"):
                 msg_exists.append(c)
                 continue
-            else:
-                bb.note("%s exists in sysroot, but is stale (%s vs. %s), removing." % (c, lnk, c + "." + taskhash))
-                sstate_clean_manifest(depdir + "/" + lnk, d, workdir)
-                os.unlink(depdir + "/" + c)
-                if os.path.lexists(depdir + "/" + c + ".complete"):
-                    os.unlink(depdir + "/" + c + ".complete")
-        elif os.path.lexists(depdir + "/" + c):
-            os.unlink(depdir + "/" + c)
 
         msg_adding.append(c)
 

meta/classes/systemd.bbclass

@@ -32,11 +32,7 @@ if type systemctl >/dev/null 2>/dev/null; then
 
 	if [ "${SYSTEMD_AUTO_ENABLE}" = "enable" ]; then
 		for service in ${SYSTEMD_SERVICE_ESCAPED}; do
-			case "${service}" in
-			*@*)
-				systemctl ${OPTS} enable "${service}"
-				;;
-			esac
+			systemctl ${OPTS} enable "$service"
 		done
 	fi
 

meta/classes/testimage.bbclass

@@ -262,6 +262,24 @@ def testimage_main(d):
     # It would be better to find these modules using instrospection.
     target_kwargs['target_modules_path'] = d.getVar('BBPATH')
 
+    # hardware controlled targets might need further access
+    target_kwargs['powercontrol_cmd'] = d.getVar("TEST_POWERCONTROL_CMD") or None
+    target_kwargs['powercontrol_extra_args'] = d.getVar("TEST_POWERCONTROL_EXTRA_ARGS") or ""
+    target_kwargs['serialcontrol_cmd'] = d.getVar("TEST_SERIALCONTROL_CMD") or None
+    target_kwargs['serialcontrol_extra_args'] = d.getVar("TEST_SERIALCONTROL_EXTRA_ARGS") or ""
+
+    def export_ssh_agent(d):
+        import os
+
+        variables = ['SSH_AGENT_PID', 'SSH_AUTH_SOCK']
+        for v in variables:
+            if v not in os.environ.keys():
+                val = d.getVar(v)
+                if val is not None:
+                    os.environ[v] = val
+
+    export_ssh_agent(d)
+
     # runtime use network for download projects for build
     export_proxies(d)
 

meta/classes/tinderclient.bbclass

@@ -1,368 +0,0 @@
-def tinder_http_post(server, selector, content_type, body):
-    import httplib
-    # now post it
-    for i in range(0,5):
-        try:
-            h = httplib.HTTP(server)
-            h.putrequest('POST', selector)
-            h.putheader('content-type', content_type)
-            h.putheader('content-length', str(len(body)))
-            h.endheaders()
-            h.send(body)
-            errcode, errmsg, headers = h.getreply()
-            #print(errcode, errmsg, headers)
-            return (errcode,errmsg, headers, h.file)
-        except:
-            print("Error sending the report!")
-            # try again
-            pass
-
-    # return some garbage
-    return (-1, "unknown", "unknown", None)
-
-def tinder_form_data(bound, dict, log):
-    output = []
-    # for each key in the dictionary
-    for name in dict:
-        assert dict[name]
-        output.append( "--" + bound )
-        output.append( 'Content-Disposition: form-data; name="%s"' % name )
-        output.append( "" )
-        output.append( dict[name] )
-    if log:
-        output.append( "--" + bound )
-        output.append( 'Content-Disposition: form-data; name="log"; filename="log.txt"' )
-        output.append( '' )
-        output.append( log )
-    output.append( '--' + bound + '--' )
-    output.append( '' )
-
-    return "\r\n".join(output)
-
-def tinder_time_string():
-    """
-    Return the time as GMT
-    """
-    return ""
-
-def tinder_format_http_post(d,status,log):
-    """
-    Format the Tinderbox HTTP post with the data needed
-    for the tinderbox to be happy.
-    """
-
-    import random
-
-    # the variables we will need to send on this form post
-    variables =  {
-        "tree"         : d.getVar('TINDER_TREE'),
-        "machine_name" : d.getVar('TINDER_MACHINE'),
-        "os"           : os.uname()[0],
-        "os_version"   : os.uname()[2],
-        "compiler"     : "gcc",
-        "clobber"      : d.getVar('TINDER_CLOBBER') or "0",
-        "srcdate"      : d.getVar('SRCDATE'),
-        "PN"           : d.getVar('PN'),
-        "PV"           : d.getVar('PV'),
-        "PR"           : d.getVar('PR'),
-        "FILE"         : d.getVar('FILE') or "N/A",
-        "TARGETARCH"   : d.getVar('TARGET_ARCH'),
-        "TARGETFPU"    : d.getVar('TARGET_FPU') or "Unknown",
-        "TARGETOS"     : d.getVar('TARGET_OS') or "Unknown",
-        "MACHINE"      : d.getVar('MACHINE') or "Unknown",
-        "DISTRO"       : d.getVar('DISTRO') or "Unknown",
-        "zecke-rocks"  : "sure",
-    }
-
-    # optionally add the status
-    if status:
-        variables["status"] = str(status)
-
-    # try to load the machine id
-    # we only need on build_status.pl but sending it
-    # always does not hurt
-    try:
-        f = open(d.getVar('TMPDIR')+'/tinder-machine.id', 'r')
-        id = f.read()
-        variables['machine_id'] = id
-    except:
-        pass
-
-    # the boundary we will need
-    boundary = "----------------------------------%d" % int(random.random()*1000000000000)
-
-    # now format the body
-    body = tinder_form_data( boundary, variables, log )
-
-    return ("multipart/form-data; boundary=%s" % boundary),body
-
-
-def tinder_build_start(d):
-    """
-    Inform the tinderbox that a build is starting. We do this
-    by posting our name and tree to the build_start.pl script
-    on the server.
-    """
-
-    # get the body and type
-    content_type, body = tinder_format_http_post(d,None,None)
-    server = d.getVar('TINDER_HOST')
-    url    = d.getVar('TINDER_URL')
-
-    selector = url + "/xml/build_start.pl"
-
-    #print("selector %s and url %s" % (selector, url))
-
-    # now post it
-    errcode, errmsg, headers, h_file = tinder_http_post(server,selector,content_type, body)
-    #print(errcode, errmsg, headers)
-    report = h_file.read()
-
-    # now let us find the machine id that was assigned to us
-    search = "<machine id='"
-    report = report[report.find(search)+len(search):]
-    report = report[0:report.find("'")]
-
-    bb.note("Machine ID assigned by tinderbox: %s" % report )
-
-    # now we will need to save the machine number
-    # we will override any previous numbers
-    f = open(d.getVar('TMPDIR')+"/tinder-machine.id", 'w')
-    f.write(report)
-
-
-def tinder_send_http(d, status, _log):
-    """
-    Send this log as build status
-    """
-
-    # get the body and type
-    server = d.getVar('TINDER_HOST')
-    url    = d.getVar('TINDER_URL')
-
-    selector = url + "/xml/build_status.pl"
-
-    # now post it - in chunks of 10.000 characters
-    new_log = _log
-    while len(new_log) > 0:
-        content_type, body = tinder_format_http_post(d,status,new_log[0:18000])
-        errcode, errmsg, headers, h_file = tinder_http_post(server,selector,content_type, body)
-        #print(errcode, errmsg, headers)
-        #print(h.file.read())
-        new_log = new_log[18000:]
-
-
-def tinder_print_info(d):
-    """
-    Print the TinderBox Info
-        Including informations of the BaseSystem and the Tree
-        we use.
-    """
-
-    # get the local vars
-    time    = tinder_time_string()
-    ops     = os.uname()[0]
-    version = os.uname()[2]
-    url     = d.getVar('TINDER_URL')
-    tree    = d.getVar('TINDER_TREE')
-    branch  = d.getVar('TINDER_BRANCH')
-    srcdate = d.getVar('SRCDATE')
-    machine = d.getVar('MACHINE')
-    distro  = d.getVar('DISTRO')
-    bbfiles = d.getVar('BBFILES')
-    tarch   = d.getVar('TARGET_ARCH')
-    fpu     = d.getVar('TARGET_FPU')
-    oerev   = d.getVar('OE_REVISION') or "unknown"
-
-    # there is a bug with tipple quoted strings
-    # i will work around but will fix the original
-    # bug as well
-    output = []
-    output.append("== Tinderbox Info" )
-    output.append("Time: %(time)s" )
-    output.append("OS: %(ops)s" )
-    output.append("%(version)s" )
-    output.append("Compiler: gcc" )
-    output.append("Tinderbox Client: 0.1" )
-    output.append("Tinderbox Client Last Modified: yesterday" )
-    output.append("Tinderbox Protocol: 0.1" )
-    output.append("URL: %(url)s" )
-    output.append("Tree: %(tree)s" )
-    output.append("Config:" )
-    output.append("branch = '%(branch)s'" )
-    output.append("TARGET_ARCH = '%(tarch)s'" )
-    output.append("TARGET_FPU = '%(fpu)s'" )
-    output.append("SRCDATE = '%(srcdate)s'" )
-    output.append("MACHINE = '%(machine)s'" )
-    output.append("DISTRO = '%(distro)s'" )
-    output.append("BBFILES = '%(bbfiles)s'" )
-    output.append("OEREV = '%(oerev)s'" )
-    output.append("== End Tinderbox Client Info" )
-
-    # now create the real output
-    return "\n".join(output) % vars()
-
-
-def tinder_print_env():
-    """
-    Print the environment variables of this build
-    """
-    time_start = tinder_time_string()
-    time_end   = tinder_time_string()
-
-    # build the environment
-    env = ""
-    for var in os.environ:
-        env += "%s=%s\n" % (var, os.environ[var])
-
-    output = []
-    output.append( "---> TINDERBOX RUNNING env %(time_start)s" )
-    output.append( env )
-    output.append( "<--- TINDERBOX FINISHED (SUCCESS) %(time_end)s" )
-
-    return "\n".join(output) % vars()
-
-def tinder_tinder_start(d, event):
-    """
-    PRINT the configuration of this build
-    """
-
-    time_start = tinder_time_string()
-    config = tinder_print_info(d)
-    #env    = tinder_print_env()
-    time_end   = tinder_time_string()
-    packages = " ".join( event.getPkgs() ) 
-
-    output = []
-    output.append( "---> TINDERBOX PRINTING CONFIGURATION %(time_start)s" )
-    output.append( config )
-    #output.append( env    )
-    output.append( "<--- TINDERBOX FINISHED PRINTING CONFIGURATION %(time_end)s" )
-    output.append( "---> TINDERBOX BUILDING '%(packages)s'" )
-    output.append( "<--- TINDERBOX STARTING BUILD NOW" )
-
-    output.append( "" )
-
-    return "\n".join(output) % vars()
-
-def tinder_do_tinder_report(event):
-    """
-    Report to the tinderbox:
-        On the BuildStart we will inform the box directly
-        On the other events we will write to the TINDER_LOG and
-        when the Task is finished we will send the report.
-
-    The above is not yet fully implemented. Currently we send
-    information immediately. The caching/queuing needs to be
-    implemented. Also sending more or less information is not
-    implemented yet.
-
-    We have two temporary files stored in the TMP directory. One file
-    contains the assigned machine id for the tinderclient. This id gets
-    assigned when we connect the box and start the build process the second
-    file is used to workaround an EventHandler limitation. If BitBake is ran
-    with the continue option we want the Build to fail even if we get the
-    BuildCompleted Event. In this case we have to look up the status and
-    send it instead of 100/success.
-    """
-    import glob
-
-    # variables
-    name = bb.event.getName(event)
-    log  = ""
-    status = 1
-    # Check what we need to do Build* shows we start or are done
-    if name == "BuildStarted":
-        tinder_build_start(event.data)
-        log = tinder_tinder_start(event.data,event)
-
-        try:
-            # truncate the tinder log file
-            f = open(event.data.getVar('TINDER_LOG'), 'w')
-            f.write("")
-            f.close()
-        except:
-            pass
-
-        try:
-            # write a status to the file. This is needed for the -k option
-            # of BitBake
-            g = open(event.data.getVar('TMPDIR')+"/tinder-status", 'w')
-            g.write("")
-            g.close()
-        except IOError:
-            pass
-
-    # Append the Task-Log (compile,configure...) to the log file
-    # we will send to the server
-    if name == "TaskSucceeded" or name == "TaskFailed":
-        log_file = glob.glob("%s/log.%s.*" % (event.data.getVar('T'), event.task))
-
-        if len(log_file) != 0:
-            to_file  = event.data.getVar('TINDER_LOG')
-            log     += "".join(open(log_file[0], 'r').readlines())
-
-    # set the right 'HEADER'/Summary for the TinderBox
-    if name == "TaskStarted":
-        log += "---> TINDERBOX Task %s started\n" % event.task
-    elif name == "TaskSucceeded":
-        log += "<--- TINDERBOX Task %s done (SUCCESS)\n" % event.task
-    elif name == "TaskFailed":
-        log += "<--- TINDERBOX Task %s failed (FAILURE)\n" % event.task
-    elif name == "PkgStarted":
-        log += "---> TINDERBOX Package %s started\n" % event.data.getVar('PF')
-    elif name == "PkgSucceeded":
-        log += "<--- TINDERBOX Package %s done (SUCCESS)\n" % event.data.getVar('PF')
-    elif name == "PkgFailed":
-        if not event.data.getVar('TINDER_AUTOBUILD') == "0":
-            build.exec_task('do_clean', event.data)
-        log += "<--- TINDERBOX Package %s failed (FAILURE)\n" % event.data.getVar('PF')
-        status = 200
-        # remember the failure for the -k case
-        h = open(event.data.getVar('TMPDIR')+"/tinder-status", 'w')
-        h.write("200")
-    elif name == "BuildCompleted":
-        log += "Build Completed\n"
-        status = 100
-        # Check if we have a old status...
-        try:
-            h = open(event.data.getVar('TMPDIR')+'/tinder-status', 'r')
-            status = int(h.read())
-        except:
-            pass
-
-    elif name == "MultipleProviders":
-        log += "---> TINDERBOX Multiple Providers\n"
-        log += "multiple providers are available (%s);\n" % ", ".join(event.getCandidates())
-        log += "consider defining PREFERRED_PROVIDER_%s\n" % event.getItem()
-        log += "is runtime: %d\n" % event.isRuntime()
-        log += "<--- TINDERBOX Multiple Providers\n"
-    elif name == "NoProvider":
-        log += "Error: No Provider for: %s\n" % event.getItem()
-        log += "Error:Was Runtime: %d\n" % event.isRuntime()
-        status = 200
-        # remember the failure for the -k case
-        h = open(event.data.getVar('TMPDIR')+"/tinder-status", 'w')
-        h.write("200")
-
-    # now post the log
-    if len(log) == 0:
-        return
-
-    # for now we will use the http post method as it is the only one
-    log_post_method = tinder_send_http
-    log_post_method(event.data, status, log)
-
-
-# we want to be an event handler
-addhandler tinderclient_eventhandler
-python tinderclient_eventhandler() {
-    if e.data is None or bb.event.getName(e) == "MsgNote":
-        return
-
-    do_tinder_report = e.data.getVar('TINDER_REPORT')
-    if do_tinder_report and do_tinder_report == "1":
-        tinder_do_tinder_report(e)
-
-    return
-}

meta/classes/uninative.bbclass

@@ -45,7 +45,7 @@ python uninative_event_fetchloader() {
         tarballdir = os.path.join(d.getVar("UNINATIVE_DLDIR"), chksum)
         tarballpath = os.path.join(tarballdir, tarball)
 
-        if not os.path.exists(tarballpath):
+        if not os.path.exists(tarballpath + ".done"):
             bb.utils.mkdirhier(tarballdir)
             if d.getVar("UNINATIVE_URL") == "unset":
                 bb.fatal("Uninative selected but not configured, please set UNINATIVE_URL")

meta/conf/bitbake.conf

@@ -481,14 +481,13 @@ export PATH
 HOSTTOOLS_DIR = "${TMPDIR}/hosttools"
 
 # Tools needed to run builds with OE-Core
-# python is special cased to point at python2
 HOSTTOOLS += " \
     [ ar as awk basename bash bzip2 cat chgrp chmod chown chrpath cmp comm cp cpio \
     cpp cut date dd diff diffstat dirname du echo egrep env expand expr false \
     fgrep file find flock g++ gawk gcc getconf getopt git grep gunzip gzip \
     head hostname iconv id install ld ldd ln ls make makeinfo md5sum mkdir mknod \
-    mktemp mv nm objcopy objdump od patch perl pod2man pr printf pwd python2 \
-    python2.7 python3 ranlib readelf readlink realpath rm rmdir rpcgen sed seq sh sha256sum \
+    mktemp mv nm objcopy objdump od patch perl pod2man pr printf pwd \
+    python3 ranlib readelf readlink realpath rm rmdir rpcgen sed seq sh sha256sum \
     sleep sort split stat strings strip tail tar tee test touch tr true uname \
     uniq wc wget which xargs \
 "

meta/conf/distro/include/maintainers.inc

@@ -103,7 +103,6 @@ RECIPE_MAINTAINER_pn-cogl-1.0 = "Ross Burton <ross.burton@intel.com>"
 RECIPE_MAINTAINER_pn-connman = "Changhyeok Bae <changhyeok.bae@gmail.com>"
 RECIPE_MAINTAINER_pn-connman-conf = "Ross Burton <ross.burton@intel.com>"
 RECIPE_MAINTAINER_pn-connman-gnome = "Ross Burton <ross.burton@intel.com>"
-RECIPE_MAINTAINER_pn-console-tools = "Chen Qi <Qi.Chen@windriver.com>"
 RECIPE_MAINTAINER_pn-consolekit = "Chen Qi <Qi.Chen@windriver.com>"
 RECIPE_MAINTAINER_pn-core-image-base = "Richard Purdie <richard.purdie@linuxfoundation.org>"
 RECIPE_MAINTAINER_pn-core-image-minimal = "Richard Purdie <richard.purdie@linuxfoundation.org>"
@@ -355,7 +354,7 @@ RECIPE_MAINTAINER_pn-libmatchbox = "Ross Burton <ross.burton@intel.com>"
 RECIPE_MAINTAINER_pn-libmnl = "Khem Raj <raj.khem@gmail.com>"
 RECIPE_MAINTAINER_pn-libmpc = "Khem Raj <raj.khem@gmail.com>"
 RECIPE_MAINTAINER_pn-libmodule-build-perl = "Tim Orling <timothy.t.orling@linux.intel.com>"
-RECIPE_MAINTAINER_pn-libmodulemd = "Alexander Kanavin <alex.kanavin@gmail.com>"
+RECIPE_MAINTAINER_pn-libmodulemd-v1 = "Alexander Kanavin <alex.kanavin@gmail.com>"
 RECIPE_MAINTAINER_pn-libnewt = "Hongxu Jia <hongxu.jia@windriver.com>"
 RECIPE_MAINTAINER_pn-libnewt-python = "Hongxu Jia <hongxu.jia@windriver.com>"
 RECIPE_MAINTAINER_pn-libnl = "Alexander Kanavin <alex.kanavin@gmail.com>"
@@ -678,7 +677,6 @@ RECIPE_MAINTAINER_pn-tcf-agent = "Anuj Mittal <anuj.mittal@intel.com>"
 RECIPE_MAINTAINER_pn-tcl = "Yi Zhao <yi.zhao@windriver.com>"
 RECIPE_MAINTAINER_pn-tcp-wrappers = "Robert Yang <liezhi.yang@windriver.com>"
 RECIPE_MAINTAINER_pn-testexport-tarball = "Richard Purdie <richard.purdie@linuxfoundation.org>"
-RECIPE_MAINTAINER_pn-texi2html = "Robert Yang <liezhi.yang@windriver.com>"
 RECIPE_MAINTAINER_pn-texinfo = "Anuj Mittal <anuj.mittal@intel.com>"
 RECIPE_MAINTAINER_pn-texinfo-dummy-native = "Anuj Mittal <anuj.mittal@intel.com>"
 RECIPE_MAINTAINER_pn-tiff = "Alexander Kanavin <alex.kanavin@gmail.com>"

meta/conf/distro/include/no-static-libs.inc

@@ -15,10 +15,6 @@ DISABLE_STATIC_pn-nativesdk-libcap = ""
 DISABLE_STATIC_pn-libpcap = ""
 # needed by gdb
 DISABLE_STATIC_pn-readline = ""
-# needed by pseudo
-DISABLE_STATIC_pn-sqlite3 = ""
-DISABLE_STATIC_pn-sqlite3-native = ""
-DISABLE_STATIC_pn-nativesdk-sqlite3 = ""
 # openjade/sgml-common have build issues without static libs
 DISABLE_STATIC_pn-sgml-common-native = ""
 DISABLE_STATIC_pn-openjade-native = ""

meta/conf/distro/include/tcmode-default.inc

@@ -18,15 +18,15 @@ PREFERRED_PROVIDER_virtual/gettext ??= "gettext"
 
 GCCVERSION ?= "9.%"
 SDKGCCVERSION ?= "${GCCVERSION}"
-BINUVERSION ?= "2.32%"
+BINUVERSION ?= "2.33%"
 GDBVERSION ?= "8.3%"
 GLIBCVERSION ?= "2.30%"
 LINUXLIBCVERSION ?= "5.2%"
 QEMUVERSION ?= "4.1%"
-GOVERSION ?= "1.12%"
+GOVERSION ?= "1.13%"
 # This can not use wildcards like 8.0.% since it is also used in mesa to denote
 # llvm version being used, so always bump it with llvm recipe version bump
-LLVMVERSION ?= "8.0.1"
+LLVMVERSION ?= "9.0.0"
 
 PREFERRED_VERSION_gcc ?= "${GCCVERSION}"
 PREFERRED_VERSION_gcc-cross-${TARGET_ARCH} ?= "${GCCVERSION}"

meta/conf/documentation.conf

@@ -113,7 +113,7 @@ COMPATIBLE_MACHINE[doc] = "A regular expression that resolves to one or more tar
 COMPLEMENTARY_GLOB[doc] = "Defines wildcards to match when installing a list of complementary packages for all the packages installed in an image."
 CONFFILES[doc] = "Identifies editable or configurable files that are part of a package."
 CONFIG_SITE[doc] = "A list of files that contains autoconf test results relevant to the current build. This variable is used by the Autotools utilities when running configure."
-CONFLICT_DISTRO_FEATURES[doc] = "When a recipe inherits the distro_features_check class, this variable identifies distribution features that would be in conflict should the recipe be built."
+CONFLICT_DISTRO_FEATURES[doc] = "When a recipe inherits the features_check class, this variable identifies distribution features that would be in conflict should the recipe be built."
 CORE_IMAGE_EXTRA_INSTALL[doc] = "Specifies the list of packages to be added to the image. You should only set this variable in the conf/local.conf file in the Build Directory."
 COREBASE[doc] = "Specifies the parent directory of the OpenEmbedded Core Metadata layer (i.e. meta)."
 CONF_VERSION[doc] = "Tracks the version of local.conf.  Increased each time build/conf/ changes incompatibly."
@@ -346,7 +346,7 @@ QMAKE_PROFILES[doc] = "Specifies your own subset of .pro files to be built for u
 
 RCONFLICTS[doc] = "The list of packages that conflict with another package. Note that the package will not be installed if the conflicting packages are not first removed."
 RDEPENDS[doc] = "Lists a package's runtime dependencies (i.e. other packages) that must be installed for the package to be built. They must be the names of other packages as listed in the PACKAGES variable, not recipe names (PN)."
-REQUIRED_DISTRO_FEATURES[doc] = "When a recipe inherits the distro_features_check class, this variable identifies distribution features that must exist in the current configuration in order for the OpenEmbedded build system to build the recipe."
+REQUIRED_DISTRO_FEATURES[doc] = "When a recipe inherits the features_check class, this variable identifies distribution features that must exist in the current configuration in order for the OpenEmbedded build system to build the recipe."
 RM_WORK_EXCLUDE[doc] = "With rm_work enabled, this variable specifies a list of packages whose work directories should not be removed."
 ROOTFS[doc] = "Indicates a filesystem image to include as the root filesystem."
 ROOTFS_POSTPROCESS_COMMAND[doc] = "Added by classes to run post processing commands once the OpenEmbedded build system has created the root filesystem."

meta/conf/machine/include/riscv/arch-riscv.inc

@@ -8,3 +8,8 @@ TUNE_CCARGS .= ""
 
 # QEMU usermode fails with invalid instruction error (For riscv32)
 MACHINE_FEATURES_BACKFILL_CONSIDERED_append = "${@bb.utils.contains('TUNE_FEATURES', 'riscv32', ' qemu-usermode', '', d)}"
+
+# Fix: ld: unrecognized option '--hash-style=sysv'
+LINKER_HASH_STYLE_libc-newlib = ""
+# Fix: ld: unrecognized option '--hash-style=gnu'
+LINKER_HASH_STYLE_libc-baremetal = ""

meta/conf/machine/include/tune-cortexa32.inc

@@ -10,9 +10,9 @@ require conf/machine/include/arm/arch-armv8a.inc
 AVAILTUNES += "cortexa32 cortexa32-crypto"
 ARMPKGARCH_tune-cortexa32             = "cortexa32"
 ARMPKGARCH_tune-cortexa32-crypto      = "cortexa32"
-TUNE_FEATURES_tune-cortexa32          = "aarch64 cortexa32 crc"
-TUNE_FEATURES_tune-cortexa32-crypto   = "aarch64 cortexa32 crc crypto"
+TUNE_FEATURES_tune-cortexa32          = "armv8a cortexa32 crc"
+TUNE_FEATURES_tune-cortexa32-crypto   = "armv8a cortexa32 crc crypto"
 PACKAGE_EXTRA_ARCHS_tune-cortexa32             = "${PACKAGE_EXTRA_ARCHS_tune-armv8a-crc} cortexa32"
 PACKAGE_EXTRA_ARCHS_tune-cortexa32-crypto      = "${PACKAGE_EXTRA_ARCHS_tune-armv8a-crc-crypto} cortexa32 cortexa32-crypto"
-BASE_LIB_tune-cortexa32               = "lib64"
-BASE_LIB_tune-cortexa32-crypto        = "lib64"
+BASE_LIB_tune-cortexa32               = "lib"
+BASE_LIB_tune-cortexa32-crypto        = "lib"

meta/conf/machine/qemuriscv64.conf

@@ -5,5 +5,5 @@
 require conf/machine/include/riscv/qemuriscv.inc
 
 EXTRA_IMAGEDEPENDS += "u-boot"
-UBOOT_MACHINE = "qemu-riscv64_defconfig"
+UBOOT_MACHINE = "qemu-riscv64_smode_defconfig"
 UBOOT_ELF = "u-boot"

meta/files/common-licenses/Intel

@@ -0,0 +1,105 @@
+1. Copyright Notice
+
+Some or all of this work - Copyright (c) 1999 - 2017, Intel Corp.
+All rights reserved.
+
+2. License
+
+2.1. This is your license from Intel Corp. under its intellectual property
+rights. You may have additional license terms from the party that provided
+you this software, covering your right to use that party's intellectual
+property rights.
+
+2.2. Intel grants, free of charge, to any person ("Licensee") obtaining a
+copy of the source code appearing in this file ("Covered Code") an
+irrevocable, perpetual, worldwide license under Intel's copyrights in the
+base code distributed originally by Intel ("Original Intel Code") to copy,
+make derivatives, distribute, use and display any portion of the Covered
+Code in any form, with the right to sublicense such rights; and
+
+2.3. Intel grants Licensee a non-exclusive and non-transferable patent
+license (with the right to sublicense), under only those claims of Intel
+patents that are infringed by the Original Intel Code, to make, use, sell,
+offer to sell, and import the Covered Code and derivative works thereof
+solely to the minimum extent necessary to exercise the above copyright
+license, and in no event shall the patent license extend to any additions
+to or modifications of the Original Intel Code. No other license or right
+is granted directly or by implication, estoppel or otherwise;
+
+The above copyright and patent license is granted only if the following
+conditions are met:
+
+3. Conditions
+
+3.1. Redistribution of Source with Rights to Further Distribute Source.
+Redistribution of source code of any substantial portion of the Covered
+Code or modification with rights to further distribute source must include
+the above Copyright Notice, the above License, this list of Conditions,
+and the following Disclaimer and Export Compliance provision. In addition,
+Licensee must cause all Covered Code to which Licensee contributes to
+contain a file documenting the changes Licensee made to create that Covered
+Code and the date of any change. Licensee must include in that file the
+documentation of any changes made by any predecessor Licensee. Licensee
+must include a prominent statement that the modification is derived,
+directly or indirectly, from Original Intel Code.
+
+3.2. Redistribution of Source with no Rights to Further Distribute Source.
+Redistribution of source code of any substantial portion of the Covered
+Code or modification without rights to further distribute source must
+include the following Disclaimer and Export Compliance provision in the
+documentation and/or other materials provided with distribution. In
+addition, Licensee may not authorize further sublicense of source of any
+portion of the Covered Code, and must include terms to the effect that the
+license from Licensee to its licensee is limited to the intellectual
+property embodied in the software Licensee provides to its licensee, and
+not to intellectual property embodied in modifications its licensee may
+make.
+
+3.3. Redistribution of Executable. Redistribution in executable form of any
+substantial portion of the Covered Code or modification must reproduce the
+above Copyright Notice, and the following Disclaimer and Export Compliance
+provision in the documentation and/or other materials provided with the
+distribution.
+
+3.4. Intel retains all right, title, and interest in and to the Original
+Intel Code.
+
+3.5. Neither the name Intel nor any other trademark owned or controlled by
+Intel shall be used in advertising or otherwise to promote the sale, use or
+other dealings in products derived from or relating to the Covered Code
+without prior written authorization from Intel.
+
+4. Disclaimer and Export Compliance
+
+4.1. INTEL MAKES NO WARRANTY OF ANY KIND REGARDING ANY SOFTWARE PROVIDED
+HERE. ANY SOFTWARE ORIGINATING FROM INTEL OR DERIVED FROM INTEL SOFTWARE
+IS PROVIDED "AS IS," AND INTEL WILL NOT PROVIDE ANY SUPPORT, ASSISTANCE,
+INSTALLATION, TRAINING OR OTHER SERVICES. INTEL WILL NOT PROVIDE ANY
+UPDATES, ENHANCEMENTS OR EXTENSIONS. INTEL SPECIFICALLY DISCLAIMS ANY
+IMPLIED WARRANTIES OF MERCHANTABILITY, NONINFRINGEMENT AND FITNESS FOR A
+PARTICULAR PURPOSE.
+
+4.2. IN NO EVENT SHALL INTEL HAVE ANY LIABILITY TO LICENSEE, ITS LICENSEES
+OR ANY OTHER THIRD PARTY, FOR ANY LOST PROFITS, LOST DATA, LOSS OF USE OR
+COSTS OF PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES, OR FOR ANY INDIRECT,
+SPECIAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THIS AGREEMENT, UNDER ANY
+CAUSE OF ACTION OR THEORY OF LIABILITY, AND IRRESPECTIVE OF WHETHER INTEL
+HAS ADVANCE NOTICE OF THE POSSIBILITY OF SUCH DAMAGES. THESE LIMITATIONS
+SHALL APPLY NOTWITHSTANDING THE FAILURE OF THE ESSENTIAL PURPOSE OF ANY
+LIMITED REMEDY.
+
+4.3. Licensee shall not export, either directly or indirectly, any of this
+software or system incorporating such software without first obtaining any
+required license or other approval from the U. S. Department of Commerce or
+any other agency or department of the United States Government. In the
+event Licensee exports any such software from the United States or
+re-exports any such software from a foreign destination, Licensee shall
+ensure that the distribution and export/re-export of the software is in
+compliance with all laws, regulations, orders, or other restrictions of the
+U.S. Export Administration Regulations. Licensee agrees that neither it nor
+any of its subsidiaries will export/re-export any technical data, process,
+software, or service, directly or indirectly, to any country for which the
+United States government or any agency thereof requires an export license,
+other governmental approval, or letter of assurance, without first obtaining
+such license, approval or letter.
+

meta/lib/oe/buildhistory_analysis.py

@@ -413,7 +413,7 @@ def compare_dict_blobs(path, ablob, bblob, report_all, report_ver):
                 if abs(percentchg) < monitor_numeric_threshold:
                     continue
             elif (not report_all) and key in list_fields:
-                if key == "FILELIST" and path.endswith("-dbg") and bstr.strip() != '':
+                if key == "FILELIST" and (path.endswith("-dbg") or path.endswith("-src")) and bstr.strip() != '':
                     continue
                 if key in ['RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RREPLACES', 'RCONFLICTS']:
                     (depvera, depverb) = compare_pkg_lists(astr, bstr)

meta/lib/oe/package_manager.py

@@ -217,7 +217,7 @@ class OpkgIndexer(Indexer):
                 if not os.path.exists(pkgs_file):
                     open(pkgs_file, "w").close()
 
-                index_cmds.add('%s -r %s -p %s -m %s' %
+                index_cmds.add('%s --checksum md5 --checksum sha256 -r %s -p %s -m %s' %
                                   (opkg_index_cmd, pkgs_file, pkgs_file, pkgs_dir))
 
                 index_sign_files.add(pkgs_file)
@@ -298,7 +298,7 @@ class DpkgIndexer(Indexer):
                 release.write("Label: %s\n" % arch)
 
             cmd += "PSEUDO_UNLOAD=1 %s release . >> Release" % apt_ftparchive
-            
+
             index_cmds.append(cmd)
 
             deb_dirs_found = True
@@ -570,6 +570,8 @@ class PackageManager(object, metaclass=ABCMeta):
 
             for lang in split_linguas:
                 globs += " *-locale-%s" % lang
+                for complementary_linguas in (self.d.getVar('IMAGE_LINGUAS_COMPLEMENTARY') or "").split():
+                    globs += (" " + complementary_linguas) % lang
 
         if globs is None:
             return
@@ -655,7 +657,7 @@ def create_packages_dir(d, subrepo_dir, deploydir, taskname, filterbydependencie
     pn = d.getVar("PN")
     seendirs = set()
     multilibs = {}
-   
+
     bb.utils.remove(subrepo_dir, recurse=True)
     bb.utils.mkdirhier(subrepo_dir)
 
@@ -1006,8 +1008,8 @@ class RpmPM(PackageManager):
     def load_old_install_solution(self):
         if not os.path.exists(self.solution_manifest):
             return []
-
-        return open(self.solution_manifest, 'r').read().split()
+        with open(self.solution_manifest, 'r') as fd:
+            return fd.read().split()
 
     def _script_num_prefix(self, path):
         files = os.listdir(path)

meta/lib/oe/packagedata.py

@@ -17,9 +17,8 @@ def read_pkgdatafile(fn):
 
     if os.access(fn, os.R_OK):
         import re
-        f = open(fn, 'r')
-        lines = f.readlines()
-        f.close()
+        with open(fn, 'r') as f:
+            lines = f.readlines()
         r = re.compile("([^:]+):\s*(.*)")
         for l in lines:
             m = r.match(l)

meta/lib/oe/types.py

@@ -154,7 +154,8 @@ def path(value, relativeto='', normalize='true', mustexist='false'):
 
     if boolean(mustexist):
         try:
-            open(value, 'r')
+            with open(value, 'r'):
+                pass
         except IOError as exc:
             if exc.errno == errno.ENOENT:
                 raise ValueError("{0}: {1}".format(value, os.strerror(errno.ENOENT)))
@@ -183,4 +184,3 @@ def qemu_use_kvm(kvm, target_arch):
         elif build_arch == target_arch:
             use_kvm = True
     return use_kvm
-

meta/lib/oeqa/core/decorator/data.py

@@ -18,6 +18,26 @@ def has_feature(td, feature):
         return True
     return False
 
+def has_machine(td, machine):
+    """
+        Checks for MACHINE.
+    """
+
+    if (machine in td.get('MACHINE', '')):
+        return True
+    return False
+
+def is_qemu(td, qemu):
+    """
+        Checks if MACHINE is qemu.
+    """
+
+    machine = td.get('MACHINE', '')
+    if (qemu in td.get('MACHINE', '') or
+    machine.startswith('qemu')):
+        return True
+    return False
+
 @registerDecorator
 class skipIfDataVar(OETestDecorator):
     """
@@ -113,3 +133,90 @@ class skipIfNotFeature(OETestDecorator):
         self.logger.debug(msg)
         if not has_feature(self.case.td, self.value):
             self.case.skipTest(self.msg)
+
+@registerDecorator
+class skipIfFeature(OETestDecorator):
+    """
+        Skip test based on DISTRO_FEATURES.
+
+        value must not be in distro features or it will skip the test
+        with msg as the reason.
+    """
+
+    attrs = ('value', 'msg')
+
+    def setUpDecorator(self):
+        msg = ('Checking if %s is not in DISTRO_FEATURES '
+               'or IMAGE_FEATURES' % (self.value))
+        self.logger.debug(msg)
+        if has_feature(self.case.td, self.value):
+            self.case.skipTest(self.msg)
+
+@registerDecorator
+class skipIfNotMachine(OETestDecorator):
+    """
+        Skip test based on MACHINE.
+
+        value must be match MACHINE or it will skip the test
+        with msg as the reason.
+    """
+
+    attrs = ('value', 'msg')
+
+    def setUpDecorator(self):
+        msg = ('Checking if %s is not this MACHINE' % self.value)
+        self.logger.debug(msg)
+        if not has_machine(self.case.td, self.value):
+            self.case.skipTest(self.msg)
+
+@registerDecorator
+class skipIfMachine(OETestDecorator):
+    """
+        Skip test based on Machine.
+
+        value must not be this machine or it will skip the test
+        with msg as the reason.
+    """
+
+    attrs = ('value', 'msg')
+
+    def setUpDecorator(self):
+        msg = ('Checking if %s is this MACHINE' % self.value)
+        self.logger.debug(msg)
+        if has_machine(self.case.td, self.value):
+            self.case.skipTest(self.msg)
+
+@registerDecorator
+class skipIfNotQemu(OETestDecorator):
+    """
+        Skip test based on MACHINE.
+
+        value must be a qemu MACHINE or it will skip the test
+        with msg as the reason.
+    """
+
+    attrs = ('value', 'msg')
+
+    def setUpDecorator(self):
+        msg = ('Checking if %s is not this MACHINE' % self.value)
+        self.logger.debug(msg)
+        if not is_qemu(self.case.td, self.value):
+            self.case.skipTest(self.msg)
+
+@registerDecorator
+class skipIfQemu(OETestDecorator):
+    """
+        Skip test based on Qemu Machine.
+
+        value must not be a qemu machine or it will skip the test
+        with msg as the reason.
+   """
+
+    attrs = ('value', 'msg')
+
+    def setUpDecorator(self):
+        msg = ('Checking if %s is this MACHINE' % self.value)
+        self.logger.debug(msg)
+        if is_qemu(self.case.td, self.value):
+             self.case.skipTest(self.msg)
+

meta/lib/oeqa/core/utils/concurrencytest.py

@@ -221,6 +221,15 @@ def removebuilddir(d):
     while delay and os.path.exists(d + "/bitbake.lock"):
         time.sleep(1)
         delay = delay - 1
+    # Deleting these directories takes a lot of time, use autobuilder
+    # clobberdir if its available
+    clobberdir = os.path.expanduser("~/yocto-autobuilder-helper/janitor/clobberdir")
+    if os.path.exists(clobberdir):
+        try:
+            subprocess.check_call([clobberdir, d])
+            return
+        except subprocess.CalledProcessError:
+            pass
     bb.utils.prunedir(d, ionice=True)
 
 def fork_for_tests(concurrency_num, suite):

meta/lib/oeqa/manual/bsp-hw.json

@@ -1,30 +1,4 @@
 [
-    {
-        "test": {
-            "@alias": "bsps-hw.bsps-hw.rpm_-__install_dependency_package",
-            "author": [
-                {
-                    "email": "alexandru.c.georgescu@intel.com",
-                    "name": "alexandru.c.georgescu@intel.com"
-                }
-            ],
-            "execution": {
-                "1": {
-                    "action": "Get a not previously installed RPM package or build one on local machine, which should have run-time dependency.For example, \"mc\" (Midnight Commander, which is a visual file manager) should depend on \"ncurses-terminfo\".   \n\n$ bitbake mc  \n\n\n",
-                    "expected_results": ""
-                },
-                "2": {
-                    "action": "Copy the package into a system folder (for example /home/root/rpm_packages).  \n\n\n",
-                    "expected_results": ""
-                },
-                "3": {
-                    "action": "Run \"rpm -ivh package_name\" and check the output, for example \"rpm -ivh mc.rpm*\" should report the dependency on \"ncurses-terminfo\".\n\n\n\n",
-                    "expected_results": "3 . rpm command should report message when some RPM installation depends on other packages."
-                }
-            },
-            "summary": "rpm_-__install_dependency_package"
-        }
-    },
     {
         "test": {
             "@alias": "bsps-hw.bsps-hw.boot_and_install_from_USB",
@@ -171,28 +145,6 @@
             "summary": "shutdown_system"
         }
     },
-    {
-        "test": {
-            "@alias": "bsps-hw.bsps-hw.reboot_system",
-            "author": [
-                {
-                    "email": "alexandru.c.georgescu@intel.com",
-                    "name": "alexandru.c.georgescu@intel.com"
-                }
-            ],
-            "execution": {
-                "1": {
-                    "action": "boot system",
-                    "expected_results": ""
-                },
-                "2": {
-                    "action": "launch terminal and run \"reboot\"",
-                    "expected_results": "System can reboot successfully . "
-                }
-            },
-            "summary": "reboot_system"
-        }
-    },
     {
         "test": {
             "@alias": "bsps-hw.bsps-hw.switch_among_multi_applications_and_desktop",
@@ -223,114 +175,6 @@
             "summary": "switch_among_multi_applications_and_desktop"
         }
     },
-    {
-        "test": {
-            "@alias": "bsps-hw.bsps-hw.USB_-_mount",
-            "author": [
-                {
-                    "email": "alexandru.c.georgescu@intel.com",
-                    "name": "alexandru.c.georgescu@intel.com"
-                }
-            ],
-            "execution": {
-                "1": {
-                    "action": "Boot system \n\n",
-                    "expected_results": ""
-                },
-                "2": {
-                    "action": "Plug USB stick, it should be mount in  /run/media/sd(x) If X-window system does not start and show USB device, then use the terminal to mount it, following the next steps: \na. Locate the usb stick (usually it is on /dev/sdb) \nb. Create a directory with \"mkdir stick\"  (so you will have such a path as: /home/root/stick). \nc. Run the command  \"mount  /dev/sdb  /home/root/stick\" to mount  USB device on it. \n\n",
-                    "expected_results": "USB device should be mounted in  /run/media/sd(x)  \nor in  /home/root/stick  \n\n"
-                },
-                "3": {
-                    "action": "Then you can access USB stick (/home/root/stick) via Terminal or GUI  and try various commands and actions like \"cp\", \"mv\", \"touch\" and \"rm\".  Type \"dmesg\" command and check for recent mounted devices.",
-                    "expected_results": "Basic commands work properly. The system sends a notification in \"dmesg\" command, showing that the USB stick is accessible and the device is mounted ."
-                }
-            },
-            "summary": "USB_-_mount"
-        }
-    },
-    {
-        "test": {
-            "@alias": "bsps-hw.bsps-hw.USB_-_read_files",
-            "author": [
-                {
-                    "email": "alexandru.c.georgescu@intel.com",
-                    "name": "alexandru.c.georgescu@intel.com"
-                }
-            ],
-            "execution": {
-                "1": {
-                    "action": "boot system",
-                    "expected_results": "view/copy successfully"
-                },
-                "2": {
-                    "action": "plug usb stick",
-                    "expected_results": ""
-                },
-                "3": {
-                    "action": "view files in usb by file browser",
-                    "expected_results": ""
-                },
-                "4": {
-                    "action": "copy some files from usb to local hardware",
-                    "expected_results": ""
-                }
-            },
-            "summary": "USB_-_read_files"
-        }
-    },
-    {
-        "test": {
-            "@alias": "bsps-hw.bsps-hw.USB_-_umount",
-            "author": [
-                {
-                    "email": "alexandru.c.georgescu@intel.com",
-                    "name": "alexandru.c.georgescu@intel.com"
-                }
-            ],
-            "execution": {
-                "1": {
-                    "action": "boot system",
-                    "expected_results": "usb directory in file browser automatically missed"
-                },
-                "2": {
-                    "action": "plug usb stick",
-                    "expected_results": ""
-                },
-                "3": {
-                    "action": "view files in usb by file browser \n4.unplug usb",
-                    "expected_results": ""
-                }
-            },
-            "summary": "USB_-_umount"
-        }
-    },
-    {
-        "test": {
-            "@alias": "bsps-hw.bsps-hw.USB_-_write_files",
-            "author": [
-                {
-                    "email": "alexandru.c.georgescu@intel.com",
-                    "name": "alexandru.c.georgescu@intel.com"
-                }
-            ],
-            "execution": {
-                "1": {
-                    "action": "boot system",
-                    "expected_results": "create/copy successfully"
-                },
-                "2": {
-                    "action": "plug usb stick",
-                    "expected_results": ""
-                },
-                "3": {
-                    "action": "create files in usb \n4.copy some files from local hardware to usb",
-                    "expected_results": ""
-                }
-            },
-            "summary": "USB_-_write_files"
-        }
-    },
     {
         "test": {
             "@alias": "bsps-hw.bsps-hw.ethernet_static_ip_set_in_connman",
@@ -611,136 +455,6 @@
             "summary": "Add_multiple_files_in_media_player"
         }
     },
-    {
-        "test": {
-            "@alias": "bsps-hw.bsps-hw.check_bash_in_image",
-            "author": [
-                {
-                    "email": "alexandru.c.georgescu@intel.com",
-                    "name": "alexandru.c.georgescu@intel.com"
-                }
-            ],
-            "execution": {
-                "1": {
-                    "action": "After system is up, check if bash command exists with command \"which bash\"",
-                    "expected_results": "bash command should exist in image giving something as below  \"/bin/bash\""
-                }
-            },
-            "summary": "check_bash_in_image"
-        }
-    },
-    {
-        "test": {
-            "@alias": "bsps-hw.bsps-hw.MicroSD_-__mount",
-            "author": [
-                {
-                    "email": "alexandru.c.georgescu@intel.com",
-                    "name": "alexandru.c.georgescu@intel.com"
-                }
-            ],
-            "execution": {
-                "1": {
-                    "action": "boot system",
-                    "expected_results": "system notify that MicroSDis accessible"
-                },
-                "2": {
-                    "action": "plug MicroSD card",
-                    "expected_results": ""
-                }
-            },
-            "summary": "MicroSD_-__mount"
-        }
-    },
-    {
-        "test": {
-            "@alias": "bsps-hw.bsps-hw.MicroSD_-__read_files",
-            "author": [
-                {
-                    "email": "alexandru.c.georgescu@intel.com",
-                    "name": "alexandru.c.georgescu@intel.com"
-                }
-            ],
-            "execution": {
-                "1": {
-                    "action": "boot system",
-                    "expected_results": "view/copy successfully"
-                },
-                "2": {
-                    "action": "plug MicroSD card",
-                    "expected_results": ""
-                },
-                "3": {
-                    "action": "view files inMicroSD by file browser",
-                    "expected_results": ""
-                },
-                "4": {
-                    "action": "copy some files fromMicroSD to local hardware",
-                    "expected_results": ""
-                }
-            },
-            "summary": "MicroSD_-__read_files"
-        }
-    },
-    {
-        "test": {
-            "@alias": "bsps-hw.bsps-hw.MicroSD_-__umount",
-            "author": [
-                {
-                    "email": "alexandru.c.georgescu@intel.com",
-                    "name": "alexandru.c.georgescu@intel.com"
-                }
-            ],
-            "execution": {
-                "1": {
-                    "action": "boot system",
-                    "expected_results": "MicroSD in file browser automatically missed . "
-                },
-                "2": {
-                    "action": "plug MicroSD card",
-                    "expected_results": ""
-                },
-                "3": {
-                    "action": "view files in MicroSDby file browser",
-                    "expected_results": ""
-                },
-                "4": {
-                    "action": "unplug MicroSD",
-                    "expected_results": ""
-                }				
-            },
-            "summary": "MicroSD_-__umount"
-        }
-    },
-    {
-        "test": {
-            "@alias": "bsps-hw.bsps-hw.MicroSD_-__write_files",
-            "author": [
-                {
-                    "email": "alexandru.c.georgescu@intel.com",
-                    "name": "alexandru.c.georgescu@intel.com"
-                }
-            ],
-            "execution": {
-                "1": {
-                    "action": "boot system",
-                    "expected_results": "create/copy successfully"
-                },
-                "2": {
-                    "action": "plug MicroSD card",
-                    "expected_results": ""
-                },
-                "3": {
-                    "action": "create files in MicroSD",
-                    "expected_results": ""
-                },
-                "4": {
-                    "action": "copy some files from local hardware to MicroSD",
-                    "expected_results": ""
-                }
-            },
-            "summary": "MicroSD_-__write_files"
-        }
-    },
     {
         "test": {
             "@alias": "bsps-hw.bsps-hw.video_-_libva_check_(ogg_video_play)",
@@ -1283,4 +997,4 @@
             "summary": "Boot_from_JFFS2_image"
         }
     }
-]
+]

meta/lib/oeqa/manual/bsp-qemu.json

@@ -1,222 +0,0 @@
-[
-  {
-    "test": {
-      "@alias": "bsps-qemu.bsps-tools.qemu_can_be_started_with_KVM_enabled",
-      "author": [
-        {
-          "email": "alexandru.c.georgescu@intel.com",
-          "name": "alexandru.c.georgescu@intel.com"
-        }
-      ],
-      "execution": {
-        "1": {
-          "action": "Build a kernel with KVM enabled  \n\nIn Local.conf add  \n\nQEMU_USE_KVM = \"${@ '1' if os.access('/dev/kvm', os.R_OK|os.W_OK) else '0' }\"  \n\n ",
-          "expected_results": ""
-        },
-        "2": {
-          "action": "Start qemu with option \"kvm\" with runqemu \n    a. If you start qemu with kvm failed, maybe it is because host not install kvm and vhost_net module. Follow below link to install them. \n    b. vhost_test refer:  https://wiki.yoctoproject.org/wiki/Running_an_x86_Yocto_Linux_image_under_QEMU_KVM \n    c. kvm refer: https://wiki.yoctoproject.org/wiki/How_to_enable_KVM_for_Poky_qemu",
-          "expected_results": ""
-        },
-        "3": {
-          "action": "Check if qemu starts up and if kvm_intel module is used",
-          "expected_results": ""
-        },
-        "4": {
-          "action": "If kvm_intel module is not used when starting qemu, it will show 0 in \"Used by\" column when you run \"lsmod | grep kvm_intel\" ",
-          "expected_results": "KVM enabled with qemu \nExecute \"lsmod | grep kvm_intel\" from your host twice, before and after you \nstart the qemu with kvm option. Before start, the number should be 0, \nafter start, the number should bigger than 0."
-        }
-      },
-      "summary": "qemu_can_be_started_with_KVM_enabled"
-    }
-  },
-  {
-    "test": {
-      "@alias": "bsps-qemu.bsps-tools.Post-installation_logging",
-      "author": [
-        {
-          "email": "yi.zhao@windriver.com",
-          "name": "yi.zhao@windriver.com"
-        }
-      ],
-      "execution": {
-        "1": {
-          "action": "Download the poky source and set environment \n",
-          "expected_results": "The /var/log/postinstall.log should exist in the first boot. The content of this log is like below:  \n\nRunning postinst /etc/rpm-postinsts/man... \nList directory to check the output log \nbin \nboot \ndev \netc \nhome \nlib \nlost+found \nmedia \nmnt \nproc \nrun \nsbin \nsys \ntmp \nusr \nvar \nList nonexist directory to check the stderr redirection log \nls: /nonexist: No such file or directory "
-        },
-        "2": {
-          "action": "Add the following lines to a .bb file. For expample, meta/recipes-connectivity/openssh/openssh_6.2p2.bb:   \n\npkg_postinst_ontarget_${PN} () {  \n       #!/bin/sh -e  \n       if [ x\"$D\" = \"x\" ]; then  \n       echo \"List directory to check the output log\"  \n       ls /  \n       echo \"List nonexist directory to check the stderr redirection log\"  \n       ls /nonexist  \n       else  \n       exit 1  \n       fi  \n}  \n\nMake sure the feature \"debug-tweaks\" is added in conf/local.conf \n",
-          "expected_results": ""
-        },
-        "3": {
-          "action": "Add ssh-server-openssh to EXTRA_IMAGE_FEATURES in local.conf \n",
-          "expected_results": ""
-        },
-        "4": {
-          "action": "Build core-image-minimal \n",
-          "expected_results": ""
-        },
-        "5": {
-          "action": "Boot up the image and check the /var/log/postinstall.log  ",
-          "expected_results": ""
-        }
-      },
-      "summary": "Post-installation_logging"
-    }
-  },
-  {
-    "test": {
-      "@alias": "bsps-qemu.bsps-tools.Add_user_with_cleartext_type_password_during_filesystem_construction",
-      "author": [
-        {
-          "email": "ke.zou@windriver.com",
-          "name": "ke.zou@windriver.com"
-        }
-      ],
-      "execution": {
-        "1": {
-          "action": "Download the poky source and set the environment  \n\n",
-          "expected_results": "No error during image building procedure. \n"
-        },
-        "2": {
-          "action": "Add the following lines in conf/local.conf  \n\nINHERIT += \"extrausers\"  \n\nEXTRA_USERS_PARAMS = \"\\ \nuseradd -s /bin/sh -P 'tester3' tester3;\\ \n\"  \n\nThe above settings do the following things: \na. Add a user tester3 with cleartext password 'tester3' ",
-          "expected_results": "Image can boot up \n"
-        },
-        "3": {
-          "action": "Build the image\n ",
-          "expected_results": "Login with user name \"tester3\" and password \"tester3\" "
-        }
-      },
-      "summary": "Add_user_with_cleartext_type_password_during_filesystem_construction"
-    }
-  },
-  {
-    "test": {
-      "@alias": "bsps-qemu.bsps-tools.rpm_-__install_dependency_package",
-      "author": [
-        {
-          "email": "alexandru.c.georgescu@intel.com",
-          "name": "alexandru.c.georgescu@intel.com"
-        }
-      ],
-      "execution": {
-        "1": {
-          "action": "Get a not previously installed RPM package or build one on local machine, which should have run-time dependency.For example, \"mc\" (Midnight Commander, which is a visual file manager) should depend on \"ncurses-terminfo\".   \n\n$ bitbake mc  \n\n\n",
-          "expected_results": ""
-        },
-        "2": {
-          "action": "Copy the package into a system folder (for example /home/root/rpm_packages).  \n\n\n",
-          "expected_results": ""
-        },
-        "3": {
-          "action": "Run \"rpm -ivh package_name\" and check the output, for example \"rpm -ivh mc.rpm*\" should report the dependency on \"ncurses-terminfo\".\n\n\n\n",
-          "expected_results": "3 . rpm command should report message when some RPM installation depends on other packages."
-        }
-      },
-      "summary": "rpm_-__install_dependency_package"
-    }
-  },
-  {
-    "test": {
-      "@alias": "bsps-qemu.bsps-tools.Check_rpm_install/removal_log_file_size(auto)",
-      "author": [
-        {
-          "email": "alexandru.c.georgescu@intel.com",
-          "name": "alexandru.c.georgescu@intel.com"
-        }
-      ],
-      "execution": {
-        "1": {
-          "action": "Get some rpm or other kind of installation packages.  \n\n",
-          "expected_results": "Steps 1- 4 (more than 2.3) \nEach file will occupy around 10MB, and there should be some method to keep rpm log in a small size. (the size of the db of RPMs must not be taking so much space)  \nStep 5 (less than or equal to 2.3)\nThe size on /var/lib/rpm/ must keep around 30MB"
-        },
-        "2": {
-          "action": "After system is up, check the size of log file named as \"log.xxxxxx\" on  /var/lib/rpm/log  \n\n",
-          "expected_results": ""
-        },
-        "3": {
-          "action": "After several install/removal of packages, with either of the install/removal commands   (rpm/smart/zypper/dnf install/removal), check again the size of log file.  \n\n",
-          "expected_results": ""
-        },
-        "4": {
-          "action": "For packages installation, there will be some database files under /var/lib/rpm/, named as \"__db.xxx\" and there will be some log files   \nunder /var/lib/rpm/log, named as \"\"log.xxxxxx\"\".   \n\nNote: You will only see the log.xxxx on /var/lib/rpm/log mentioned above if the poky version is minor than 2.3.For poky 2.3 or major versions this has been modified and the package RPM4 does not show the logs.xxxx. if major, follow the next step.  \n\n",
-          "expected_results": ""
-        },
-        "5": {
-          "action": "Repeat steps (1 and 3)  and check the size of /var/lib/rpm/  \n\nMore info: https://bugzilla.yoctoproject.org/show_bug.cgi?id=9259",
-          "expected_results": ""
-        }
-      },
-      "summary": "Check_rpm_install/removal_log_file_size"
-    }
-  },
-  {
-    "test": {
-      "@alias": "bsps-qemu.bsps-runtime.only_one_connmand_in_background(auto)",
-      "author": [
-        {
-          "email": "alexandru.c.georgescu@intel.com",
-          "name": "alexandru.c.georgescu@intel.com"
-        }
-      ],
-      "execution": {
-        "1": {
-          "action": "Boot system",
-          "expected_results": ""
-        },
-        "2": {
-          "action": "Run \"ps aux |grep connmand\" or \"ps -ef | grep connmand\" or \"ps | grep connmand\"",
-          "expected_results": "Connmand (connection manager, used to manage internet connections)  should be shown as an active process \n\n"
-        },
-        "3": {
-          "action": "Run command \"connmand\" to try to launch to a second connmand process",
-          "expected_results": ""
-        },
-        "4": {
-          "action": "Check, with \"ps\" connmand  if a second connmand can be generated ",
-          "expected_results": "There should be only one connmand process instance in background ."
-        }
-      },
-      "summary": "only_one_connmand_in_background"
-    }
-  },
-  {
-    "test": {
-      "@alias": "bsps-qemu.bsps-runtime.X_server_can_start_up_with_runlevel_5_boot",
-      "author": [
-        {
-          "email": "alexandru.c.georgescu@intel.com",
-          "name": "alexandru.c.georgescu@intel.com"
-        }
-      ],
-      "execution": {
-        "1": {
-          "action": "boot up system with default runlevel  \n\n",
-          "expected_results": "X server can start up well and desktop display has no problem .  \n\n"
-        },
-        "2": {
-          "action": "type runlevel at command prompt",
-          "expected_results": "Output:N 5"
-        }
-      },
-      "summary": "X_server_can_start_up_with_runlevel_5_boot"
-    }
-  },
-  {
-    "test": {
-      "@alias": "bsps-qemu.bsps-runtime.check_bash_in_image",
-      "author": [
-        {
-          "email": "alexandru.c.georgescu@intel.com",
-          "name": "alexandru.c.georgescu@intel.com"
-        }
-      ],
-      "execution": {
-        "1": {
-          "action": "After system is up, check if bash command exists with command \"which bash\"",
-          "expected_results": "bash command should exist in image giving something as below  \"/bin/bash\""
-        }
-      },
-      "summary": "check_bash_in_image"
-    }
-  }
-]

meta/lib/oeqa/manual/compliance-test.json

@@ -1,76 +0,0 @@
-[
-    {
-        "test": {
-            "@alias": "compliance-test.compliance-test.stress_test_-_Genericx86-64",
-            "author": [
-                {
-                    "email": "corneliux.stoicescu@intel.com",
-                    "name": "corneliux.stoicescu@intel.com"
-                }
-            ],
-            "execution": {
-                "1": {
-                    "action": "Bootup with core-image-sato-sdk image",
-                    "expected_results": ""
-                },
-                "2": {
-                    "action": "Execute the crashme test with below command  \n\n./opt/ltp/runltp f  crashme",
-                    "expected_results": "The stress testing should not make the target crash. Check CPU usage and basic functionality of the system after the tests are over. "
-                }
-            },
-            "summary": "stress_test_-_Genericx86-64"
-        }
-    },
-     {
-    "test": {
-      "@alias": "compliance-test.compliance-test.stress_test_-_- crashme_-_-Beaglebone",
-      "author": [
-        {
-          "email": "corneliux.stoicescu@intel.com",
-          "name": "corneliux.stoicescu@intel.com"
-        }
-      ],
-      "execution": {
-        "1": {
-          "action": " Get crashme from http://people.delphiforums.com/gjc/crashme.html",
-          "expected_results": ""
-        },
-        "2": {
-          "action": "Follow the setup steps on above URL, build crashme in target",
-          "expected_results": ""
-        },
-        "3": {
-          "action": " Run crashme for 24 hours",
-          "expected_results": "Target should not crash with the program."
-        }
-      },
-      "summary": "stress_test_-_crashme_-Beaglebone"
-    }
-  },
-  {
-    "test": {
-      "@alias": "compliance-test.compliance-test.stress_test_-_ltp_-Beaglebone",
-      "author": [
-        {
-          "email": "corneliux.stoicescu@intel.com",
-          "name": "corneliux.stoicescu@intel.com"
-        }
-      ],
-      "execution": {
-        "1": {
-          "action": "Build LTP with toolchain or in sdk image",
-          "expected_results": ""
-        },
-        "2": {
-          "action": "Copy LTP folder into target, for example, /opt/ltp. Modify script,  testscripts/ltpstress.sh, set Iostat=1, NO_NETWORK=1",
-          "expected_results": ""
-        },
-        "3": {
-          "action": "cd testscripts/ && ./ltpstress.sh",
-          "expected_results": "This stress case will run for 24 hours Check the result\ntarget should not crash with the program "
-        }
-      },
-      "summary": "stress_test_-_-ltp_-Beaglebone"
-    }
-  }
-]

meta/lib/oeqa/manual/oe-core.json

@@ -1,80 +1,4 @@
 [
-  {
-    "test": {
-      "@alias": "oe-core.scripts.Crosstap_script_check",
-      "author": [
-        {
-          "email": "alexandru.c.georgescu@intel.com",
-          "name": "alexandru.c.georgescu@intel.com"
-        }
-      ],
-      "execution": {
-        "1": {
-          "action": "Create the trace_open.stp script as follows in the host machine:  \n\n\nprobe syscall.open     \n\n{  \n\n\n        printf (\"%s(%d) open (%s)\\n\", execname(), pid(), argstr)  \n\n}  \n\n\n\nif the above failed, then create the below instead.  \n\nprobe syscall.open \n{ \n  printf (\"%s(%d) open\\n\", execname(), pid()) \n\n}  \n \n",
-          "expected_results": ""
-        },
-        "2": {
-          "action": "Add 'tools-profile' and 'ssh-server-openssh' to EXTRA_IMAGE_FEATURES in local.conf \n\n\n",
-          "expected_results": ""
-        },
-        "3": {
-          "action": "Build a core-image-minimal image, build systemtap-native. Start the image under qemu. \n\n",
-          "expected_results": ""
-        },
-        "4": {
-          "action": "Make sure that the ssh service is started on the Qemu machine. \n\n",
-          "expected_results": ""
-        },
-        "5": {
-          "action": "From the host machine poky build_dir, run \"crosstap root@192.168.7.2 trace_open.stp\".",
-          "expected_results": ""
-        },
-        "6": {
-          "action": "In QEMU, try to open some applications, such as open a terminal, input some command,  \n\n",
-          "expected_results": ""
-        },
-        "7": {
-          "action": "Check the host machine, \"crosstap\" has related output. \n\n\n\nNOTE:  Do not build the kernel from shared state(sstate-cache) for this to work.",
-          "expected_results": "The script should successfully connect to the qemu machine and there \nshould be presented a list of services(pid, process name) which run on \nthe qemu machine. "
-        }
-      },
-      "summary": "Crosstap_script_check"
-    }
-  },
-  {
-    "test": {
-      "@alias": "oe-core.scripts.List_all_the_PACKAGECONFIG's_flags",
-      "author": [
-        {
-          "email": "yi.zhao@windriver.com",
-          "name": "yi.zhao@windriver.com"
-        }
-      ],
-      "execution": {
-        "1": {
-          "action": " Download the poky source and setup the environment. ",
-          "expected_results": ""
-        },
-        "2": {
-          "action": "Run \"../scripts/contrib/list-packageconfig-flags.py\" ",
-          "expected_results": "In step 2, will list available pkgs which have PACKAGECONFIG flags:  \nPACKAGE NAME (or RECIPE NAME)           PACKAGECONFIG FLAGS  \n==============================================================  \nalsa-tools-1.0.26.1                                         defaultval gtk+  \navahi-ui-0.6.31                                                defaultval python  \nbluez4-4.101                                                alsa defaultval pie  \n"
-        },
-        "3": {
-          "action": "Run \"../scripts/contrib/list-packageconfig-flags.py -f\" ",
-          "expected_results": "In step 3, will list available PACKAGECONFIG flags and all affected pkgs  \nPACKAGECONFIG FLAG       PACKAGE NAMES (or RECIPE NAMES)  \n====================================  \n3g                             connman-1.16  \n        \navahi                        cups-1.6.3  pulseaudio-4.0  \nbeecrypt                   rpm-5.4.9  rpm-native-5.4.9  \n"
-        },
-        "4": {
-          "action": "Run \"../scripts/contrib/list-packageconfig-flags.py -a\" ",
-          "expected_results": "In step 4, will list all pkgs and PACKAGECONFIG information:  \n==================================================  \ngtk+-2.24.18  \n/home/jiahongxu/yocto/poky/meta/recipes-gnome/gtk+/gtk+_2.24.18.bb  \nPACKAGECONFIG x11  \nPACKAGECONFIG[x11] --with-x=yes --with-gdktarget=x11,--with-x=no,${X11DEPENDS}  \nxf86-video-intel-2.21.9  \n/home/jiahongxu/yocto/poky/meta/recipes-graphics/xorg-driver/xf86-video-intel_2.21.9.bb  \nPACKAGECONFIG None  \nPACKAGECONFIG[xvmc] --enable-xvmc,--disable-xvmc,libxvmc  \nPACKAGECONFIG[sna] --enable-sna,--disable-sna  \n"
-        },
-        "5": {
-          "action": "Run \"../scripts/contrib/list-packageconfig-flags.py -p\"   ",
-          "expected_results": "In step 5, will list pkgs with preferred version:  \nPACKAGE NAME (or RECIPE NAME)              PACKAGECONFIG FLAGS  \n===================================================  \nalsa-tools-1.0.26.1                                           defaultval gtk+  \navahi-ui-0.6.31                                                   defaultval python  \nbluez4-4.101                                                       alsa defaultval pie  \nbluez5-5.7                                                            alsa defaultval obex-profiles  \n\n\n\n "
-        }
-      },
-      "summary": "List_all_the_PACKAGECONFIG's_flags"
-    }
-  },
   {
     "test": {
       "@alias": "oe-core.bitbake.Test_bitbake_menuconfig",
@@ -231,4 +155,4 @@
       "summary": "test_bitbake_sane_error_for_invalid_layer"
     }
   }
-]
+]

meta/lib/oeqa/runtime/cases/boot.py

@@ -0,0 +1,33 @@
+#
+# SPDX-License-Identifier: MIT
+#
+
+from subprocess import Popen, PIPE
+import time
+
+from oeqa.runtime.case import OERuntimeTestCase
+from oeqa.core.decorator.depends import OETestDepends
+from oeqa.core.decorator.oetimeout import OETimeout
+from oeqa.core.decorator.data import skipIfQemu
+
+class BootTest(OERuntimeTestCase):
+
+    @OETimeout(120)
+    @skipIfQemu('qemuall', 'Test only runs on real hardware')
+    @OETestDepends(['ssh.SSHTest.test_ssh'])
+    def test_reboot(self):
+        output = ''
+        count = 0
+        (status, output) = self.target.run('reboot -h')
+        while count < 5:
+            time.sleep(5)
+            cmd = 'ping -c 1 %s' % self.target.ip
+            proc = Popen(cmd, shell=True, stdout=PIPE)
+            output += proc.communicate()[0].decode('utf-8')
+            if proc.poll() == 0:
+                count += 1
+            else:
+                count = 0
+        msg = ('Expected 5 consecutive, got %d.\n'
+               'ping output is:\n%s' % (count,output))
+        self.assertEqual(count, 5, msg = msg)

meta/lib/oeqa/runtime/cases/df.py

@@ -11,7 +11,7 @@ class DfTest(OERuntimeTestCase):
     @OETestDepends(['ssh.SSHTest.test_ssh'])
     @OEHasPackage(['coreutils', 'busybox'])
     def test_df(self):
-        cmd = "df / | sed -n '2p' | awk '{print $4}'"
+        cmd = "df -P / | sed -n '2p' | awk '{print $4}'"
         (status,output) = self.target.run(cmd)
         msg = 'Not enough space on image. Current size is %s' % output
         self.assertTrue(int(output)>5120, msg=msg)

meta/lib/oeqa/runtime/cases/ltp_stress.py

@@ -0,0 +1,98 @@
+# LTP Stress runtime
+#
+# Copyright (c) 2019 MontaVista Software, LLC
+#
+# SPDX-License-Identifier: MIT
+#
+
+import time
+import datetime
+import pprint
+
+from oeqa.runtime.case import OERuntimeTestCase
+from oeqa.core.decorator.depends import OETestDepends
+from oeqa.runtime.decorator.package import OEHasPackage
+from oeqa.core.decorator.data import skipIfQemu
+from oeqa.utils.logparser import LtpParser
+
+class LtpStressBase(OERuntimeTestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        cls.ltp_startup()
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.ltp_finishup()
+
+    @classmethod
+    def ltp_startup(cls):
+        cls.sections = {}
+        cls.failmsg = ""
+        test_log_dir = os.path.join(cls.td.get('WORKDIR', ''), 'testimage')
+        timestamp = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
+
+        cls.ltptest_log_dir_link = os.path.join(test_log_dir, 'ltpstress_log')
+        cls.ltptest_log_dir = '%s.%s' % (cls.ltptest_log_dir_link, timestamp)
+        os.makedirs(cls.ltptest_log_dir)
+
+        cls.tc.target.run("mkdir -p /opt/ltp/results")
+
+        if not hasattr(cls.tc, "extraresults"):
+            cls.tc.extraresults = {}
+        cls.extras = cls.tc.extraresults
+        cls.extras['ltpstressresult.rawlogs'] = {'log': ""}
+
+ 
+    @classmethod
+    def ltp_finishup(cls):
+        cls.extras['ltpstressresult.sections'] =  cls.sections
+
+        # update symlink to ltp_log
+        if os.path.exists(cls.ltptest_log_dir_link):
+            os.remove(cls.ltptest_log_dir_link)
+
+        os.symlink(os.path.basename(cls.ltptest_log_dir), cls.ltptest_log_dir_link)
+
+        if cls.failmsg:
+            cls.fail(cls.failmsg)
+
+class LtpStressTest(LtpStressBase):
+
+    def runltp(self, stress_group):
+            cmd = '/opt/ltp/runltp -f %s -p -q 2>@1 | tee /opt/ltp/results/%s' % (stress_group, stress_group)
+            starttime = time.time()
+            (status, output) = self.target.run(cmd)
+            endtime = time.time()
+            with open(os.path.join(self.ltptest_log_dir, "%s" % stress_group), 'w') as f:
+                f.write(output)
+
+            self.extras['ltpstressresult.rawlogs']['log'] = self.extras['ltpstressresult.rawlogs']['log'] + output
+
+            parser = LtpParser()
+            results, sections  = parser.parse(os.path.join(self.ltptest_log_dir, "%s" % stress_group))
+
+            runtime = int(endtime-starttime)
+            sections['duration'] = runtime
+            self.sections[stress_group] =  sections
+ 
+            failed_tests = {}
+            for test in results:
+                result = results[test]
+                testname = ("ltpstressresult." + stress_group + "." + test)
+                self.extras[testname] = {'status': result}
+                if result == 'FAILED':
+                    failed_tests[stress_group] = test 
+
+            if failed_tests:
+                self.failmsg = self.failmsg + "Failed ptests:\n%s" % pprint.pformat(failed_tests)
+
+    # LTP stress runtime tests
+    #
+    @skipIfQemu('qemuall', 'Test only runs on real hardware')
+
+    @OETestDepends(['ssh.SSHTest.test_ssh'])
+    @OEHasPackage(["ltp"])
+    def test_ltp_stress(self):
+        self.tc.target.run("sed -i -r 's/^fork12.*//' /opt/ltp/runtest/crashme")
+        self.runltp('crashme')

meta/lib/oeqa/runtime/cases/opkg.py

@@ -5,7 +5,7 @@
 import os
 from oeqa.utils.httpserver import HTTPService
 from oeqa.runtime.case import OERuntimeTestCase
-from oeqa.core.decorator.data import skipIfNotDataVar, skipIfNotFeature
+from oeqa.core.decorator.data import skipIfNotDataVar, skipIfNotFeature, skipIfFeature
 from oeqa.runtime.decorator.package import OEHasPackage
 
 class OpkgTest(OERuntimeTestCase):
@@ -45,6 +45,8 @@ class OpkgRepoTest(OpkgTest):
                       'Test requires package-management to be in IMAGE_FEATURES')
     @skipIfNotDataVar('IMAGE_PKGTYPE', 'ipk',
                       'IPK is not the primary package manager')
+    @skipIfFeature('read-only-rootfs',
+                   'Test does not work with read-only-rootfs in IMAGE_FEATURES')
     @OEHasPackage(['opkg'])
     def test_opkg_install_from_repo(self):
         self.setup_source_config_for_package_install()