classes: Replace "if test" file tests with POSIX file tests

In entire meta/classes/ directory, replace shell tests of the form
"if test -? ..." with POSIX tests of the form "if [ -? ...

(From OE-Core rev: 78928016f4cf38cf6751cb089200bf950d07ae93)

Signed-off-by: Robert P. J. Day <rpjday@crashcourse.ca>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

bitbake/bin/bitbake

@@ -38,7 +38,7 @@ from bb.main import bitbake_main, BitBakeConfigParameters, BBMainException
 if sys.getfilesystemencoding() != "utf-8":
     sys.exit("Please use a locale setting which supports utf-8.\nPython can't change the filesystem locale after loading so we need a utf-8 when python starts or things won't work.")
 
-__version__ = "1.33.0"
+__version__ = "1.33.2"
 
 if __name__ == "__main__":
     if __version__ != bb.__version__:

bitbake/bin/bitbake-diffsigs

@@ -94,6 +94,8 @@ def find_compare_task(bbhandler, pn, taskname):
             return recout
 
         # Recurse into signature comparison
+        logger.debug("Signature file (previous): %s" % latestfiles[0])
+        logger.debug("Signature file (latest): %s" % latestfiles[1])
         output = bb.siggen.compare_sigfiles(latestfiles[0], latestfiles[1], recursecb)
         if output:
             print('\n'.join(output))
@@ -108,12 +110,19 @@ parser = optparse.OptionParser(
   %prog sigdatafile1 sigdatafile2
   %prog sigdatafile1""")
 
+parser.add_option("-D", "--debug",
+        help = "enable debug",
+        action = "store_true", dest="debug", default = False)
+
 parser.add_option("-t", "--task",
         help = "find the signature data files for last two runs of the specified task and compare them",
         action="store", dest="taskargs", nargs=2, metavar='recipename taskname')
 
 options, args = parser.parse_args(sys.argv)
 
+if options.debug:
+    logger.setLevel(logging.DEBUG)
+
 if options.taskargs:
     with bb.tinfoil.Tinfoil() as tinfoil:
         tinfoil.prepare(config_only=True)

bitbake/bin/bitbake-dumpsig

@@ -27,6 +27,7 @@ import pickle
 
 sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
 
+import bb.tinfoil
 import bb.siggen
 
 def logger_create(name, output=sys.stderr):
@@ -42,24 +43,62 @@ def logger_create(name, output=sys.stderr):
 
 logger = logger_create('bitbake-dumpsig')
 
+def find_siginfo_task(bbhandler, pn, taskname):
+    """ Find the most recent signature file for the specified PN/task """
+
+    if not hasattr(bb.siggen, 'find_siginfo'):
+        logger.error('Metadata does not support finding signature data files')
+        sys.exit(1)
+
+    if not taskname.startswith('do_'):
+        taskname = 'do_%s' % taskname
+
+    filedates = bb.siggen.find_siginfo(pn, taskname, None, bbhandler.config_data)
+    latestfiles = sorted(filedates.keys(), key=lambda f: filedates[f])[-1:]
+    if not latestfiles:
+        logger.error('No sigdata files found matching %s %s' % (pn, taskname))
+        sys.exit(1)
+
+    return latestfiles[0]
+
 parser = optparse.OptionParser(
     description = "Dumps siginfo/sigdata files written out by BitBake",
     usage = """
+  %prog -t recipename taskname
   %prog sigdatafile""")
 
+parser.add_option("-D", "--debug",
+        help = "enable debug",
+        action = "store_true", dest="debug", default = False)
+
+parser.add_option("-t", "--task",
+        help = "find the signature data file for the specified task",
+        action="store", dest="taskargs", nargs=2, metavar='recipename taskname')
+
 options, args = parser.parse_args(sys.argv)
 
-if len(args) == 1:
-    parser.print_help()
-else:
-    try:
-        output = bb.siggen.dump_sigfile(args[1])
-    except IOError as e:
-        logger.error(str(e))
-        sys.exit(1)
-    except (pickle.UnpicklingError, EOFError):
-        logger.error('Invalid signature data - ensure you are specifying a sigdata/siginfo file')
-        sys.exit(1)
+if options.debug:
+    logger.setLevel(logging.DEBUG)
 
-    if output:
-        print('\n'.join(output))
+if options.taskargs:
+    tinfoil = bb.tinfoil.Tinfoil()
+    tinfoil.prepare(config_only = True)
+    file = find_siginfo_task(tinfoil, options.taskargs[0], options.taskargs[1])
+    logger.debug("Signature file: %s" % file)
+elif len(args) == 1:
+    parser.print_help()
+    sys.exit(0)
+else:
+    file = args[1]
+
+try:
+    output = bb.siggen.dump_sigfile(file)
+except IOError as e:
+    logger.error(str(e))
+    sys.exit(1)
+except (pickle.UnpicklingError, EOFError):
+    logger.error('Invalid signature data - ensure you are specifying a sigdata/siginfo file')
+    sys.exit(1)
+
+if output:
+    print('\n'.join(output))

bitbake/bin/bitbake-layers

@@ -31,15 +31,6 @@ sys.path[0:0] = [os.path.join(topdir, 'lib')]
 
 import bb.tinfoil
 
-
-def tinfoil_init(parserecipes):
-    import bb.tinfoil
-    tinfoil = bb.tinfoil.Tinfoil(tracking=True)
-    tinfoil.prepare(not parserecipes)
-    tinfoil.logger.setLevel(logger.getEffectiveLevel())
-    return tinfoil
-
-
 def logger_create(name, output=sys.stderr):
     logger = logging.getLogger(name)
     loggerhandler = logging.StreamHandler(output)
@@ -86,8 +77,10 @@ def main():
     logger_setup_color(logger, global_args.color)
 
     plugins = []
-    tinfoil = tinfoil_init(False)
+    tinfoil = bb.tinfoil.Tinfoil(tracking=True)
+    tinfoil.logger.setLevel(logger.getEffectiveLevel())
     try:
+        tinfoil.prepare(True)
         for path in ([topdir] +
                     tinfoil.config_data.getVar('BBPATH').split(':')):
             pluginpath = os.path.join(path, 'lib', 'bblayers')

bitbake/bin/toaster

@@ -151,15 +151,7 @@ fi
 
 unset OE_ROOT
 
-# this defines the dir toaster will use for
-# 1) clones of layers (in _toaster_clones )
-# 2) the build dir (in build)
-# 3) the sqlite db if that is being used.
-# 4) pid's we need to clean up on exit/shutdown
-# note: for future. in order to make this an arbitrary directory, we need to
-# make sure that the toaster.sqlite file doesn't default to `pwd` like it currently does.
-export BB_ENV_EXTRAWHITE="$BB_ENV_EXTRAWHITE TOASTER_DIR"
-export TOASTER_DIR=`pwd`
+
 
 WEBSERVER=1
 ADDR_PORT="localhost:8000"
@@ -215,10 +207,8 @@ fi
 # 2) the build dir (in build)
 # 3) the sqlite db if that is being used.
 # 4) pid's we need to clean up on exit/shutdown
-# note: for future. in order to make this an arbitrary directory, we need to
-# make sure that the toaster.sqlite file doesn't default to `pwd`
-# like it currently does.
 export TOASTER_DIR=`dirname $BUILDDIR`
+export BB_ENV_EXTRAWHITE="$BB_ENV_EXTRAWHITE TOASTER_DIR"
 
 # Determine the action. If specified by arguments, fine, if not, toggle it
 if [ "$CMD" = "start" ] ; then
@@ -255,7 +245,10 @@ case $CMD in
             return 4
         fi
         export BITBAKE_UI='toasterui'
-        $MANAGE runbuilds &
+        $MANAGE runbuilds \
+           </dev/null >>${BUILDDIR}/toaster_runbuilds.log 2>&1 \
+           & echo $! >${BUILDDIR}/.runbuilds.pid
+
         # set fail safe stop system on terminal exit
         trap stop_system SIGHUP
         echo "Successful ${CMD}."

bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml

@@ -385,7 +385,8 @@
                 The supported parameters are as follows:
                 <itemizedlist>
                     <listitem><para><emphasis>"method":</emphasis>
-                        The protocol over which to communicate with the CVS server.
+                        The protocol over which to communicate with the CVS
+                        server.
                         By default, this protocol is "pserver".
                         If "method" is set to "ext", BitBake examines the
                         "rsh" parameter and sets <filename>CVS_RSH</filename>.
@@ -469,25 +470,29 @@
                         You can think of this parameter as the top-level
                         directory of the repository data you want.
                         </para></listitem>
+                    <listitem><para><emphasis>"path_spec":</emphasis>
+                        A specific directory in which to checkout the
+                        specified svn module.
+                        </para></listitem>
                     <listitem><para><emphasis>"protocol":</emphasis>
                         The protocol to use, which defaults to "svn".
-                        Other options are "svn+ssh" and "rsh".
-                        For "rsh", the "rsh" parameter is also used.
+                        If "protocol" is set to "svn+ssh", the "ssh"
+                        parameter is also used.
                         </para></listitem>
                     <listitem><para><emphasis>"rev":</emphasis>
                         The revision of the source code to checkout.
                         </para></listitem>
-                    <listitem><para><emphasis>"date":</emphasis>
-                        The date of the source code to checkout.
-                        Specific revisions are generally much safer to checkout
-                        rather than by date as they do not involve timezones
-                        (e.g. they are much more deterministic).
-                        </para></listitem>
                     <listitem><para><emphasis>"scmdata":</emphasis>
                         Causes the “.svn” directories to be available during
                         compile-time when set to "keep".
                         By default, these directories are removed.
                         </para></listitem>
+                    <listitem><para><emphasis>"ssh":</emphasis>
+                        An optional parameter used when "protocol" is set
+                        to "svn+ssh".
+                        You can use this parameter to specify the ssh
+                        program used by svn.
+                        </para></listitem>
                     <listitem><para><emphasis>"transportuser":</emphasis>
                         When required, sets the username for the transport.
                         By default, this parameter is empty.
@@ -496,10 +501,11 @@
                         command.
                         </para></listitem>
                 </itemizedlist>
-                Following are two examples using svn:
+                Following are three examples using svn:
                 <literallayout class='monospaced'>
-     SRC_URI = "svn://svn.oe.handhelds.org/svn;module=vip;proto=http;rev=667"
-     SRC_URI = "svn://svn.oe.handhelds.org/svn/;module=opie;proto=svn+ssh;date=20060126"
+     SRC_URI = "svn://myrepos/proj1;module=vip;protocol=http;rev=667"
+     SRC_URI = "svn://myrepos/proj1;module=opie;protocol=svn+ssh"
+     SRC_URI = "svn://myrepos/proj1;module=trunk;protocol=http;path_spec=${MY_DIR}/proj1"
                 </literallayout>
             </para>
         </section>

bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.xml

@@ -529,7 +529,7 @@
        -l DEBUG_DOMAINS, --log-domains=DEBUG_DOMAINS
                              Show debug logging for the specified logging domains
        -P, --profile         Profile the command and save reports.
-       -u UI, --ui=UI        The user interface to use (depexp, goggle, knotty or
+       -u UI, --ui=UI        The user interface to use (taskexp, knotty or
                              ncurses - default knotty).
        -t SERVERTYPE, --servertype=SERVERTYPE
                              Choose which server type to use (process or xmlrpc -
@@ -666,21 +666,21 @@
                 </para>
 
                 <para>
-                    When you generate a dependency graph, BitBake writes four files
+                    When you generate a dependency graph, BitBake writes three files
                     to the current working directory:
                     <itemizedlist>
-                        <listitem><para><emphasis><filename>package-depends.dot</filename>:</emphasis>
-                            Shows BitBake's knowledge of dependencies between
-                            runtime targets.
+                        <listitem><para>
+                            <emphasis><filename>recipe-depends.dot</filename>:</emphasis>
+                            Shows dependencies between recipes (i.e. a collapsed version of
+                            <filename>task-depends.dot</filename>).
                             </para></listitem>
-                        <listitem><para><emphasis><filename>pn-depends.dot</filename>:</emphasis>
-                            Shows dependencies between build-time targets
-                            (i.e. recipes).
-                            </para></listitem>
-                        <listitem><para><emphasis><filename>task-depends.dot</filename>:</emphasis>
+                        <listitem><para>
+                            <emphasis><filename>task-depends.dot</filename>:</emphasis>
                             Shows dependencies between tasks.
+                            These dependencies match BitBake's internal task execution list.
                             </para></listitem>
-                        <listitem><para><emphasis><filename>pn-buildlist</filename>:</emphasis>
+                        <listitem><para>
+                            <emphasis><filename>pn-buildlist</filename>:</emphasis>
                             Shows a simple list of targets that are to be built.
                             </para></listitem>
                     </itemizedlist>

bitbake/doc/bitbake.1

@@ -105,7 +105,7 @@ Show debug logging for the specified logging domains
 profile the command and print a report
 .TP
 .B \-uUI, \-\-ui=UI
-User interface to use. Currently, hob, depexp, goggle or ncurses can be specified as UI.
+User interface to use. Currently, knotty, taskexp or ncurses can be specified as UI.
 .TP
 .B \-tSERVERTYPE, \-\-servertype=SERVERTYPE
 Choose which server to use, none, process or xmlrpc.

bitbake/lib/bb/__init__.py

@@ -21,7 +21,7 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-__version__ = "1.33.0"
+__version__ = "1.33.2"
 
 import sys
 if sys.version_info < (3, 4, 0):

bitbake/lib/bb/build.py

@@ -878,3 +878,30 @@ def preceedtask(task, with_recrdeptasks, d):
         if recrdeptask:
             preceed.update(recrdeptask.split())
     return preceed
+
+def tasksbetween(task_start, task_end, d):
+    """
+    Return the list of tasks between two tasks in the current recipe,
+    where task_start is to start at and task_end is the task to end at
+    (and task_end has a dependency chain back to task_start).
+    """
+    outtasks = []
+    tasks = list(filter(lambda k: d.getVarFlag(k, "task"), d.keys()))
+    def follow_chain(task, endtask, chain=None):
+        if not chain:
+            chain = []
+        chain.append(task)
+        for othertask in tasks:
+            if othertask == task:
+                continue
+            if task == endtask:
+                for ctask in chain:
+                    if ctask not in outtasks:
+                        outtasks.append(ctask)
+            else:
+                deps = d.getVarFlag(othertask, 'deps', False)
+                if task in deps:
+                    follow_chain(othertask, endtask, chain)
+        chain.pop()
+    follow_chain(task_start, task_end)
+    return outtasks

bitbake/lib/bb/command.py

@@ -457,16 +457,22 @@ class CommandsSync:
         dsindex = params[0]
         name = params[1]
         datastore = command.remotedatastores[dsindex]
-        value = datastore._findVar(name)
+        value, overridedata = datastore._findVar(name)
 
         if value:
             content = value.get('_content', None)
             if isinstance(content, bb.data_smart.DataSmart):
                 # Value is a datastore (e.g. BB_ORIGENV) - need to handle this carefully
                 idx = command.remotedatastores.check_store(content, True)
-                return {'_content': DataStoreConnectionHandle(idx), '_connector_origtype': 'DataStoreConnectionHandle'}
+                return {'_content': DataStoreConnectionHandle(idx),
+                        '_connector_origtype': 'DataStoreConnectionHandle',
+                        '_connector_overrides': overridedata}
             elif isinstance(content, set):
-                return {'_content': list(content), '_connector_origtype': 'set'}
+                return {'_content': list(content),
+                        '_connector_origtype': 'set',
+                        '_connector_overrides': overridedata}
+            else:
+                value['_connector_overrides'] = overridedata
         return value
     dataStoreConnectorFindVar.readonly = True
 
@@ -499,6 +505,31 @@ class CommandsSync:
             raise CommandError('dataStoreConnectorRelease: invalid index %d' % dsindex)
         command.remotedatastores.release(dsindex)
 
+    def dataStoreConnectorSetVarFlag(self, command, params):
+        dsindex = params[0]
+        name = params[1]
+        flag = params[2]
+        value = params[3]
+        datastore = command.remotedatastores[dsindex]
+        datastore.setVarFlag(name, flag, value)
+
+    def dataStoreConnectorDelVar(self, command, params):
+        dsindex = params[0]
+        name = params[1]
+        datastore = command.remotedatastores[dsindex]
+        if len(params) > 2:
+            flag = params[2]
+            datastore.delVarFlag(name, flag)
+        else:
+            datastore.delVar(name)
+
+    def dataStoreConnectorRenameVar(self, command, params):
+        dsindex = params[0]
+        name = params[1]
+        newname = params[2]
+        datastore = command.remotedatastores[dsindex]
+        datastore.renameVar(name, newname)
+
     def parseRecipeFile(self, command, params):
         """
         Parse the specified recipe file (with or without bbappends)

bitbake/lib/bb/cooker.py

@@ -642,14 +642,13 @@ class BBCooker:
             logger.plain(env.getvalue())
 
         # emit variables and shell functions
-        data.update_data(envdata)
         with closing(StringIO()) as env:
             data.emit_env(env, envdata, True)
             logger.plain(env.getvalue())
 
         # emit the metadata which isnt valid shell
         data.expandKeys(envdata)
-        for e in envdata.keys():
+        for e in sorted(envdata.keys()):
             if envdata.getVarFlag(e, 'func', False) and envdata.getVarFlag(e, 'python', False):
                 logger.plain("\npython %s () {\n%s}\n", e, envdata.getVar(e, False))
 
@@ -703,7 +702,6 @@ class BBCooker:
         for mc in self.multiconfigs:
             taskdata[mc] = bb.taskdata.TaskData(abort, skiplist=self.skiplist, allowincomplete=allowincomplete)
             localdata[mc] = data.createCopy(self.databuilder.mcdata[mc])
-            bb.data.update_data(localdata[mc])
             bb.data.expandKeys(localdata[mc])
 
         current = 0
@@ -764,7 +762,7 @@ class BBCooker:
     @staticmethod
     def add_mc_prefix(mc, pn):
         if mc:
-            return "multiconfig:%s.%s" % (mc, pn)
+            return "multiconfig:%s:%s" % (mc, pn)
         return pn
 
     def buildDependTree(self, rq, taskdata):
@@ -949,62 +947,54 @@ class BBCooker:
 
         depgraph = self.generateTaskDepTreeData(pkgs_to_build, task)
 
-        # Prints a flattened form of package-depends below where subpackages of a package are merged into the main pn
-        depends_file = open('pn-depends.dot', 'w' )
-        buildlist_file = open('pn-buildlist', 'w' )
-        print("digraph depends {", file=depends_file)
-        for pn in depgraph["pn"]:
-            fn = depgraph["pn"][pn]["filename"]
-            version = depgraph["pn"][pn]["version"]
-            print('"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn), file=depends_file)
-            print("%s" % pn, file=buildlist_file)
-        buildlist_file.close()
+        with open('pn-buildlist', 'w') as f:
+            for pn in depgraph["pn"]:
+                f.write(pn + "\n")
         logger.info("PN build list saved to 'pn-buildlist'")
-        for pn in depgraph["depends"]:
-            for depend in depgraph["depends"][pn]:
-                print('"%s" -> "%s" [style=solid]' % (pn, depend), file=depends_file)
-        for pn in depgraph["rdepends-pn"]:
-            for rdepend in depgraph["rdepends-pn"][pn]:
-                print('"%s" -> "%s" [style=dashed]' % (pn, rdepend), file=depends_file)
-        print("}", file=depends_file)
-        depends_file.close()
-        logger.info("PN dependencies saved to 'pn-depends.dot'")
 
-        depends_file = open('package-depends.dot', 'w' )
-        print("digraph depends {", file=depends_file)
-        for package in depgraph["packages"]:
-            pn = depgraph["packages"][package]["pn"]
-            fn = depgraph["packages"][package]["filename"]
-            version = depgraph["packages"][package]["version"]
-            if package == pn:
-                print('"%s" [label="%s %s\\n%s"]' % (pn, pn, version, fn), file=depends_file)
-            else:
-                print('"%s" [label="%s(%s) %s\\n%s"]' % (package, package, pn, version, fn), file=depends_file)
-            for depend in depgraph["depends"][pn]:
-                print('"%s" -> "%s" [style=solid]' % (package, depend), file=depends_file)
-        for package in depgraph["rdepends-pkg"]:
-            for rdepend in depgraph["rdepends-pkg"][package]:
-                print('"%s" -> "%s" [style=dashed]' % (package, rdepend), file=depends_file)
-        for package in depgraph["rrecs-pkg"]:
-            for rdepend in depgraph["rrecs-pkg"][package]:
-                print('"%s" -> "%s" [style=dotted]' % (package, rdepend), file=depends_file)
-        print("}", file=depends_file)
-        depends_file.close()
-        logger.info("Package dependencies saved to 'package-depends.dot'")
+        # Remove old format output files to ensure no confusion with stale data
+        try:
+            os.unlink('pn-depends.dot')
+        except FileNotFoundError:
+            pass
+        try:
+            os.unlink('package-depends.dot')
+        except FileNotFoundError:
+            pass
 
-        tdepends_file = open('task-depends.dot', 'w' )
-        print("digraph depends {", file=tdepends_file)
-        for task in depgraph["tdepends"]:
-            (pn, taskname) = task.rsplit(".", 1)
-            fn = depgraph["pn"][pn]["filename"]
-            version = depgraph["pn"][pn]["version"]
-            print('"%s.%s" [label="%s %s\\n%s\\n%s"]' % (pn, taskname, pn, taskname, version, fn), file=tdepends_file)
-            for dep in depgraph["tdepends"][task]:
-                print('"%s" -> "%s"' % (task, dep), file=tdepends_file)
-        print("}", file=tdepends_file)
-        tdepends_file.close()
+        with open('task-depends.dot', 'w') as f:
+            f.write("digraph depends {\n")
+            for task in depgraph["tdepends"]:
+                (pn, taskname) = task.rsplit(".", 1)
+                fn = depgraph["pn"][pn]["filename"]
+                version = depgraph["pn"][pn]["version"]
+                f.write('"%s.%s" [label="%s %s\\n%s\\n%s"]\n' % (pn, taskname, pn, taskname, version, fn))
+                for dep in depgraph["tdepends"][task]:
+                    f.write('"%s" -> "%s"\n' % (task, dep))
+            f.write("}\n")
         logger.info("Task dependencies saved to 'task-depends.dot'")
 
+        with open('recipe-depends.dot', 'w') as f:
+            f.write("digraph depends {\n")
+            pndeps = {}
+            for task in depgraph["tdepends"]:
+                (pn, taskname) = task.rsplit(".", 1)
+                if pn not in pndeps:
+                    pndeps[pn] = set()
+                for dep in depgraph["tdepends"][task]:
+                    (deppn, deptaskname) = dep.rsplit(".", 1)
+                    pndeps[pn].add(deppn)
+            for pn in pndeps:
+                fn = depgraph["pn"][pn]["filename"]
+                version = depgraph["pn"][pn]["version"]
+                f.write('"%s" [label="%s\\n%s\\n%s"]\n' % (pn, pn, version, fn))
+                for dep in pndeps[pn]:
+                    if dep == pn:
+                        continue
+                    f.write('"%s" -> "%s"\n' % (pn, dep))
+            f.write("}\n")
+        logger.info("Flatened recipe dependencies saved to 'recipe-depends.dot'")
+
     def show_appends_with_no_recipes(self):
         # Determine which bbappends haven't been applied
 
@@ -1035,7 +1025,6 @@ class BBCooker:
 
         for mc in self.multiconfigs:
             localdata = data.createCopy(self.databuilder.mcdata[mc])
-            bb.data.update_data(localdata)
             bb.data.expandKeys(localdata)
 
             # Handle PREFERRED_PROVIDERS
@@ -1641,7 +1630,8 @@ class BBCooker:
         if self.state != state.parsing and not self.parsecache_valid:
             self.parseConfiguration ()
             if CookerFeatures.SEND_SANITYEVENTS in self.featureset:
-                bb.event.fire(bb.event.SanityCheck(False), self.data)
+                for mc in self.multiconfigs:
+                    bb.event.fire(bb.event.SanityCheck(False), self.databuilder.mcdata[mc])
 
             for mc in self.multiconfigs:
                 ignore = self.databuilder.mcdata[mc].getVar("ASSUME_PROVIDED") or ""
@@ -1910,6 +1900,11 @@ class CookerCollectFiles(object):
             # that do not compile
             bbmasks = []
             for mask in bbmask.split():
+                # When constructing an older style single regex, it's possible for BBMASK
+                # to end up beginning with '|', which matches and masks _everything_.
+                if mask.startswith("|"):
+                    collectlog.warn("BBMASK contains regular expression beginning with '|', fixing: %s" % mask)
+                    mask = mask[1:]
                 try:
                     re.compile(mask)
                     bbmasks.append(mask)

bitbake/lib/bb/cookerdata.py

@@ -148,6 +148,7 @@ class CookerConfiguration(object):
         self.writeeventlog = False
         self.server_only = False
         self.limited_deps = False
+        self.runall = None
 
         self.env = {}
 

bitbake/lib/bb/data.py

@@ -196,7 +196,7 @@ def emit_env(o=sys.__stdout__, d = init(), all=False):
     keys = sorted((key for key in d.keys() if not key.startswith("__")), key=isfunc)
     grouped = groupby(keys, isfunc)
     for isfunc, keys in grouped:
-        for key in keys:
+        for key in sorted(keys):
             emit_var(key, o, d, all and not isfunc) and o.write('\n')
 
 def exported_keys(d):
@@ -220,7 +220,7 @@ def emit_func(func, o=sys.__stdout__, d = init()):
     """Emits all items in the data store in a format such that it can be sourced by a shell."""
 
     keys = (key for key in d.keys() if not key.startswith("__") and not d.getVarFlag(key, "func", False))
-    for key in keys:
+    for key in sorted(keys):
         emit_var(key, o, d, False)
 
     o.write('\n')

bitbake/lib/bb/data_smart.py

@@ -473,7 +473,7 @@ class DataSmart(MutableMapping):
         dest = self.dict
         while dest:
             if var in dest:
-                return dest[var]
+                return dest[var], self.overridedata.get(var, None)
 
             if "_remote_data" in dest:
                 connector = dest["_remote_data"]["_content"]
@@ -482,12 +482,13 @@ class DataSmart(MutableMapping):
             if "_data" not in dest:
                 break
             dest = dest["_data"]
+        return None, self.overridedata.get(var, None)
 
     def _makeShadowCopy(self, var):
         if var in self.dict:
             return
 
-        local_var = self._findVar(var)
+        local_var, _ = self._findVar(var)
 
         if local_var:
             self.dict[var] = copy.copy(local_var)
@@ -608,6 +609,12 @@ class DataSmart(MutableMapping):
         """
         Rename the variable key to newkey
         """
+        if '_remote_data' in self.dict:
+            connector = self.dict["_remote_data"]["_content"]
+            res = connector.renameVar(key, newkey)
+            if not res:
+                return
+
         val = self.getVar(key, 0, parsing=True)
         if val is not None:
             loginfo['variable'] = newkey
@@ -651,6 +658,12 @@ class DataSmart(MutableMapping):
         self.setVar(var + "_prepend", value, ignore=True, parsing=True)
 
     def delVar(self, var, **loginfo):
+        if '_remote_data' in self.dict:
+            connector = self.dict["_remote_data"]["_content"]
+            res = connector.delVar(var)
+            if not res:
+                return
+
         loginfo['detail'] = ""
         loginfo['op'] = 'del'
         self.varhistory.record(**loginfo)
@@ -677,6 +690,12 @@ class DataSmart(MutableMapping):
                          override = None
 
     def setVarFlag(self, var, flag, value, **loginfo):
+        if '_remote_data' in self.dict:
+            connector = self.dict["_remote_data"]["_content"]
+            res = connector.setVarFlag(var, flag, value)
+            if not res:
+                return
+
         self.expand_cache = {}
         if 'op' not in loginfo:
             loginfo['op'] = "set"
@@ -699,13 +718,13 @@ class DataSmart(MutableMapping):
             self.dict["__exportlist"]["_content"].add(var)
 
     def getVarFlag(self, var, flag, expand=True, noweakdefault=False, parsing=False):
-        local_var = self._findVar(var)
+        local_var, overridedata = self._findVar(var)
         value = None
-        if flag == "_content" and var in self.overridedata and not parsing:
+        if flag == "_content" and overridedata is not None and not parsing:
             match = False
             active = {}
             self.need_overrides()
-            for (r, o) in self.overridedata[var]:
+            for (r, o) in overridedata:
                 # What about double overrides both with "_" in the name?
                 if o in self.overridesset:
                     active[o] = r
@@ -795,8 +814,14 @@ class DataSmart(MutableMapping):
         return value
 
     def delVarFlag(self, var, flag, **loginfo):
+        if '_remote_data' in self.dict:
+            connector = self.dict["_remote_data"]["_content"]
+            res = connector.delVarFlag(var, flag)
+            if not res:
+                return
+
         self.expand_cache = {}
-        local_var = self._findVar(var)
+        local_var, _ = self._findVar(var)
         if not local_var:
             return
         if not var in self.dict:
@@ -839,7 +864,7 @@ class DataSmart(MutableMapping):
             self.dict[var][i] = flags[i]
 
     def getVarFlags(self, var, expand = False, internalflags=False):
-        local_var = self._findVar(var)
+        local_var, _ = self._findVar(var)
         flags = {}
 
         if local_var:
@@ -937,7 +962,10 @@ class DataSmart(MutableMapping):
 
             if "_remote_data" in d:
                 connector = d["_remote_data"]["_content"]
-                klist |= connector.getKeys()
+                for key in connector.getKeys():
+                    if key in deleted:
+                        continue
+                    klist.add(key)
 
             return klist
 
@@ -976,7 +1004,6 @@ class DataSmart(MutableMapping):
         data = {}
         d = self.createCopy()
         bb.data.expandKeys(d)
-        bb.data.update_data(d)
 
         config_whitelist = set((d.getVar("BB_HASHCONFIG_WHITELIST") or "").split())
         keys = set(key for key in iter(d) if not key.startswith("__"))

bitbake/lib/bb/fetch2/__init__.py

@@ -35,9 +35,9 @@ import operator
 import collections
 import subprocess
 import pickle
+import errno
 import bb.persist_data, bb.utils
 import bb.checksum
-from bb import data
 import bb.process
 
 __version__ = "2"
@@ -355,7 +355,7 @@ def decodeurl(url):
     user, password, parameters).
     """
 
-    m = re.compile('(?P<type>[^:]*)://((?P<user>[^/]+)@)?(?P<location>[^;]+)(;(?P<parm>.*))?').match(url)
+    m = re.compile('(?P<type>[^:]*)://((?P<user>[^/;]+)@)?(?P<location>[^;]+)(;(?P<parm>.*))?').match(url)
     if not m:
         raise MalformedUrl(url)
 
@@ -537,7 +537,11 @@ def fetcher_compare_revisions():
     return False
 
 def mirror_from_string(data):
-    return [ i.split() for i in (data or "").replace('\\n','\n').split('\n') if i ]
+    mirrors = (data or "").replace('\\n',' ').split()
+    # Split into pairs
+    if len(mirrors) % 2 != 0:
+        bb.warn('Invalid mirror data %s, should have paired members.' % data)
+    return list(zip(*[iter(mirrors)]*2))
 
 def verify_checksum(ud, d, precomputed={}):
     """
@@ -724,7 +728,7 @@ def get_autorev(d):
 
 def get_srcrev(d, method_name='sortable_revision'):
     """
-    Return the revsion string, usually for use in the version string (PV) of the current package
+    Return the revision string, usually for use in the version string (PV) of the current package
     Most packages usually only have one SCM so we just pass on the call.
     In the multi SCM case, we build a value based on SRCREV_FORMAT which must
     have been set.
@@ -998,6 +1002,12 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
     except bb.fetch2.NetworkAccess:
         raise
 
+    except IOError as e:
+        if e.errno in [os.errno.ESTALE]:
+            logger.warn("Stale Error Observed %s." % ud.url)
+            return False
+        raise
+
     except bb.fetch2.BBFetchException as e:
         if isinstance(e, ChecksumError):
             logger.warning("Mirror checksum failure for url %s (original url: %s)\nCleaning and trying again." % (ud.url, origud.url))
@@ -1172,7 +1182,7 @@ class FetchData(object):
         self.mirrortarball = None
         self.basename = None
         self.basepath = None
-        (self.type, self.host, self.path, self.user, self.pswd, self.parm) = decodeurl(data.expand(url, d))
+        (self.type, self.host, self.path, self.user, self.pswd, self.parm) = decodeurl(d.expand(url))
         self.date = self.getSRCDate(d)
         self.url = url
         if not self.user and "user" in self.parm:
@@ -1189,13 +1199,13 @@ class FetchData(object):
             self.sha256_name = "sha256sum"
         if self.md5_name in self.parm:
             self.md5_expected = self.parm[self.md5_name]
-        elif self.type not in ["http", "https", "ftp", "ftps", "sftp"]:
+        elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3"]:
             self.md5_expected = None
         else:
             self.md5_expected = d.getVarFlag("SRC_URI", self.md5_name)
         if self.sha256_name in self.parm:
             self.sha256_expected = self.parm[self.sha256_name]
-        elif self.type not in ["http", "https", "ftp", "ftps", "sftp"]:
+        elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3"]:
             self.sha256_expected = None
         else:
             self.sha256_expected = d.getVarFlag("SRC_URI", self.sha256_name)
@@ -1247,7 +1257,7 @@ class FetchData(object):
         self.donestamp = basepath + '.done'
         self.lockfile = basepath + '.lock'
 
-    def setup_revisons(self, d):
+    def setup_revisions(self, d):
         self.revisions = {}
         for name in self.names:
             self.revisions[name] = srcrev_internal_helper(self, d, name)
@@ -1394,6 +1404,10 @@ class FetchMethod(object):
                 cmd = 'lzip -dc %s | tar x --no-same-owner -f -' % file
             elif file.endswith('.lz'):
                 cmd = 'lzip -dc %s > %s' % (file, efile)
+            elif file.endswith('.tar.7z'):
+                cmd = '7z x -so %s | tar x --no-same-owner -f -' % file
+            elif file.endswith('.7z'):
+                cmd = '7za x -y %s 1>/dev/null' % file
             elif file.endswith('.zip') or file.endswith('.jar'):
                 try:
                     dos = bb.utils.to_boolean(urldata.parm.get('dos'), False)
@@ -1425,10 +1439,6 @@ class FetchMethod(object):
                 else:
                     raise UnpackError("Unable to unpack deb/ipk package - could not list contents", urldata.url)
                 cmd = 'ar x %s %s && tar --no-same-owner -xpf %s && rm %s' % (file, datafile, datafile, datafile)
-            elif file.endswith('.tar.7z'):
-                cmd = '7z x -so %s | tar xf - ' % file
-            elif file.endswith('.7z'):
-                cmd = '7za x -y %s 1>/dev/null' % file
 
         # If 'subdir' param exists, create a dir and use it as destination for unpack cmd
         if 'subdir' in urldata.parm:
@@ -1646,6 +1656,11 @@ class Fetch(object):
 
                 update_stamp(ud, self.d)
 
+            except IOError as e:
+                if e.errno in [os.errno.ESTALE]:
+                    logger.error("Stale Error Observed %s." % u)
+                    raise ChecksumError("Stale Error Detected")
+
             except BBFetchException as e:
                 if isinstance(e, ChecksumError):
                     logger.error("Checksum failure fetching %s" % u)
@@ -1775,6 +1790,7 @@ from . import svn
 from . import wget
 from . import ssh
 from . import sftp
+from . import s3
 from . import perforce
 from . import bzr
 from . import hg
@@ -1792,6 +1808,7 @@ methods.append(gitannex.GitANNEX())
 methods.append(cvs.Cvs())
 methods.append(ssh.SSH())
 methods.append(sftp.SFTP())
+methods.append(s3.S3())
 methods.append(perforce.Perforce())
 methods.append(bzr.Bzr())
 methods.append(hg.Hg())

bitbake/lib/bb/fetch2/bzr.py

@@ -27,7 +27,6 @@ import os
 import sys
 import logging
 import bb
-from bb import data
 from bb.fetch2 import FetchMethod
 from bb.fetch2 import FetchError
 from bb.fetch2 import runfetchcmd
@@ -43,14 +42,14 @@ class Bzr(FetchMethod):
         """
         # Create paths to bzr checkouts
         relpath = self._strip_leading_slashes(ud.path)
-        ud.pkgdir = os.path.join(data.expand('${BZRDIR}', d), ud.host, relpath)
+        ud.pkgdir = os.path.join(d.expand('${BZRDIR}'), ud.host, relpath)
 
-        ud.setup_revisons(d)
+        ud.setup_revisions(d)
 
         if not ud.revision:
             ud.revision = self.latest_revision(ud, d)
 
-        ud.localfile = data.expand('bzr_%s_%s_%s.tar.gz' % (ud.host, ud.path.replace('/', '.'), ud.revision), d)
+        ud.localfile = d.expand('bzr_%s_%s_%s.tar.gz' % (ud.host, ud.path.replace('/', '.'), ud.revision))
 
     def _buildbzrcommand(self, ud, d, command):
         """
@@ -58,7 +57,7 @@ class Bzr(FetchMethod):
         command is "fetch", "update", "revno"
         """
 
-        basecmd = data.expand('${FETCHCMD_bzr}', d)
+        basecmd = d.expand('${FETCHCMD_bzr}')
 
         proto =  ud.parm.get('protocol', 'http')
 

bitbake/lib/bb/fetch2/clearcase.py

@@ -65,7 +65,6 @@ import os
 import sys
 import shutil
 import bb
-from   bb import data
 from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import FetchError
 from   bb.fetch2 import runfetchcmd

bitbake/lib/bb/fetch2/cvs.py

@@ -63,7 +63,7 @@ class Cvs(FetchMethod):
         if 'fullpath' in ud.parm:
             fullpath = '_fullpath'
 
-        ud.localfile = bb.data.expand('%s_%s_%s_%s%s%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.tag, ud.date, norecurse, fullpath), d)
+        ud.localfile = d.expand('%s_%s_%s_%s%s%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.tag, ud.date, norecurse, fullpath))
 
     def need_update(self, ud, d):
         if (ud.date == "now"):

bitbake/lib/bb/fetch2/git.py

@@ -50,7 +50,7 @@ Supported SRC_URI options are:
    The default is "0", set nobranch=1 if needed.
 
 - usehead
-   For local git:// urls to use the current branch HEAD as the revsion for use with
+   For local git:// urls to use the current branch HEAD as the revision for use with
    AUTOREV. Implies nobranch.
 
 """
@@ -76,7 +76,6 @@ import re
 import bb
 import errno
 import bb.progress
-from   bb    import data
 from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import runfetchcmd
 from   bb.fetch2 import logger
@@ -174,8 +173,8 @@ class Git(FetchMethod):
         if len(branches) != len(ud.names):
             raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url)
         ud.branches = {}
-        for name in ud.names:
-            branch = branches[ud.names.index(name)]
+        for pos, name in enumerate(ud.names):
+            branch = branches[pos]
             ud.branches[name] = branch
             ud.unresolvedrev[name] = branch
 
@@ -186,7 +185,7 @@ class Git(FetchMethod):
 
         ud.write_tarballs = ((d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0") != "0") or ud.rebaseable
 
-        ud.setup_revisons(d)
+        ud.setup_revisions(d)
 
         for name in ud.names:
             # Ensure anything that doesn't look like a sha256 checksum/revision is translated into one
@@ -206,7 +205,7 @@ class Git(FetchMethod):
         if ud.rebaseable:
             for name in ud.names:
                 gitsrcname = gitsrcname + '_' + ud.revisions[name]
-        ud.mirrortarball = 'git2_%s.tar.gz' % (gitsrcname)
+        ud.mirrortarball = 'git2_%s.tar.gz' % gitsrcname
         ud.fullmirror = os.path.join(d.getVar("DL_DIR"), ud.mirrortarball)
         gitdir = d.getVar("GITDIR") or (d.getVar("DL_DIR") + "/git2/")
         ud.clonedir = os.path.join(gitdir, gitsrcname)
@@ -241,7 +240,7 @@ class Git(FetchMethod):
         # If the checkout doesn't exist and the mirror tarball does, extract it
         if not os.path.exists(ud.clonedir) and os.path.exists(ud.fullmirror):
             bb.utils.mkdirhier(ud.clonedir)
-            runfetchcmd("tar -xzf %s" % (ud.fullmirror), d, workdir=ud.clonedir)
+            runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir)
 
         repourl = self._get_repo_url(ud)
 
@@ -292,15 +291,15 @@ class Git(FetchMethod):
                 os.unlink(ud.fullmirror)
 
             logger.info("Creating tarball of git repository")
-            runfetchcmd("tar -czf %s %s" % (ud.fullmirror, os.path.join(".") ), d, workdir=ud.clonedir)
-            runfetchcmd("touch %s.done" % (ud.fullmirror), d, workdir=ud.clonedir)
+            runfetchcmd("tar -czf %s ." % ud.fullmirror, d, workdir=ud.clonedir)
+            runfetchcmd("touch %s.done" % ud.fullmirror, d)
 
     def unpack(self, ud, destdir, d):
         """ unpack the downloaded src to destdir"""
 
         subdir = ud.parm.get("subpath", "")
         if subdir != "":
-            readpathspec = ":%s" % (subdir)
+            readpathspec = ":%s" % subdir
             def_destsuffix = "%s/" % os.path.basename(subdir.rstrip('/'))
         else:
             readpathspec = ""
@@ -470,7 +469,7 @@ class Git(FetchMethod):
             if not os.path.exists(rev_file) or not os.path.getsize(rev_file):
                 from pipes import quote
                 commits = bb.fetch2.runfetchcmd(
-                        "git rev-list %s -- | wc -l" % (quote(rev)),
+                        "git rev-list %s -- | wc -l" % quote(rev),
                         d, quiet=True).strip().lstrip('0')
                 if commits:
                     open(rev_file, "w").write("%d\n" % int(commits))
@@ -485,5 +484,5 @@ class Git(FetchMethod):
         try:
             self._lsremote(ud, d, "")
             return True
-        except FetchError:
+        except bb.fetch2.FetchError:
             return False

bitbake/lib/bb/fetch2/gitannex.py

@@ -22,7 +22,6 @@ BitBake 'Fetch' git annex implementation
 
 import os
 import bb
-from   bb import data
 from   bb.fetch2.git import Git
 from   bb.fetch2 import runfetchcmd
 from   bb.fetch2 import logger

bitbake/lib/bb/fetch2/gitsm.py

@@ -31,7 +31,6 @@ NOTE: Switching a SRC_URI from "git://" to "gitsm://" requires a clean of your r
 
 import os
 import bb
-from   bb    import data
 from   bb.fetch2.git import Git
 from   bb.fetch2 import runfetchcmd
 from   bb.fetch2 import logger

bitbake/lib/bb/fetch2/hg.py

@@ -29,7 +29,6 @@ import sys
 import logging
 import bb
 import errno
-from bb import data
 from bb.fetch2 import FetchMethod
 from bb.fetch2 import FetchError
 from bb.fetch2 import MissingParameterError
@@ -67,7 +66,7 @@ class Hg(FetchMethod):
         else:
             ud.proto = "hg"
 
-        ud.setup_revisons(d)
+        ud.setup_revisions(d)
 
         if 'rev' in ud.parm:
             ud.revision = ud.parm['rev']

bitbake/lib/bb/fetch2/local.py

@@ -29,7 +29,6 @@ import os
 import urllib.request, urllib.parse, urllib.error
 import bb
 import bb.utils
-from   bb import data
 from   bb.fetch2 import FetchMethod, FetchError
 from   bb.fetch2 import logger
 

bitbake/lib/bb/fetch2/npm.py

@@ -25,7 +25,6 @@ import json
 import subprocess
 import signal
 import bb
-from   bb import data
 from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import FetchError
 from   bb.fetch2 import ChecksumError
@@ -80,6 +79,7 @@ class Npm(FetchMethod):
         if not ud.version:
             raise ParameterError("NPM fetcher requires a version parameter", ud.url)
         ud.bbnpmmanifest = "%s-%s.deps.json" % (ud.pkgname, ud.version)
+        ud.bbnpmmanifest = ud.bbnpmmanifest.replace('/', '-')
         ud.registry = "http://%s" % (ud.url.replace('npm://', '', 1).split(';'))[0]
         prefixdir = "npm/%s" % ud.pkgname
         ud.pkgdatadir = d.expand("${DL_DIR}/%s" % prefixdir)
@@ -92,6 +92,7 @@ class Npm(FetchMethod):
 
         ud.write_tarballs = ((d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0") != "0")
         ud.mirrortarball = 'npm_%s-%s.tar.xz' % (ud.pkgname, ud.version)
+        ud.mirrortarball = ud.mirrortarball.replace('/', '-')
         ud.fullmirror = os.path.join(d.getVar("DL_DIR"), ud.mirrortarball)
 
     def need_update(self, ud, d):
@@ -133,8 +134,7 @@ class Npm(FetchMethod):
 
     def unpack(self, ud, destdir, d):
         dldir = d.getVar("DL_DIR")
-        depdumpfile = "%s-%s.deps.json" % (ud.pkgname, ud.version)
-        with open("%s/npm/%s" % (dldir, depdumpfile)) as datafile:
+        with open("%s/npm/%s" % (dldir, ud.bbnpmmanifest)) as datafile:
             workobj = json.load(datafile)
         dldir = "%s/%s" % (os.path.dirname(ud.localpath), ud.pkgname)
 
@@ -182,7 +182,12 @@ class Npm(FetchMethod):
             if pkg_os:
                 if not isinstance(pkg_os, list):
                     pkg_os = [pkg_os]
-                if 'linux' not in pkg_os or '!linux' in pkg_os:
+                blacklist = False
+                for item in pkg_os:
+                    if item.startswith('!'):
+                        blacklist = True
+                        break
+                if (not blacklist and 'linux' not in pkg_os) or '!linux' in pkg_os:
                     logger.debug(2, "Skipping %s since it's incompatible with Linux" % pkg)
                     return
         #logger.debug(2, "Output URL is %s - %s - %s" % (ud.basepath, ud.basename, ud.localfile))
@@ -195,6 +200,7 @@ class Npm(FetchMethod):
 
         dependencies = pdata.get('dependencies', {})
         optionalDependencies = pdata.get('optionalDependencies', {})
+        dependencies.update(optionalDependencies)
         depsfound = {}
         optdepsfound = {}
         data[pkg]['deps'] = {}

bitbake/lib/bb/fetch2/osc.py

@@ -10,7 +10,6 @@ import  os
 import  sys
 import logging
 import  bb
-from    bb       import data
 from    bb.fetch2 import FetchMethod
 from    bb.fetch2 import FetchError
 from    bb.fetch2 import MissingParameterError
@@ -47,7 +46,7 @@ class Osc(FetchMethod):
             else:
                 ud.revision = ""
 
-        ud.localfile = data.expand('%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.path.replace('/', '.'), ud.revision), d)
+        ud.localfile = d.expand('%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.path.replace('/', '.'), ud.revision))
 
     def _buildosccommand(self, ud, d, command):
         """
@@ -55,7 +54,7 @@ class Osc(FetchMethod):
         command is "fetch", "update", "info"
         """
 
-        basecmd = data.expand('${FETCHCMD_osc}', d)
+        basecmd = d.expand('${FETCHCMD_osc}')
 
         proto = ud.parm.get('protocol', 'ocs')
 

bitbake/lib/bb/fetch2/perforce.py

@@ -26,7 +26,6 @@ BitBake 'Fetch' implementation for perforce
 import os
 import logging
 import bb
-from   bb import data
 from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import FetchError
 from   bb.fetch2 import logger
@@ -87,9 +86,9 @@ class Perforce(FetchMethod):
         cleanedhost = ud.host.replace(':', '.')
         ud.pkgdir = os.path.join(ud.dldir, cleanedhost, cleanedpath)
 
-        ud.setup_revisons(d)
+        ud.setup_revisions(d)
 
-        ud.localfile = data.expand('%s_%s_%s.tar.gz' % (cleanedhost, cleanedpath, ud.revision), d)
+        ud.localfile = d.expand('%s_%s_%s.tar.gz' % (cleanedhost, cleanedpath, ud.revision))
 
     def _buildp4command(self, ud, d, command, depot_filename=None):
         """

bitbake/lib/bb/fetch2/repo.py

@@ -25,7 +25,6 @@ BitBake "Fetch" repo (git) implementation
 
 import os
 import bb
-from   bb    import data
 from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import runfetchcmd
 

bitbake/lib/bb/fetch2/s3.py

@@ -0,0 +1,96 @@
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+"""
+BitBake 'Fetch' implementation for Amazon AWS S3.
+
+Class for fetching files from Amazon S3 using the AWS Command Line Interface.
+The aws tool must be correctly installed and configured prior to use.
+
+"""
+
+# Copyright (C) 2017, Andre McCurdy <armccurdy@gmail.com>
+#
+# Based in part on bb.fetch2.wget:
+#    Copyright (C) 2003, 2004  Chris Larson
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Based on functions from the base bb module, Copyright 2003 Holger Schurig
+
+import os
+import bb
+import urllib.request, urllib.parse, urllib.error
+from bb.fetch2 import FetchMethod
+from bb.fetch2 import FetchError
+from bb.fetch2 import runfetchcmd
+
+class S3(FetchMethod):
+    """Class to fetch urls via 'aws s3'"""
+
+    def supports(self, ud, d):
+        """
+        Check to see if a given url can be fetched with s3.
+        """
+        return ud.type in ['s3']
+
+    def recommends_checksum(self, urldata):
+        return True
+
+    def urldata_init(self, ud, d):
+        if 'downloadfilename' in ud.parm:
+            ud.basename = ud.parm['downloadfilename']
+        else:
+            ud.basename = os.path.basename(ud.path)
+
+        ud.localfile = d.expand(urllib.parse.unquote(ud.basename))
+
+    def download(self, ud, d):
+        """
+        Fetch urls
+        Assumes localpath was called first
+        """
+
+        cmd = 'aws s3 cp s3://%s%s %s' % (ud.host, ud.path, ud.localpath)
+        bb.fetch2.check_network_access(d, cmd, ud.url)
+        runfetchcmd(cmd, d)
+
+        # Additional sanity checks copied from the wget class (although there
+        # are no known issues which mean these are required, treat the aws cli
+        # tool with a little healthy suspicion).
+
+        if not os.path.exists(ud.localpath):
+            raise FetchError("The aws cp command returned success for s3://%s%s but %s doesn't exist?!" % (ud.host, ud.path, ud.localpath))
+
+        if os.path.getsize(ud.localpath) == 0:
+            os.remove(ud.localpath)
+            raise FetchError("The aws cp command for s3://%s%s resulted in a zero size file?! Deleting and failing since this isn't right." % (ud.host, ud.path))
+
+        return True
+
+    def checkstatus(self, fetch, ud, d):
+        """
+        Check the status of a URL
+        """
+
+        cmd = 'aws s3 ls s3://%s%s' % (ud.host, ud.path)
+        bb.fetch2.check_network_access(d, cmd, ud.url)
+        output = runfetchcmd(cmd, d)
+
+        # "aws s3 ls s3://mybucket/foo" will exit with success even if the file
+        # is not found, so check output of the command to confirm success.
+
+        if not output:
+            raise FetchError("The aws ls command for s3://%s%s gave empty output" % (ud.host, ud.path))
+
+        return True

bitbake/lib/bb/fetch2/sftp.py

@@ -62,12 +62,10 @@ SRC_URI = "sftp://user@host.example.com/dir/path.file.txt"
 import os
 import bb
 import urllib.request, urllib.parse, urllib.error
-from bb import data
 from bb.fetch2 import URI
 from bb.fetch2 import FetchMethod
 from bb.fetch2 import runfetchcmd
 
-
 class SFTP(FetchMethod):
     """Class to fetch urls via 'sftp'"""
 

bitbake/lib/bb/fetch2/ssh.py

@@ -43,7 +43,6 @@ IETF secsh internet draft:
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 import re, os
-from   bb import data
 from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import FetchError
 from   bb.fetch2 import logger

bitbake/lib/bb/fetch2/svn.py

@@ -28,7 +28,6 @@ import sys
 import logging
 import bb
 import re
-from   bb import data
 from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import FetchError
 from   bb.fetch2 import MissingParameterError
@@ -61,15 +60,15 @@ class Svn(FetchMethod):
 
         # Create paths to svn checkouts
         relpath = self._strip_leading_slashes(ud.path)
-        ud.pkgdir = os.path.join(data.expand('${SVNDIR}', d), ud.host, relpath)
+        ud.pkgdir = os.path.join(d.expand('${SVNDIR}'), ud.host, relpath)
         ud.moddir = os.path.join(ud.pkgdir, ud.module)
 
-        ud.setup_revisons(d)
+        ud.setup_revisions(d)
 
         if 'rev' in ud.parm:
             ud.revision = ud.parm['rev']
 
-        ud.localfile = data.expand('%s_%s_%s_%s_.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision), d)
+        ud.localfile = d.expand('%s_%s_%s_%s_.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision))
 
     def _buildsvncommand(self, ud, d, command):
         """
@@ -79,9 +78,9 @@ class Svn(FetchMethod):
 
         proto = ud.parm.get('protocol', 'svn')
 
-        svn_rsh = None
-        if proto == "svn+ssh" and "rsh" in ud.parm:
-            svn_rsh = ud.parm["rsh"]
+        svn_ssh = None
+        if proto == "svn+ssh" and "ssh" in ud.parm:
+            svn_ssh = ud.parm["ssh"]
 
         svnroot = ud.host + ud.path
 
@@ -113,8 +112,8 @@ class Svn(FetchMethod):
             else:
                 raise FetchError("Invalid svn command %s" % command, ud.url)
 
-        if svn_rsh:
-            svncmd = "svn_RSH=\"%s\" %s" % (svn_rsh, svncmd)
+        if svn_ssh:
+            svncmd = "SVN_SSH=\"%s\" %s" % (svn_ssh, svncmd)
 
         return svncmd
 

bitbake/lib/bb/fetch2/wget.py

@@ -33,7 +33,6 @@ import logging
 import bb
 import bb.progress
 import urllib.request, urllib.parse, urllib.error
-from   bb import data
 from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import FetchError
 from   bb.fetch2 import logger
@@ -84,9 +83,9 @@ class Wget(FetchMethod):
         else:
             ud.basename = os.path.basename(ud.path)
 
-        ud.localfile = data.expand(urllib.parse.unquote(ud.basename), d)
+        ud.localfile = d.expand(urllib.parse.unquote(ud.basename))
         if not ud.localfile:
-            ud.localfile = data.expand(urllib.parse.unquote(ud.host + ud.path).replace("/", "."), d)
+            ud.localfile = d.expand(urllib.parse.unquote(ud.host + ud.path).replace("/", "."))
 
         self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -t 2 -T 30 --passive-ftp --no-check-certificate"
 
@@ -547,7 +546,7 @@ class Wget(FetchMethod):
 
         # src.rpm extension was added only for rpm package. Can be removed if the rpm
         # packaged will always be considered as having to be manually upgraded
-        psuffix_regex = "(tar\.gz|tgz|tar\.bz2|zip|xz|rpm|bz2|orig\.tar\.gz|tar\.xz|src\.tar\.gz|src\.tgz|svnr\d+\.tar\.bz2|stable\.tar\.gz|src\.rpm)"
+        psuffix_regex = "(tar\.gz|tgz|tar\.bz2|zip|xz|tar\.lz|rpm|bz2|orig\.tar\.gz|tar\.xz|src\.tar\.gz|src\.tgz|svnr\d+\.tar\.bz2|stable\.tar\.gz|src\.rpm)"
 
         # match name, version and archive type of a package
         package_regex_comp = re.compile("(?P<name>%s?\.?v?)(?P<pver>%s)(?P<arch>%s)?[\.-](?P<type>%s$)"

bitbake/lib/bb/main.py

@@ -287,6 +287,9 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
                           help="Writes the event log of the build to a bitbake event json file. "
                                "Use '' (empty string) to assign the name automatically.")
 
+        parser.add_option("", "--runall", action="store", dest="runall",
+                          help="Run the specified task for all build targets and their dependencies.")
+
         options, targets = parser.parse_args(argv)
 
         if options.quiet and options.verbose:

bitbake/lib/bb/parse/ast.py

@@ -130,7 +130,6 @@ class DataNode(AstNode):
                 val = groupd["value"]
         elif "colon" in groupd and groupd["colon"] != None:
             e = data.createCopy()
-            bb.data.update_data(e)
             op = "immediate"
             val = e.expand(groupd["value"], key + "[:=]")
         elif "append" in groupd and groupd["append"] != None:
@@ -350,12 +349,10 @@ def finalize(fn, d, variant = None):
     bb.event.fire(bb.event.RecipePreFinalise(fn), d)
 
     bb.data.expandKeys(d)
-    bb.data.update_data(d)
     code = []
     for funcname in d.getVar("__BBANONFUNCS", False) or []:
         code.append("%s(d)" % funcname)
     bb.utils.better_exec("\n".join(code), {"d": d})
-    bb.data.update_data(d)
 
     tasklist = d.getVar('__BBTASKS', False) or []
     bb.event.fire(bb.event.RecipeTaskPreProcess(fn, list(tasklist)), d)

bitbake/lib/bb/parse/parse_py/ConfHandler.py

@@ -33,7 +33,7 @@ from bb.parse import ParseError, resolve_file, ast, logger, handle
 __config_regexp__  = re.compile( r"""
     ^
     (?P<exp>export\s*)?
-    (?P<var>[a-zA-Z0-9\-~_+.${}/]+?)
+    (?P<var>[a-zA-Z0-9\-_+.${}/~]+?)
     (\[(?P<flag>[a-zA-Z0-9\-_+.]+)\])?
 
     \s* (
@@ -56,9 +56,9 @@ __config_regexp__  = re.compile( r"""
     """, re.X)
 __include_regexp__ = re.compile( r"include\s+(.+)" )
 __require_regexp__ = re.compile( r"require\s+(.+)" )
-__export_regexp__ = re.compile( r"export\s+([a-zA-Z0-9\-_+.${}/]+)$" )
-__unset_regexp__ = re.compile( r"unset\s+([a-zA-Z0-9\-_+.${}/]+)$" )
-__unset_flag_regexp__ = re.compile( r"unset\s+([a-zA-Z0-9\-_+.${}/]+)\[([a-zA-Z0-9\-_+.${}/]+)\]$" )
+__export_regexp__ = re.compile( r"export\s+([a-zA-Z0-9\-_+.${}/~]+)$" )
+__unset_regexp__ = re.compile( r"unset\s+([a-zA-Z0-9\-_+.${}/~]+)$" )
+__unset_flag_regexp__ = re.compile( r"unset\s+([a-zA-Z0-9\-_+.${}/~]+)\[([a-zA-Z0-9\-_+.]+)\]$" )
 
 def init(data):
     topdir = data.getVar('TOPDIR', False)

bitbake/lib/bb/process.py

@@ -162,9 +162,9 @@ def run(cmd, input=None, log=None, extrafiles=None, **options):
         stdout, stderr = _logged_communicate(pipe, log, input, extrafiles)
     else:
         stdout, stderr = pipe.communicate(input)
-        if stdout:
+        if not stdout is None:
             stdout = stdout.decode("utf-8")
-        if stderr:
+        if not stderr is None:
             stderr = stderr.decode("utf-8")
 
     if pipe.returncode != 0:

bitbake/lib/bb/providers.py

@@ -48,7 +48,6 @@ def findProviders(cfgData, dataCache, pkg_pn = None):
 
     # Need to ensure data store is expanded
     localdata = data.createCopy(cfgData)
-    bb.data.update_data(localdata)
     bb.data.expandKeys(localdata)
 
     preferred_versions = {}

bitbake/lib/bb/runqueue.py

@@ -803,6 +803,23 @@ class RunQueueData:
 
         self.init_progress_reporter.next_stage()
 
+        if self.cooker.configuration.runall is not None:
+            runall = "do_%s" % self.cooker.configuration.runall
+            runall_tids = { k: v for k, v in self.runtaskentries.items() if taskname_from_tid(k) == runall }
+
+            # re-run the mark_active and then drop unused tasks from new list
+            runq_build = {}
+            for tid in list(runall_tids):
+                mark_active(tid,1)
+
+            for tid in list(self.runtaskentries.keys()):
+                if tid not in runq_build:
+                    del self.runtaskentries[tid]
+                    delcount += 1
+
+            if len(self.runtaskentries) == 0:
+                bb.msg.fatal("RunQueue", "No remaining tasks to run for build target %s with runall %s" % (target, runall))
+
         #
         # Step D - Sanity checks and computation
         #
@@ -2010,6 +2027,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
         for tid in self.rqdata.runq_setscene_tids:
             #bb.warn("Added endpoint 2 %s" % (tid))
             for dep in self.rqdata.runtaskentries[tid].depends:
+                    if tid in sq_revdeps[dep]:
+                        sq_revdeps[dep].remove(tid)
                     if dep not in endpoints:
                         endpoints[dep] = set()
                     #bb.warn("  Added endpoint 3 %s" % (dep))
@@ -2029,12 +2048,13 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
                 if point in self.rqdata.runq_setscene_tids:
                     sq_revdeps_new[point] = tasks
                     tasks = set()
+                    continue
                 for dep in self.rqdata.runtaskentries[point].depends:
                     if point in sq_revdeps[dep]:
                         sq_revdeps[dep].remove(point)
                     if tasks:
                         sq_revdeps_new[dep] |= tasks
-                    if (len(sq_revdeps[dep]) == 0 or len(sq_revdeps_new[dep]) != 0) and dep not in self.rqdata.runq_setscene_tids:
+                    if len(sq_revdeps[dep]) == 0 and dep not in self.rqdata.runq_setscene_tids:
                         newendpoints[dep] = task
             if len(newendpoints) != 0:
                 process_endpoints(newendpoints)

bitbake/lib/bb/taskdata.py

@@ -97,7 +97,7 @@ class TaskData:
                     if dep:
                         parts = dep.split(":")
                         if len(parts) != 2:
-                            bb.msg.fatal("TaskData", "Error for %s, dependency %s does not contain exactly one ':' character.\n Task '%s' should be specified in the form 'packagename:task'" % (fn, dep, dep_name))
+                            bb.msg.fatal("TaskData", "Error for %s:%s[%s], dependency %s in '%s' does not contain exactly one ':' character.\n Task '%s' should be specified in the form 'packagename:task'" % (fn, task, dep_name, dep, task_deps[dep_name][task], dep_name))
                         ids.append((parts[0], parts[1]))
                         seen(parts[0])
                 depends.extend(ids)

bitbake/lib/bb/tests/codeparser.py

@@ -285,8 +285,8 @@ def a():
 
 test(d)
 
-bb.data.expand(d.getVar("something", False), d)
-bb.data.expand("${inexpand} somethingelse", d)
+d.expand(d.getVar("something", False))
+d.expand("${inexpand} somethingelse")
 d.getVar(a(), False)
 """
 

bitbake/lib/bb/tests/data.py

@@ -258,13 +258,11 @@ class TestConcatOverride(unittest.TestCase):
     def test_prepend(self):
         self.d.setVar("TEST", "${VAL}")
         self.d.setVar("TEST_prepend", "${FOO}:")
-        bb.data.update_data(self.d)
         self.assertEqual(self.d.getVar("TEST"), "foo:val")
 
     def test_append(self):
         self.d.setVar("TEST", "${VAL}")
         self.d.setVar("TEST_append", ":${BAR}")
-        bb.data.update_data(self.d)
         self.assertEqual(self.d.getVar("TEST"), "val:bar")
 
     def test_multiple_append(self):
@@ -272,47 +270,47 @@ class TestConcatOverride(unittest.TestCase):
         self.d.setVar("TEST_prepend", "${FOO}:")
         self.d.setVar("TEST_append", ":val2")
         self.d.setVar("TEST_append", ":${BAR}")
-        bb.data.update_data(self.d)
         self.assertEqual(self.d.getVar("TEST"), "foo:val:val2:bar")
 
     def test_append_unset(self):
         self.d.setVar("TEST_prepend", "${FOO}:")
         self.d.setVar("TEST_append", ":val2")
         self.d.setVar("TEST_append", ":${BAR}")
-        bb.data.update_data(self.d)
         self.assertEqual(self.d.getVar("TEST"), "foo::val2:bar")
 
     def test_remove(self):
         self.d.setVar("TEST", "${VAL} ${BAR}")
         self.d.setVar("TEST_remove", "val")
-        bb.data.update_data(self.d)
         self.assertEqual(self.d.getVar("TEST"), "bar")
 
+    # Ensure the value is unchanged if we have an inactive remove override
+    # (including that whitespace is preserved)
+    def test_remove_inactive_override(self):
+        self.d.setVar("TEST", "${VAL} ${BAR}    123")
+        self.d.setVar("TEST_remove_inactiveoverride", "val")
+        self.assertEqual(self.d.getVar("TEST"), "val bar    123")
+
     def test_doubleref_remove(self):
         self.d.setVar("TEST", "${VAL} ${BAR}")
         self.d.setVar("TEST_remove", "val")
         self.d.setVar("TEST_TEST", "${TEST} ${TEST}")
-        bb.data.update_data(self.d)
         self.assertEqual(self.d.getVar("TEST_TEST"), "bar bar")
 
     def test_empty_remove(self):
         self.d.setVar("TEST", "")
         self.d.setVar("TEST_remove", "val")
-        bb.data.update_data(self.d)
         self.assertEqual(self.d.getVar("TEST"), "")
 
     def test_remove_expansion(self):
         self.d.setVar("BAR", "Z")
         self.d.setVar("TEST", "${BAR}/X Y")
         self.d.setVar("TEST_remove", "${BAR}/X")
-        bb.data.update_data(self.d)
         self.assertEqual(self.d.getVar("TEST"), "Y")
 
     def test_remove_expansion_items(self):
         self.d.setVar("TEST", "A B C D")
         self.d.setVar("BAR", "B D")
         self.d.setVar("TEST_remove", "${BAR}")
-        bb.data.update_data(self.d)
         self.assertEqual(self.d.getVar("TEST"), "A C")
 
 class TestOverrides(unittest.TestCase):
@@ -322,17 +320,15 @@ class TestOverrides(unittest.TestCase):
         self.d.setVar("TEST", "testvalue")
 
     def test_no_override(self):
-        bb.data.update_data(self.d)
         self.assertEqual(self.d.getVar("TEST"), "testvalue")
 
     def test_one_override(self):
         self.d.setVar("TEST_bar", "testvalue2")
-        bb.data.update_data(self.d)
         self.assertEqual(self.d.getVar("TEST"), "testvalue2")
 
     def test_one_override_unset(self):
         self.d.setVar("TEST2_bar", "testvalue2")
-        bb.data.update_data(self.d)
+
         self.assertEqual(self.d.getVar("TEST2"), "testvalue2")
         self.assertCountEqual(list(self.d.keys()), ['TEST', 'TEST2', 'OVERRIDES', 'TEST2_bar'])
 
@@ -340,18 +336,15 @@ class TestOverrides(unittest.TestCase):
         self.d.setVar("TEST_bar", "testvalue2")
         self.d.setVar("TEST_local", "testvalue3")
         self.d.setVar("TEST_foo", "testvalue4")
-        bb.data.update_data(self.d)
         self.assertEqual(self.d.getVar("TEST"), "testvalue3")
         self.assertCountEqual(list(self.d.keys()), ['TEST', 'TEST_foo', 'OVERRIDES', 'TEST_bar', 'TEST_local'])
 
     def test_multiple_combined_overrides(self):
         self.d.setVar("TEST_local_foo_bar", "testvalue3")
-        bb.data.update_data(self.d)
         self.assertEqual(self.d.getVar("TEST"), "testvalue3")
 
     def test_multiple_overrides_unset(self):
         self.d.setVar("TEST2_local_foo_bar", "testvalue3")
-        bb.data.update_data(self.d)
         self.assertEqual(self.d.getVar("TEST2"), "testvalue3")
 
     def test_keyexpansion_override(self):
@@ -359,14 +352,12 @@ class TestOverrides(unittest.TestCase):
         self.d.setVar("TEST_bar", "testvalue2")
         self.d.setVar("TEST_${LOCAL}", "testvalue3")
         self.d.setVar("TEST_foo", "testvalue4")
-        bb.data.update_data(self.d)
         bb.data.expandKeys(self.d)
         self.assertEqual(self.d.getVar("TEST"), "testvalue3")
 
     def test_rename_override(self):
         self.d.setVar("ALTERNATIVE_ncurses-tools_class-target", "a")
         self.d.setVar("OVERRIDES", "class-target")
-        bb.data.update_data(self.d)
         self.d.renameVar("ALTERNATIVE_ncurses-tools", "ALTERNATIVE_lib32-ncurses-tools")
         self.assertEqual(self.d.getVar("ALTERNATIVE_lib32-ncurses-tools"), "a")
 
@@ -469,26 +460,43 @@ class Serialize(unittest.TestCase):
         self.assertEqual(newd.getVarFlag('HELLO', 'other'), 'planet')
 
 
+# Remote datastore tests
+# These really only test the interface, since in actual usage we have a
+# tinfoil connector that does everything over RPC, and this doesn't test
+# that.
+
+class TestConnector:
+    d = None
+    def __init__(self, d):
+        self.d = d
+    def getVar(self, name):
+        return self.d._findVar(name)
+    def getKeys(self):
+        return set(self.d.keys())
+    def getVarHistory(self, name):
+        return self.d.varhistory.variable(name)
+    def expandPythonRef(self, varname, expr, d):
+        localdata = self.d.createCopy()
+        for key in d.localkeys():
+            localdata.setVar(d.getVar(key))
+        varparse = bb.data_smart.VariableParse(varname, localdata)
+        return varparse.python_sub(expr)
+    def setVar(self, name, value):
+        self.d.setVar(name, value)
+    def setVarFlag(self, name, flag, value):
+        self.d.setVarFlag(name, flag, value)
+    def delVar(self, name):
+        self.d.delVar(name)
+        return False
+    def delVarFlag(self, name, flag):
+        self.d.delVarFlag(name, flag)
+        return False
+    def renameVar(self, name, newname):
+        self.d.renameVar(name, newname)
+        return False
+
 class Remote(unittest.TestCase):
     def test_remote(self):
-        class TestConnector:
-            d = None
-            def __init__(self, d):
-                self.d = d
-            def getVar(self, name):
-                return self.d._findVar(name)
-            def getKeys(self):
-                return self.d.localkeys()
-            def getVarHistory(self, name):
-                return self.d.varhistory.variable(name)
-            def expandPythonRef(self, varname, expr, d):
-                localdata = self.d.createCopy()
-                for key in d.localkeys():
-                    localdata.setVar(d.getVar(key))
-                varparse = bb.data_smart.VariableParse(varname, localdata)
-                return varparse.python_sub(expr)
-            def setVar(self, name, value):
-                self.d.setVar(name, value)
 
         d1 = bb.data.init()
         d1.enableTracking()
@@ -509,6 +517,85 @@ class Remote(unittest.TestCase):
         # Test setVar on client side affects server
         d2.setVar('HELLO', 'other-world')
         self.assertEqual(d1.getVar('HELLO'), 'other-world')
+        # Test setVarFlag on client side affects server
+        d2.setVarFlag('HELLO', 'flagname', 'flagvalue')
+        self.assertEqual(d1.getVarFlag('HELLO', 'flagname'), 'flagvalue')
         # Test client side data is incorporated in python expansion (which is done on server)
         d2.setVar('FOO', 'bar')
         self.assertEqual(d2.expand('${@d.getVar("FOO")}'), 'bar')
+        # Test overrides work
+        d1.setVar('FOO_test', 'baz')
+        d1.appendVar('OVERRIDES', ':test')
+        self.assertEqual(d2.getVar('FOO'), 'baz')
+
+
+# Remote equivalents of local test classes
+# Note that these aren't perfect since we only test in one direction
+
+class RemoteDataExpansions(DataExpansions):
+    def setUp(self):
+        self.d1 = bb.data.init()
+        self.d = bb.data.init()
+        self.d1["foo"] = "value_of_foo"
+        self.d1["bar"] = "value_of_bar"
+        self.d1["value_of_foo"] = "value_of_'value_of_foo'"
+        connector = TestConnector(self.d1)
+        self.d.setVar('_remote_data', connector)
+
+class TestRemoteNestedExpansions(TestNestedExpansions):
+    def setUp(self):
+        self.d1 = bb.data.init()
+        self.d = bb.data.init()
+        self.d1["foo"] = "foo"
+        self.d1["bar"] = "bar"
+        self.d1["value_of_foobar"] = "187"
+        connector = TestConnector(self.d1)
+        self.d.setVar('_remote_data', connector)
+
+class TestRemoteConcat(TestConcat):
+    def setUp(self):
+        self.d1 = bb.data.init()
+        self.d = bb.data.init()
+        self.d1.setVar("FOO", "foo")
+        self.d1.setVar("VAL", "val")
+        self.d1.setVar("BAR", "bar")
+        connector = TestConnector(self.d1)
+        self.d.setVar('_remote_data', connector)
+
+class TestRemoteConcatOverride(TestConcatOverride):
+    def setUp(self):
+        self.d1 = bb.data.init()
+        self.d = bb.data.init()
+        self.d1.setVar("FOO", "foo")
+        self.d1.setVar("VAL", "val")
+        self.d1.setVar("BAR", "bar")
+        connector = TestConnector(self.d1)
+        self.d.setVar('_remote_data', connector)
+
+class TestRemoteOverrides(TestOverrides):
+    def setUp(self):
+        self.d1 = bb.data.init()
+        self.d = bb.data.init()
+        self.d1.setVar("OVERRIDES", "foo:bar:local")
+        self.d1.setVar("TEST", "testvalue")
+        connector = TestConnector(self.d1)
+        self.d.setVar('_remote_data', connector)
+
+class TestRemoteKeyExpansion(TestKeyExpansion):
+    def setUp(self):
+        self.d1 = bb.data.init()
+        self.d = bb.data.init()
+        self.d1.setVar("FOO", "foo")
+        self.d1.setVar("BAR", "foo")
+        connector = TestConnector(self.d1)
+        self.d.setVar('_remote_data', connector)
+
+class TestRemoteFlags(TestFlags):
+    def setUp(self):
+        self.d1 = bb.data.init()
+        self.d = bb.data.init()
+        self.d1.setVar("foo", "value of foo")
+        self.d1.setVarFlag("foo", "flag1", "value of flag1")
+        self.d1.setVarFlag("foo", "flag2", "value of flag2")
+        connector = TestConnector(self.d1)
+        self.d.setVar('_remote_data', connector)

bitbake/lib/bb/tinfoil.py

@@ -1,6 +1,6 @@
 # tinfoil: a simple wrapper around cooker for bitbake-based command-line utilities
 #
-# Copyright (C) 2012-2016 Intel Corporation
+# Copyright (C) 2012-2017 Intel Corporation
 # Copyright (C) 2011 Mentor Graphics Corporation
 #
 # This program is free software; you can redistribute it and/or modify
@@ -60,12 +60,15 @@ class TinfoilDataStoreConnector:
         self.dsindex = dsindex
     def getVar(self, name):
         value = self.tinfoil.run_command('dataStoreConnectorFindVar', self.dsindex, name)
+        overrides = None
         if isinstance(value, dict):
             if '_connector_origtype' in value:
                 value['_content'] = self.tinfoil._reconvert_type(value['_content'], value['_connector_origtype'])
                 del value['_connector_origtype']
-
-        return value
+            if '_connector_overrides' in value:
+                overrides = value['_connector_overrides']
+                del value['_connector_overrides']
+        return value, overrides
     def getKeys(self):
         return set(self.tinfoil.run_command('dataStoreConnectorGetKeys', self.dsindex))
     def getVarHistory(self, name):
@@ -81,6 +84,34 @@ class TinfoilDataStoreConnector:
             # Not currently implemented - indicate that setting should
             # be redirected to local side
             return True
+    def setVarFlag(self, varname, flagname, value):
+        if self.dsindex is None:
+            self.tinfoil.run_command('dataStoreConnectorSetVarFlag', self.dsindex, varname, flagname, value)
+        else:
+            # Not currently implemented - indicate that setting should
+            # be redirected to local side
+            return True
+    def delVar(self, varname):
+        if self.dsindex is None:
+            self.tinfoil.run_command('dataStoreConnectorDelVar', self.dsindex, varname)
+        else:
+            # Not currently implemented - indicate that setting should
+            # be redirected to local side
+            return True
+    def delVarFlag(self, varname, flagname):
+        if self.dsindex is None:
+            self.tinfoil.run_command('dataStoreConnectorDelVar', self.dsindex, varname, flagname)
+        else:
+            # Not currently implemented - indicate that setting should
+            # be redirected to local side
+            return True
+    def renameVar(self, name, newname):
+        if self.dsindex is None:
+            self.tinfoil.run_command('dataStoreConnectorRenameVar', self.dsindex, name, newname)
+        else:
+            # Not currently implemented - indicate that setting should
+            # be redirected to local side
+            return True
 
 class TinfoilCookerAdapter:
     """
@@ -211,6 +242,19 @@ class Tinfoil:
 
         self.ui_module = ui_module
 
+        # Ensure the path to bitbake's bin directory is in PATH so that things like
+        # bitbake-worker can be run (usually this is the case, but it doesn't have to be)
+        path = os.getenv('PATH').split(':')
+        bitbakebinpath = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', 'bin'))
+        for entry in path:
+            if entry.endswith(os.sep):
+                entry = entry[:-1]
+            if os.path.abspath(entry) == bitbakebinpath:
+                break
+        else:
+            path.insert(0, bitbakebinpath)
+            os.environ['PATH'] = ':'.join(path)
+
         if self.server_connection:
             _server_connections.append(self.server_connection)
             if config_only:

bitbake/lib/bb/ui/buildinfohelper.py

@@ -1258,6 +1258,14 @@ class BuildInfoHelper(object):
                 candidates = [x for x in self.internal_state['taskdata'].keys() if x.endswith(identifier)]
                 if len(candidates) == 1:
                     identifier = candidates[0]
+                elif len(candidates) > 1 and hasattr(event,'_package'):
+                    if 'native-' in event._package:
+                        identifier = 'native:' + identifier
+                    if 'nativesdk-' in event._package:
+                        identifier = 'nativesdk:' + identifier
+                    candidates = [x for x in self.internal_state['taskdata'].keys() if x.endswith(identifier)]
+                    if len(candidates) == 1:
+                        identifier = candidates[0]
 
         assert identifier in self.internal_state['taskdata']
         identifierlist = identifier.split(":")

bitbake/lib/bb/ui/knotty.py

@@ -75,10 +75,8 @@ class BBProgress(progressbar.ProgressBar):
                 extrastr = str(extra)
                 if extrastr[0] != ' ':
                     extrastr = ' ' + extrastr
-                if extrastr[-1] != ' ':
-                    extrastr += ' '
             else:
-                extrastr = ' '
+                extrastr = ''
             self.widgets[self.extrapos] = extrastr
 
     def _need_update(self):

bitbake/lib/bb/ui/taskexp.py

@@ -78,7 +78,7 @@ class PackageReverseDepView(Gtk.TreeView):
 class DepExplorer(Gtk.Window):
     def __init__(self):
         Gtk.Window.__init__(self)
-        self.set_title("Dependency Explorer")
+        self.set_title("Task Dependency Explorer")
         self.set_default_size(500, 500)
         self.connect("delete-event", Gtk.main_quit)
 
@@ -106,30 +106,21 @@ class DepExplorer(Gtk.Window):
 
         box = Gtk.VBox(homogeneous=True, spacing=4)
 
-        # Runtime Depends
+        # Task Depends
         scrolled = Gtk.ScrolledWindow()
         scrolled.set_policy(Gtk.PolicyType.AUTOMATIC, Gtk.PolicyType.AUTOMATIC)
         scrolled.set_shadow_type(Gtk.ShadowType.IN)
-        self.rdep_treeview = PackageDepView(self.depends_model, TYPE_RDEP, "Runtime Depends")
-        self.rdep_treeview.connect("row-activated", self.on_package_activated, COL_DEP_PACKAGE)
-        scrolled.add(self.rdep_treeview)
-        box.add(scrolled)
-
-        # Build Depends
-        scrolled = Gtk.ScrolledWindow()
-        scrolled.set_policy(Gtk.PolicyType.AUTOMATIC, Gtk.PolicyType.AUTOMATIC)
-        scrolled.set_shadow_type(Gtk.ShadowType.IN)
-        self.dep_treeview = PackageDepView(self.depends_model, TYPE_DEP, "Build Depends")
+        self.dep_treeview = PackageDepView(self.depends_model, TYPE_DEP, "Dependencies")
         self.dep_treeview.connect("row-activated", self.on_package_activated, COL_DEP_PACKAGE)
         scrolled.add(self.dep_treeview)
         box.add(scrolled)
         pane.add2(box)
 
-        # Reverse Depends
+        # Reverse Task Depends
         scrolled = Gtk.ScrolledWindow()
         scrolled.set_policy(Gtk.PolicyType.AUTOMATIC, Gtk.PolicyType.AUTOMATIC)
         scrolled.set_shadow_type(Gtk.ShadowType.IN)
-        self.revdep_treeview = PackageReverseDepView(self.depends_model, "Reverse Depends")
+        self.revdep_treeview = PackageReverseDepView(self.depends_model, "Dependent Tasks")
         self.revdep_treeview.connect("row-activated", self.on_package_activated, COL_DEP_PARENT)
         scrolled.add(self.revdep_treeview)
         box.add(scrolled)
@@ -160,22 +151,15 @@ class DepExplorer(Gtk.Window):
             current_package = None
         else:
             current_package = model.get_value(it, COL_PKG_NAME)
-        self.rdep_treeview.set_current_package(current_package)
         self.dep_treeview.set_current_package(current_package)
         self.revdep_treeview.set_current_package(current_package)
 
 
     def parse(self, depgraph):
-        for package in depgraph["pn"]:
-            self.pkg_model.insert(0, (package,))
-
-        for package in depgraph["depends"]:
-            for depend in depgraph["depends"][package]:
-                self.depends_model.insert (0, (TYPE_DEP, package, depend))
-
-        for package in depgraph["rdepends-pn"]:
-            for rdepend in depgraph["rdepends-pn"][package]:
-                self.depends_model.insert (0, (TYPE_RDEP, package, rdepend))
+        for task in depgraph["tdepends"]:
+            self.pkg_model.insert(0, (task,))
+            for depend in depgraph["tdepends"][task]:
+                self.depends_model.insert (0, (TYPE_DEP, task, depend))
 
 
 class gtkthread(threading.Thread):

bitbake/lib/bb/utils.py

@@ -979,6 +979,30 @@ def contains_any(variable, checkvalues, truevalue, falsevalue, d):
         return truevalue
     return falsevalue
 
+def filter(variable, checkvalues, d):
+    """Return all words in the variable that are present in the checkvalues.
+
+    Arguments:
+
+    variable -- the variable name. This will be fetched and expanded (using
+    d.getVar(variable)) and then split into a set().
+
+    checkvalues -- if this is a string it is split on whitespace into a set(),
+    otherwise coerced directly into a set().
+
+    d -- the data store.
+    """
+
+    val = d.getVar(variable)
+    if not val:
+        return ''
+    val = set(val.split())
+    if isinstance(checkvalues, str):
+        checkvalues = set(checkvalues.split())
+    else:
+        checkvalues = set(checkvalues)
+    return ' '.join(sorted(checkvalues & val))
+
 def cpu_count():
     return multiprocessing.cpu_count()
 

bitbake/lib/bblayers/layerindex.py

@@ -56,7 +56,7 @@ class LayerIndexPlugin(ActionPlugin):
         r = conn.getresponse()
         if r.status != 200:
             raise Exception("Failed to read " + path + ": %d %s" % (r.status, r.reason))
-        return json.loads(r.read())
+        return json.loads(r.read().decode())
 
     def get_layer_deps(self, layername, layeritems, layerbranches, layerdependencies, branchnum, selfname=False):
         def layeritems_info_id(items_name, layeritems):

bitbake/lib/toaster/orm/models.py

@@ -902,7 +902,7 @@ class Target_Image_File(models.Model):
         'ext4.gz', 'ext3', 'ext3.gz', 'hdddirect', 'hddimg', 'iso', 'jffs2',
         'jffs2.sum', 'multiubi', 'qcow2', 'squashfs', 'squashfs-lzo',
         'squashfs-xz', 'tar', 'tar.bz2', 'tar.gz', 'tar.lz4', 'tar.xz', 'ubi',
-        'ubifs', 'vdi', 'vmdk', 'wic', 'wic.bz2', 'wic.gz', 'wic.lzma'
+        'ubifs', 'vdi', 'vmdk', 'wic', 'wic.bmap', 'wic.bz2', 'wic.gz', 'wic.lzma'
     }
 
     target = models.ForeignKey(Target)

bitbake/lib/toaster/toastermain/settings.py

@@ -38,8 +38,7 @@ ADMINS = (
 
 MANAGERS = ADMINS
 
-TOASTER_SQLITE_DEFAULT_DIR = os.path.join(os.environ.get('TOASTER_DIR', ''),
-                                          'build')
+TOASTER_SQLITE_DEFAULT_DIR = os.environ.get('TOASTER_DIR')
 
 DATABASES = {
     'default': {

documentation/bsp-guide/bsp-guide.xml

@@ -115,7 +115,7 @@
             </revision>
             <revision>
                 <revnumber>2.3</revnumber>
-                <date>Proposed for April 2017</date>
+                <date>May 2017</date>
                 <revremark>Released with the Yocto Project 2.3 Release.</revremark>
             </revision>
         </revhistory>
@@ -130,12 +130,34 @@
         Permission is granted to copy, distribute and/or modify this document under
         the terms of the <ulink type="http" url="http://creativecommons.org/licenses/by-nc-sa/2.0/uk/">Creative Commons Attribution-Share Alike 2.0 UK: England &amp; Wales</ulink> as published by Creative Commons.
       </para>
-      <note>
-          For the latest version of this manual associated with this
-          Yocto Project release, see the
-          <ulink url='&YOCTO_DOCS_BSP_URL;'>Yocto Project Board Support Package (BSP) Developer's Guide</ulink>
-          from the Yocto Project website.
-      </note>
+            <note><title>Manual Notes</title>
+                <itemizedlist>
+                    <listitem><para>
+                        For the latest version of the Yocto Project Board
+                        Support Package (BSP) Developer's Guide associated with
+                        this Yocto Project release (version
+                        &YOCTO_DOC_VERSION;),
+                        see the Yocto Project Board Support Package (BSP)
+                        Developer's Guide from the
+                        <ulink url='&YOCTO_HOME_URL;/documentation'>Yocto Project documentation page</ulink>.
+                        </para></listitem>
+                    <listitem><para>
+                        This version of the manual is version
+                        &YOCTO_DOC_VERSION;.
+                        For later releases of the Yocto Project (if they exist),
+                        go to the
+                        <ulink url='&YOCTO_HOME_URL;/documentation'>Yocto Project documentation page</ulink>
+                        and use the drop-down "Active Releases" button
+                        and choose the Yocto Project version for which you want
+                        the manual.
+                        </para></listitem>
+                    <listitem><para>
+                        For an in-development version of the Yocto Project
+                        Board Support Package (BSP) Developer's Guide, see
+                        <ulink url='&YOCTO_DOCS_URL;/latest/bsp-guide/bsp-guide.html'></ulink>.
+                        </para></listitem>
+                </itemizedlist>
+            </note>
     </legalnotice>
 
     </bookinfo>

documentation/dev-manual/dev-manual-common-tasks.xml

@@ -45,6 +45,10 @@
             You can accomplish this through a recipe that is a BitBake append
             (<filename>.bbappend</filename>) file, which is described later
             in this section.
+            <note>
+                For general information on BSP layer structure, see the
+                <ulink url='&YOCTO_DOCS_BSP_URL;#bsp'>Board Support Packages (BSP) - Developer's Guide</ulink>.
+            </note>
         </para>
 
         <para>
@@ -4475,7 +4479,7 @@
                         <listitem><para>
                             Wic is a completely independent
                             standalone utility that initially provides
-                            easier-to-use and more flexible replacements for a
+                            easier-to-use and more flexible replacements for an
                             existing functionality in OE Core's
                             <ulink url='&YOCTO_DOCS_REF_URL;#ref-classes-image-live'><filename>image-live</filename></ulink>
                             class and <filename>mkefidisk.sh</filename> script.
@@ -5038,6 +5042,14 @@
                     <filename>--source</filename> keyword to a
                     particular plug-in implementation that populates a
                     corresponding partition.
+                    <note>
+                        If you use plug-ins that have build-time dependencies
+                        (e.g. native tools, bootloaders, and so forth)
+                        when building a Wic image, you need to specify those
+                        dependencies using the
+                        <ulink url='&YOCTO_DOCS_REF_URL;#var-WKS_FILE_DEPENDS'><filename>WKS_FILE_DEPENDS</filename></ulink>
+                        variable.
+                    </note>
                 </para>
 
                 <para>
@@ -5232,9 +5244,7 @@
                         <itemizedlist>
                             <listitem><para>
                                 <filename>/<replaceable>path</replaceable></filename>:
-                                For example, <filename>/</filename>,
-                                <filename>/usr</filename>, or
-                                <filename>/home</filename>
+                                For example, "/", "/usr", or "/home"
                                 </para></listitem>
                             <listitem><para>
                                 <filename>swap</filename>:
@@ -8372,6 +8382,127 @@
             </section>
         </section>
 
+        <section id='generating-and-using-signed-packages'>
+            <title>Generating and Using Signed Packages</title>
+            <para>
+                In order to add security to RPM packages used during a build,
+                you can take steps to securely sign them.
+                Once a signature is verified, the OpenEmbedded build system
+                can use the package in the build.
+                If security fails for a signed package, the build system
+                aborts the build.
+            </para>
+
+            <para>
+                This section describes how to sign RPM packages during a build
+                and how to use signed package feeds (repositories) when
+                doing a build.
+            </para>
+
+            <section id='signing-rpm-packages'>
+                <title>Signing RPM Packages</title>
+
+                <para>
+                    To enable signing RPM packages, you must set up the
+                    following configurations in either your
+                    <filename>local.config</filename> or
+                    <filename>distro.config</filename> file:
+                    <literallayout class='monospaced'>
+     # Inherit sign_rpm.bbclass to enable signing functionality
+     INHERIT += " sign_rpm"
+     # Define the GPG key that will be used for signing.
+     RPM_GPG_NAME = "<replaceable>key_name</replaceable>"
+     # Provide passphrase for the key
+     RPM_GPG_PASSPHRASE = "<replaceable>passphrase</replaceable>"
+                    </literallayout>
+                    <note>
+                        Be sure to supply appropriate values for both
+                        <replaceable>key_name</replaceable> and
+                        <replaceable>passphrase</replaceable>
+                    </note>
+                    Aside from the
+                    <filename>RPM_GPG_NAME</filename> and
+                    <filename>RPM_GPG_PASSPHRASE</filename> variables in the
+                    previous example, two optional variables related to signing
+                    exist:
+                    <itemizedlist>
+                        <listitem><para>
+                            <emphasis><filename>GPG_BIN</filename>:</emphasis>
+                            Specifies a <filename>gpg</filename> binary/wrapper
+                            that is executed when the package is signed.
+                            </para></listitem>
+                        <listitem><para>
+                            <emphasis><filename>GPG_PATH</filename>:</emphasis>
+                            Specifies the <filename>gpg</filename> home
+                            directory used when the package is signed.
+                            </para></listitem>
+                    </itemizedlist>
+                </para>
+            </section>
+
+            <section id='processing-package-feeds'>
+                <title>Processing Package Feeds</title>
+
+                <para>
+                    In addition to being able to sign RPM packages, you can
+                    also enable the OpenEmbedded build system to be able to
+                    handle previously signed package feeds for both RPM and IPK
+                    packages.
+                    <note>
+                        The OpenEmbedded build system does not currently
+                        support signed DPKG package feeds.
+                    </note>
+                    The steps you need to take to enable signed package feed
+                    use are similar to the steps used to sign RPM packages.
+                    You must define the following in your
+                    <filename>local.config</filename> or
+                    <filename>distro.config</filename> file:
+                    <literallayout class='monospaced'>
+     INHERIT += "sign_package_feed"
+     PACKAGE_FEED_GPG_NAME = "<replaceable>key_name</replaceable>"
+     PACKAGE_FEED_GPG_PASSPHRASE_FILE = "<replaceable>path_to_file_containing_passphrase</replaceable>"
+                    </literallayout>
+                    For signed package feeds, the passphrase must exist in a
+                    separate file, which is pointed to by the
+                    <filename>PACKAGE_FEED_GPG_PASSPHRASE_FILE</filename>
+                    variable.
+                    Regarding security, keeping a plain text passphrase out of
+                    the configuration is more secure.
+                </para>
+
+                <para>
+                    Aside from the
+                    <filename>PACKAGE_FEED_GPG_NAME</filename> and
+                    <filename>PACKAGE_FEED_GPG_PASSPHRASE_FILE</filename>
+                    variables, three optional variables related to signed
+                    package feeds exist:
+                    <itemizedlist>
+                        <listitem><para>
+                            <emphasis><filename>GPG_BIN</filename>:</emphasis>
+                            Specifies a <filename>gpg</filename> binary/wrapper
+                            that is executed when the package is signed.
+                            </para></listitem>
+                        <listitem><para>
+                            <emphasis><filename>GPG_PATH</filename>:</emphasis>
+                            Specifies the <filename>gpg</filename> home
+                            directory used when the package is signed.
+                            </para></listitem>
+                        <listitem><para>
+                            <emphasis><filename>PACKAGE_FEED_GPG_SIGNATURE_TYPE</filename>:</emphasis>
+                            Specifies the type of <filename>gpg</filename>
+                            signature.
+                            This variable applies only to RPM and IPK package
+                            feeds.
+                            Allowable values for the
+                            <filename>PACKAGE_FEED_GPG_SIGNATURE_TYPE</filename>
+                            are "ASC", which is the default and specifies ascii
+                            armored, and "BIN", which specifies binary.
+                            </para></listitem>
+                    </itemizedlist>
+                </para>
+            </section>
+        </section>
+
         <section id='testing-packages-with-ptest'>
             <title>Testing Packages With ptest</title>
 
@@ -10884,10 +11015,11 @@
 
             <para>
                 Before you employ <filename>DL_DIR</filename> or the
-                archiver class, you need to decide how you choose to
-                provide source.
-                The source archiver class can generate tarballs and SRPMs
-                and can create them with various levels of compliance in mind.
+                <filename>archiver</filename> class, you need to decide how
+                you choose to provide source.
+                The source <filename>archiver</filename> class can generate
+                tarballs and SRPMs and can create them with various levels of
+                compliance in mind.
             </para>
 
             <para>
@@ -10982,9 +11114,9 @@
                     <ulink url='&YOCTO_DOCS_REF_URL;#var-COPY_LIC_DIRS'><filename>COPY_LIC_DIRS</filename></ulink>
                     and
                     <ulink url='&YOCTO_DOCS_REF_URL;#var-COPY_LIC_MANIFEST'><filename>COPY_LIC_MANIFEST</filename></ulink>
-                    add a copy of the license when the image is built but do not
-                    offer a path for adding licenses for newly installed packages
-                    to an image.
+                    add a copy of the license when the image is built but do
+                    not offer a path for adding licenses for newly installed
+                    packages to an image.
                     <ulink url='&YOCTO_DOCS_REF_URL;#var-LICENSE_CREATE_PACKAGE'><filename>LICENSE_CREATE_PACKAGE</filename></ulink>
                     adds a separate package and an upgrade path for adding
                     licenses to an image.</para>
@@ -10992,7 +11124,8 @@
             </para>
 
             <para>
-                As the source archiver has already archived the original
+                As the source <filename>archiver</filename> class has already
+                archived the original
                 unmodified source that contains the license files,
                 you would have already met the requirements for inclusion
                 of the license information with source as defined by the GPL
@@ -11004,7 +11137,7 @@
             <title>Providing Compilation Scripts and Source Code Modifications</title>
 
             <para>
-                At this point, we have addressed all we need to address
+                At this point, we have addressed all we need to
                 prior to generating the image.
                 The next two requirements are addressed during the final
                 packaging of the release.
@@ -11023,7 +11156,7 @@
                 and a distro layer, and those those layers are used to patch,
                 compile, package, or modify (in any way) any open source
                 software included in your released images, you
-                might be required to to release those layers under section 3 of
+                might be required to release those layers under section 3 of
                 GPLv2 or section 1 of GPLv3.
                 One way of doing that is with a clean
                 checkout of the version of the Yocto Project and layers used

documentation/dev-manual/dev-manual.xml

@@ -93,7 +93,7 @@
             </revision>
             <revision>
                 <revnumber>2.3</revnumber>
-                <date>Proposed for April 2017</date>
+                <date>May 2017</date>
                 <revremark>Released with the Yocto Project 2.3 Release.</revremark>
             </revision>
         </revhistory>
@@ -111,12 +111,32 @@
           Creative Commons.
       </para>
 
-      <note>
-          For the latest version of this manual associated with this
-          Yocto Project release, see the
-          <ulink url='&YOCTO_DOCS_DEV_URL;'>Yocto Project Development Manual</ulink>
-          from the Yocto Project website.
-      </note>
+            <note><title>Manual Notes</title>
+                <itemizedlist>
+                    <listitem><para>
+                        For the latest version of the Yocto Project Development
+                        Manual associated with this Yocto Project release
+                        (version &YOCTO_DOC_VERSION;),
+                        see the Yocto Project Development Manual from the
+                        <ulink url='&YOCTO_HOME_URL;/documentation'>Yocto Project documentation page</ulink>.
+                        </para></listitem>
+                    <listitem><para>
+                        This version of the manual is version
+                        &YOCTO_DOC_VERSION;.
+                        For later releases of the Yocto Project (if they exist),
+                        go to the
+                        <ulink url='&YOCTO_HOME_URL;/documentation'>Yocto Project documentation page</ulink>
+                        and use the drop-down "Active Releases" button
+                        and choose the Yocto Project version for which you want
+                        the manual.
+                        </para></listitem>
+                    <listitem><para>
+                        For an in-development version of the Yocto Project
+                        Development Manual, see
+                        <ulink url='&YOCTO_DOCS_URL;/latest/dev-manual/dev-manual.html'></ulink>.
+                        </para></listitem>
+                </itemizedlist>
+            </note>
     </legalnotice>
 
     </bookinfo>

documentation/kernel-dev/kernel-dev-advanced.xml

@@ -83,13 +83,14 @@
         The linux-yocto style recipes can optionally define the following
         variables:
         <literallayout class='monospaced'>
-     <ulink url='&YOCTO_DOCS_REF_URL;#var-KERNEL_FEATURES'>KERNEL_FEATURES</ulink>
-     <ulink url='&YOCTO_DOCS_REF_URL;#var-LINUX_KERNEL_TYPE'>LINUX_KERNEL_TYPE</ulink>
+     KERNEL_FEATURES
+     LINUX_KERNEL_TYPE
         </literallayout>
     </para>
 
     <para>
-        <filename>LINUX_KERNEL_TYPE</filename> defines the kernel type to be
+        <ulink url='&YOCTO_DOCS_REF_URL;#var-LINUX_KERNEL_TYPE'><filename>LINUX_KERNEL_TYPE</filename></ulink>
+        defines the kernel type to be
         used in assembling the configuration.
         If you do not specify a <filename>LINUX_KERNEL_TYPE</filename>,
         it defaults to "standard".
@@ -134,7 +135,9 @@
     </para>
 
     <para>
-        You can use the <filename>KERNEL_FEATURES</filename> variable
+        You can use the
+        <ulink url='&YOCTO_DOCS_REF_URL;#var-KERNEL_FEATURES'><filename>KERNEL_FEATURES</filename></ulink>
+        variable
         to include features (configuration fragments, patches, or both) that
         are not already included by the <filename>KMACHINE</filename> and
         <filename>LINUX_KERNEL_TYPE</filename> variable combination.
@@ -167,175 +170,6 @@
     </para>
 </section>
 
-<section id='kernel-metadata-location'>
-    <title>Kernel Metadata Location</title>
-
-    <para>
-        Kernel Metadata always exists outside of the kernel tree either
-        defined in a kernel recipe (recipe-space) or outside of the recipe.
-        Where you choose to define the Metadata depends on what you want
-        to do and how you intend to work.
-        Regardless of where you define the kernel Metadata, the syntax used
-        applies equally.
-    </para>
-
-    <para>
-        If you are unfamiliar with the Linux kernel and only wish
-        to apply a configuration and possibly a couple of patches provided to
-        you by others, the recipe-space method is recommended.
-        This method is also a good approach if you are working with Linux kernel
-        sources you do not control or if you just do not want to maintain a
-        Linux kernel Git repository on your own.
-        For partial information on how you can define kernel Metadata in
-        the recipe-space, see the
-        "<link linkend='modifying-an-existing-recipe'>Modifying an Existing Recipe</link>"
-        section.
-    </para>
-
-    <para>
-        Conversely, if you are actively developing a kernel and are already
-        maintaining a Linux kernel Git repository of your own, you might find
-        it more convenient to work with kernel Metadata kept outside the
-        recipe-space.
-        Working with Metadata in this area can make iterative development of
-        the Linux kernel more efficient outside of the BitBake environment.
-    </para>
-
-    <section id='recipe-space-metadata'>
-        <title>Recipe-Space Metadata</title>
-
-        <para>
-            When stored in recipe-space, the kernel Metadata files reside in a
-            directory hierarchy below
-            <ulink url='&YOCTO_DOCS_REF_URL;#var-FILESEXTRAPATHS'><filename>FILESEXTRAPATHS</filename></ulink>.
-            For a linux-yocto recipe or for a Linux kernel recipe derived
-            by copying and modifying
-            <filename>oe-core/meta-skeleton/recipes-kernel/linux/linux-yocto-custom.bb</filename>
-            to a recipe in your layer, <filename>FILESEXTRAPATHS</filename>
-            is typically set to
-            <filename>${</filename><ulink url='&YOCTO_DOCS_REF_URL;#var-THISDIR'><filename>THISDIR</filename></ulink><filename>}/${</filename><ulink url='&YOCTO_DOCS_REF_URL;#var-PN'><filename>PN</filename></ulink><filename>}</filename>.
-            See the "<link linkend='modifying-an-existing-recipe'>Modifying an Existing Recipe</link>"
-            section for more information.
-        </para>
-
-        <para>
-            Here is an example that shows a trivial tree of kernel Metadata
-            stored in recipe-space within a BSP layer:
-            <literallayout class='monospaced'>
-     meta-<replaceable>my_bsp_layer</replaceable>/
-     `-- recipes-kernel
-         `-- linux
-             `-- linux-yocto
-                 |-- bsp-standard.scc
-                 |-- bsp.cfg
-                 `-- standard.cfg
-            </literallayout>
-        </para>
-
-        <para>
-            When the Metadata is stored in recipe-space, you must take
-            steps to ensure BitBake has the necessary information to decide
-            what files to fetch and when they need to be fetched again.
-            It is only necessary to specify the <filename>.scc</filename>
-            files on the
-            <ulink url='&YOCTO_DOCS_REF_URL;#var-SRC_URI'><filename>SRC_URI</filename></ulink>.
-            BitBake parses them and fetches any files referenced in the
-            <filename>.scc</filename> files by the <filename>include</filename>,
-            <filename>patch</filename>, or <filename>kconf</filename> commands.
-            Because of this, it is necessary to bump the recipe
-            <ulink url='&YOCTO_DOCS_REF_URL;#var-PR'><filename>PR</filename></ulink>
-            value when changing the content of files not explicitly listed
-            in the <filename>SRC_URI</filename>.
-        </para>
-    </section>
-
-    <section id='metadata-outside-the-recipe-space'>
-        <title>Metadata Outside the Recipe-Space</title>
-
-        <para>
-            When stored outside of the recipe-space, the kernel Metadata
-            files reside in a separate repository.
-            The OpenEmbedded build system adds the Metadata to the build as
-            a "ktype=meta" repository through the
-            <ulink url='&YOCTO_DOCS_REF_URL;#var-SRC_URI'><filename>SRC_URI</filename></ulink>
-            variable.
-            As an example, consider the following <filename>SRC_URI</filename>
-            statement from the <filename>linux-yocto_4.4.bb</filename>
-            kernel recipe:
-            <literallayout class='monospaced'>
-     SRC_URI = "git://git.yoctoproject.org/linux-yocto-4.4.git;name=machine;branch=${KBRANCH}; \
-                git://git.yoctoproject.org/yocto-kernel-cache;type=kmeta;name=meta;branch=yocto-4.4;destsuffix=${KMETA}"
-            </literallayout>
-            <filename>${KMETA}</filename>, in this context, is simply used to
-            name the directory into which the Git fetcher places the Metadata.
-            This behavior is no different than any multi-repository
-            <filename>SRC_URI</filename> statement used in a recipe.
-        </para>
-
-        <para>
-            You can keep kernel Metadata in a "kernel-cache", which is a
-            directory containing configuration fragments.
-            As with any Metadata kept outside the recipe-space, you simply
-            need to use the <filename>SRC_URI</filename> statement with the
-            "type=kmeta" attribute.
-            Doing so makes the kernel Metadata available during the
-            configuration phase.
-        </para>
-
-<!--
-
-
-        <para>
-            Following is an example that shows how a trivial tree of Metadata
-            is stored in a custom Linux kernel Git repository:
-            <literallayout class='monospaced'>
-     meta/
-     `&dash;&dash; cfg
-         `&dash;&dash; kernel-cache
-             |&dash;&dash; bsp-standard.scc
-             |&dash;&dash; bsp.cfg
-             `&dash;&dash; standard.cfg
-            </literallayout>
-        </para>
-
-        <para>
-            To use a branch different from where the sources reside,
-            specify the branch in the <filename>KMETA</filename> variable
-            in your Linux kernel recipe.
-            Here is an example:
-            <literallayout class='monospaced'>
-     KMETA = "meta"
-            </literallayout>
-            To use the same branch as the sources, set
-            <filename>KMETA</filename> to an empty string:
-            <literallayout class='monospaced'>
-     KMETA = ""
-            </literallayout>
-            If you are working with your own sources and want to create an
-            orphan <filename>meta</filename> branch, use these commands
-            from within your Linux kernel Git repository:
-            <literallayout class='monospaced'>
-     $ git checkout &dash;&dash;orphan meta
-     $ git rm -rf .
-     $ git commit &dash;&dash;allow-empty -m "Create orphan meta branch"
-            </literallayout>
-        </para>
--->
-
-        <para>
-            If you modify the Metadata, you must not forget to update the
-            <ulink url='&YOCTO_DOCS_REF_URL;#var-SRCREV'><filename>SRCREV</filename></ulink>
-            statements in the kernel's recipe.
-            In particular, you need to update the
-            <filename>SRCREV_meta</filename> variable to match the commit in
-            the <filename>KMETA</filename> branch you wish to use.
-            Changing the data in these branches and not updating the
-            <filename>SRCREV</filename> statements to match will cause the
-            build to fetch an older commit.
-        </para>
-    </section>
-</section>
-
 <section id='kernel-metadata-syntax'>
     <title>Kernel Metadata Syntax</title>
 
@@ -881,6 +715,175 @@
     </section>
 </section>
 
+<section id='kernel-metadata-location'>
+    <title>Kernel Metadata Location</title>
+
+    <para>
+        Kernel Metadata always exists outside of the kernel tree either
+        defined in a kernel recipe (recipe-space) or outside of the recipe.
+        Where you choose to define the Metadata depends on what you want
+        to do and how you intend to work.
+        Regardless of where you define the kernel Metadata, the syntax used
+        applies equally.
+    </para>
+
+    <para>
+        If you are unfamiliar with the Linux kernel and only wish
+        to apply a configuration and possibly a couple of patches provided to
+        you by others, the recipe-space method is recommended.
+        This method is also a good approach if you are working with Linux kernel
+        sources you do not control or if you just do not want to maintain a
+        Linux kernel Git repository on your own.
+        For partial information on how you can define kernel Metadata in
+        the recipe-space, see the
+        "<link linkend='modifying-an-existing-recipe'>Modifying an Existing Recipe</link>"
+        section.
+    </para>
+
+    <para>
+        Conversely, if you are actively developing a kernel and are already
+        maintaining a Linux kernel Git repository of your own, you might find
+        it more convenient to work with kernel Metadata kept outside the
+        recipe-space.
+        Working with Metadata in this area can make iterative development of
+        the Linux kernel more efficient outside of the BitBake environment.
+    </para>
+
+    <section id='recipe-space-metadata'>
+        <title>Recipe-Space Metadata</title>
+
+        <para>
+            When stored in recipe-space, the kernel Metadata files reside in a
+            directory hierarchy below
+            <ulink url='&YOCTO_DOCS_REF_URL;#var-FILESEXTRAPATHS'><filename>FILESEXTRAPATHS</filename></ulink>.
+            For a linux-yocto recipe or for a Linux kernel recipe derived
+            by copying and modifying
+            <filename>oe-core/meta-skeleton/recipes-kernel/linux/linux-yocto-custom.bb</filename>
+            to a recipe in your layer, <filename>FILESEXTRAPATHS</filename>
+            is typically set to
+            <filename>${</filename><ulink url='&YOCTO_DOCS_REF_URL;#var-THISDIR'><filename>THISDIR</filename></ulink><filename>}/${</filename><ulink url='&YOCTO_DOCS_REF_URL;#var-PN'><filename>PN</filename></ulink><filename>}</filename>.
+            See the "<link linkend='modifying-an-existing-recipe'>Modifying an Existing Recipe</link>"
+            section for more information.
+        </para>
+
+        <para>
+            Here is an example that shows a trivial tree of kernel Metadata
+            stored in recipe-space within a BSP layer:
+            <literallayout class='monospaced'>
+     meta-<replaceable>my_bsp_layer</replaceable>/
+     `-- recipes-kernel
+         `-- linux
+             `-- linux-yocto
+                 |-- bsp-standard.scc
+                 |-- bsp.cfg
+                 `-- standard.cfg
+            </literallayout>
+        </para>
+
+        <para>
+            When the Metadata is stored in recipe-space, you must take
+            steps to ensure BitBake has the necessary information to decide
+            what files to fetch and when they need to be fetched again.
+            It is only necessary to specify the <filename>.scc</filename>
+            files on the
+            <ulink url='&YOCTO_DOCS_REF_URL;#var-SRC_URI'><filename>SRC_URI</filename></ulink>.
+            BitBake parses them and fetches any files referenced in the
+            <filename>.scc</filename> files by the <filename>include</filename>,
+            <filename>patch</filename>, or <filename>kconf</filename> commands.
+            Because of this, it is necessary to bump the recipe
+            <ulink url='&YOCTO_DOCS_REF_URL;#var-PR'><filename>PR</filename></ulink>
+            value when changing the content of files not explicitly listed
+            in the <filename>SRC_URI</filename>.
+        </para>
+    </section>
+
+    <section id='metadata-outside-the-recipe-space'>
+        <title>Metadata Outside the Recipe-Space</title>
+
+        <para>
+            When stored outside of the recipe-space, the kernel Metadata
+            files reside in a separate repository.
+            The OpenEmbedded build system adds the Metadata to the build as
+            a "ktype=meta" repository through the
+            <ulink url='&YOCTO_DOCS_REF_URL;#var-SRC_URI'><filename>SRC_URI</filename></ulink>
+            variable.
+            As an example, consider the following <filename>SRC_URI</filename>
+            statement from the <filename>linux-yocto_4.4.bb</filename>
+            kernel recipe:
+            <literallayout class='monospaced'>
+     SRC_URI = "git://git.yoctoproject.org/linux-yocto-4.4.git;name=machine;branch=${KBRANCH}; \
+                git://git.yoctoproject.org/yocto-kernel-cache;type=kmeta;name=meta;branch=yocto-4.4;destsuffix=${KMETA}"
+            </literallayout>
+            <filename>${KMETA}</filename>, in this context, is simply used to
+            name the directory into which the Git fetcher places the Metadata.
+            This behavior is no different than any multi-repository
+            <filename>SRC_URI</filename> statement used in a recipe.
+        </para>
+
+        <para>
+            You can keep kernel Metadata in a "kernel-cache", which is a
+            directory containing configuration fragments.
+            As with any Metadata kept outside the recipe-space, you simply
+            need to use the <filename>SRC_URI</filename> statement with the
+            "type=kmeta" attribute.
+            Doing so makes the kernel Metadata available during the
+            configuration phase.
+        </para>
+
+<!--
+
+
+        <para>
+            Following is an example that shows how a trivial tree of Metadata
+            is stored in a custom Linux kernel Git repository:
+            <literallayout class='monospaced'>
+     meta/
+     `&dash;&dash; cfg
+         `&dash;&dash; kernel-cache
+             |&dash;&dash; bsp-standard.scc
+             |&dash;&dash; bsp.cfg
+             `&dash;&dash; standard.cfg
+            </literallayout>
+        </para>
+
+        <para>
+            To use a branch different from where the sources reside,
+            specify the branch in the <filename>KMETA</filename> variable
+            in your Linux kernel recipe.
+            Here is an example:
+            <literallayout class='monospaced'>
+     KMETA = "meta"
+            </literallayout>
+            To use the same branch as the sources, set
+            <filename>KMETA</filename> to an empty string:
+            <literallayout class='monospaced'>
+     KMETA = ""
+            </literallayout>
+            If you are working with your own sources and want to create an
+            orphan <filename>meta</filename> branch, use these commands
+            from within your Linux kernel Git repository:
+            <literallayout class='monospaced'>
+     $ git checkout &dash;&dash;orphan meta
+     $ git rm -rf .
+     $ git commit &dash;&dash;allow-empty -m "Create orphan meta branch"
+            </literallayout>
+        </para>
+-->
+
+        <para>
+            If you modify the Metadata, you must not forget to update the
+            <ulink url='&YOCTO_DOCS_REF_URL;#var-SRCREV'><filename>SRCREV</filename></ulink>
+            statements in the kernel's recipe.
+            In particular, you need to update the
+            <filename>SRCREV_meta</filename> variable to match the commit in
+            the <filename>KMETA</filename> branch you wish to use.
+            Changing the data in these branches and not updating the
+            <filename>SRCREV</filename> statements to match will cause the
+            build to fetch an older commit.
+        </para>
+    </section>
+</section>
+
 <section id='organizing-your-source'>
     <title>Organizing Your Source</title>
 

documentation/kernel-dev/kernel-dev.xml

@@ -78,7 +78,7 @@
             </revision>
             <revision>
                 <revnumber>2.3</revnumber>
-                <date>Proposed for April 2017</date>
+                <date>May 2017</date>
                 <revremark>Released with the Yocto Project 2.3 Release.</revremark>
             </revision>
         </revhistory>
@@ -93,12 +93,33 @@
         Permission is granted to copy, distribute and/or modify this document under
         the terms of the <ulink type="http" url="http://creativecommons.org/licenses/by-sa/2.0/uk/">Creative Commons Attribution-Share Alike 2.0 UK: England &amp; Wales</ulink> as published by Creative Commons.
       </para>
-      <note>
-          For the latest version of this manual associated with this
-          Yocto Project release, see the
-          <ulink url='&YOCTO_DOCS_KERNEL_DEV_URL;'>Yocto Project Linux Kernel Development Manual</ulink>
-          from the Yocto Project website.
-      </note>
+           <note><title>Manual Notes</title>
+               <itemizedlist>
+                   <listitem><para>
+                       For the latest version of the Yocto Project Linux
+                       Kernel Development Manual associated with this Yocto
+                       Project release (version &YOCTO_DOC_VERSION;),
+                       see the Yocto Project Linux Kernel Development
+                       Manual from the
+                       <ulink url='&YOCTO_HOME_URL;/documentation'>Yocto Project documentation page</ulink>.
+                       </para></listitem>
+                   <listitem><para>
+                       This version of the manual is version
+                       &YOCTO_DOC_VERSION;.
+                       For later releases of the Yocto Project (if they exist),
+                       go to the
+                       <ulink url='&YOCTO_HOME_URL;/documentation'>Yocto Project documentation page</ulink>
+                       and use the drop-down "Active Releases" button
+                       and choose the Yocto Project version for which you want
+                       the manual.
+                       </para></listitem>
+                   <listitem><para>
+                        For an in-development version of the Yocto Project
+                        Linux Kernel Development Manual, see
+                        <ulink url='&YOCTO_DOCS_URL;/latest/kernel-dev/kernel-dev.html'></ulink>.
+                        </para></listitem>
+                </itemizedlist>
+            </note>
     </legalnotice>
 
     </bookinfo>

documentation/mega-manual/mega-manual.xml

@@ -62,7 +62,7 @@
             </revision>
             <revision>
                 <revnumber>2.3</revnumber>
-                <date>Proposed for April 2017</date>
+                <date>May 2017</date>
                 <revremark>Released with the Yocto Project 2.3 Release.</revremark>
             </revision>
        </revhistory>
@@ -77,12 +77,32 @@
         Permission is granted to copy, distribute and/or modify this document under
         the terms of the <ulink type="http" url="http://creativecommons.org/licenses/by-sa/2.0/uk/">Creative Commons Attribution-Share Alike 2.0 UK: England &amp; Wales</ulink> as published by Creative Commons.
       </para>
-      <note>
-          For the latest version of this manual associated with this
-          Yocto Project release, see the
-          <ulink url='&YOCTO_DOCS_MM_URL;'>Yocto Project Mega-Manual</ulink>
-          from the Yocto Project website.
-      </note>
+           <note><title>Manual Notes</title>
+               <itemizedlist>
+                   <listitem><para>
+                       For the latest version of the Yocto Project
+                       Mega-Manual associated with this Yocto Project release
+                       (version &YOCTO_DOC_VERSION;),
+                       see the Yocto Project Mega-Manual from the
+                       <ulink url='&YOCTO_HOME_URL;/documentation'>Yocto Project documentation page</ulink>.
+                       </para></listitem>
+                   <listitem><para>
+                       This version of the manual is version
+                       &YOCTO_DOC_VERSION;.
+                       For later releases of the Yocto Project (if they exist),
+                       go to the
+                       <ulink url='&YOCTO_HOME_URL;/documentation'>Yocto Project documentation page</ulink>
+                       and use the drop-down "Active Releases" button
+                       and choose the Yocto Project version for which you want
+                       the manual.
+                       </para></listitem>
+                   <listitem><para>
+                       For an in-development version of the Yocto Project
+                       Mega-Manual, see
+                       <ulink url='&YOCTO_DOCS_URL;/latest/mega-manual/mega-manual.html'></ulink>.
+                       </para></listitem>
+               </itemizedlist>
+           </note>
 
     </legalnotice>
 
@@ -194,6 +214,9 @@
     <xi:include
         xmlns:xi="http://www.w3.org/2003/XInclude" href="../ref-manual/technical-details.xml"/>
 
+    <xi:include
+        xmlns:xi="http://www.w3.org/2003/XInclude" href="../ref-manual/ref-release-process.xml"/>
+
     <xi:include
         xmlns:xi="http://www.w3.org/2003/XInclude" href="../ref-manual/migration.xml"/>
 

documentation/poky.ent

@@ -1,7 +1,7 @@
 <!ENTITY DISTRO "2.3">
 <!ENTITY DISTRO_COMPRESSED "23">
-<!ENTITY DISTRO_NAME_NO_CAP "tbd">
-<!ENTITY DISTRO_NAME "TBD">
+<!ENTITY DISTRO_NAME_NO_CAP "pyro">
+<!ENTITY DISTRO_NAME "Pyro">
 <!ENTITY YOCTO_DOC_VERSION "2.3">
 <!ENTITY POKYVERSION "18.0.0">
 <!ENTITY POKYVERSION_COMPRESSED "1800">
@@ -61,13 +61,16 @@
 <!ENTITY OE_INIT_PATH "&YOCTO_POKY;/oe-init-build-env">
 <!ENTITY OE_INIT_FILE "oe-init-build-env">
 <!ENTITY UBUNTU_HOST_PACKAGES_ESSENTIAL "gawk wget git-core diffstat unzip texinfo gcc-multilib \
-     build-essential chrpath socat cpio python python3 pip3 pexpect">
+     build-essential chrpath socat cpio python python3 python3-pip python3-expect \
+     xz which">
 <!ENTITY FEDORA_HOST_PACKAGES_ESSENTIAL "gawk make wget tar bzip2 gzip python3 unzip perl patch \
      diffutils diffstat git cpp gcc gcc-c++ glibc-devel texinfo chrpath \
      ccache perl-Data-Dumper perl-Text-ParseWords perl-Thread-Queue perl-bignum socat \
-     findutils which file cpio python pip3 pexpect">
+     findutils which file cpio python python3-pip python3-expect xz which">
 <!ENTITY OPENSUSE_HOST_PACKAGES_ESSENTIAL "python gcc gcc-c++ git chrpath make wget python-xml \
-     diffstat makeinfo python-curses patch socat python3 python3-curses tar pip3 pexpect">
+     diffstat makeinfo python-curses patch socat python3 python3-curses tar python3-pip \
+     python3-expect xz which">
 <!ENTITY CENTOS_HOST_PACKAGES_ESSENTIAL "gawk make wget tar bzip2 gzip python unzip perl patch \
      diffutils diffstat git cpp gcc gcc-c++ glibc-devel texinfo chrpath socat \
-     perl-Data-Dumper perl-Text-ParseWords perl-Thread-Queue pip3 pexpect">
+     perl-Data-Dumper perl-Text-ParseWords perl-Thread-Queue python3-pip python3-expect xz \
+     which">

documentation/profile-manual/profile-manual.xml

@@ -78,7 +78,7 @@
             </revision>
             <revision>
                 <revnumber>2.3</revnumber>
-                <date>Proposed for April 2017</date>
+                <date>May 2017</date>
                 <revremark>Released with the Yocto Project 2.3 Release.</revremark>
             </revision>
         </revhistory>
@@ -96,12 +96,33 @@
           Creative Commons.
       </para>
 
-      <note>
-          For the latest version of this manual associated with this
-          Yocto Project release, see the
-          <ulink url='&YOCTO_DOCS_PROF_URL;'>Yocto Project Profiling and Tracing Manual</ulink>
-          from the Yocto Project website.
-      </note>
+            <note><title>Manual Notes</title>
+                <itemizedlist>
+                    <listitem><para>
+                        For the latest version of the Yocto Project Profiling
+                        and Tracing Manual associated with this Yocto Project
+                        release (version &YOCTO_DOC_VERSION;),
+                        see the Yocto Project Profiling and Tracing Manual
+                        from the
+                        <ulink url='&YOCTO_HOME_URL;/documentation'>Yocto Project documentation page</ulink>.
+                        </para></listitem>
+                    <listitem><para>
+                        This version of the manual is version
+                        &YOCTO_DOC_VERSION;.
+                        For later releases of the Yocto Project (if they exist),
+                        go to the
+                        <ulink url='&YOCTO_HOME_URL;/documentation'>Yocto Project documentation page</ulink>
+                        and use the drop-down "Active Releases" button
+                        and choose the Yocto Project version for which you want
+                        the manual.
+                        </para></listitem>
+                    <listitem><para>
+                        For an in-development version of the Yocto Project
+                        Profiling and Tracing Manual, see
+                        <ulink url='&YOCTO_DOCS_URL;/latest/profile-manual/profile-manual.html'></ulink>.
+                        </para></listitem>
+                </itemizedlist>
+            </note>
     </legalnotice>
 
     </bookinfo>

documentation/ref-manual/ref-devtool-reference.xml

@@ -215,6 +215,17 @@
             The <filename>.bbappend</filename> file is created to point
             to the external source tree.
         </para>
+
+        <note>
+            If your recipe has runtime dependencies defined, you must be sure
+            that these packages exist on the target hardware before attempting
+            to run your application.
+            If dependent packages (e.g. libraries) do not exist on the target,
+            your application, when run, will fail to find those functions.
+            For more information, see the
+            "<link linkend='devtool-deploying-your-software-on-the-target-machine'>Deploying Your Software on the Target Machine</link>"
+            section.
+        </note>
     </section>
 
     <section id='devtool-extracting-the-source-for-an-existing-recipe'>
@@ -501,6 +512,40 @@
                 </para>
             </note>
         </para>
+
+        <para>
+            Some conditions exist that could prevent a deployed application
+            from behaving as expected.
+            When both of the following conditions exist, your application has
+            the potential to not behave correctly when run on the target:
+            <itemizedlist>
+                <listitem><para>
+                    You are deploying a new application to the target that
+                    has correctly defined runtime dependencies defined in
+                    recipe you used to build the application.
+                    </para></listitem>
+                <listitem><para>
+                    The target does not physically have the packages on which
+                    the application depends installed.
+                    </para></listitem>
+            </itemizedlist>
+            If both of these conditions exist, your application will not
+            behave as expected.
+            The reason for this misbehavior is because the
+            <filename>devtool deploy-target</filename> command does not deploy
+            the packages (e.g. libraries) on which your new application
+            depends.
+            The assumption is that the packages are already on the target.
+            Consequently, when a runtime call is made in the application
+            for a dependent function (e.g. a library call), the function
+            cannot be found.
+        </para>
+
+        <para>
+            To be sure you have all the dependencies local to the target, you
+            need to be sure that the packages are pre-deployed (installed)
+            on the target before attempting to run your application.
+        </para>
     </section>
 
     <section id='devtool-removing-your-software-from-the-target-machine'>

documentation/ref-manual/ref-manual.xml

@@ -109,7 +109,7 @@
             </revision>
             <revision>
                 <revnumber>2.3</revnumber>
-                <date>Proposed for April 2017</date>
+                <date>May 2017</date>
                 <revremark>Released with the Yocto Project 2.3 Release.</revremark>
             </revision>
         </revhistory>
@@ -124,12 +124,32 @@
         Permission is granted to copy, distribute and/or modify this document under
         the terms of the <ulink type="http" url="http://creativecommons.org/licenses/by-sa/2.0/uk/">Creative Commons Attribution-Share Alike 2.0 UK: England &amp; Wales</ulink> as published by Creative Commons.
       </para>
-      <note>
-          For the latest version of this manual associated with this
-          Yocto Project release, see the
-          <ulink url='&YOCTO_DOCS_REF_URL;'>Yocto Project Reference Manual</ulink>
-          from the Yocto Project website.
-      </note>
+           <note><title>Manual Notes</title>
+               <itemizedlist>
+                   <listitem><para>
+                       For the latest version of the Yocto Project Reference
+                       Manual associated with this Yocto Project release
+                       (version &YOCTO_DOC_VERSION;),
+                       see the Yocto Project Reference Manual from the
+                       <ulink url='&YOCTO_HOME_URL;/documentation'>Yocto Project documentation page</ulink>.
+                       </para></listitem>
+                   <listitem><para>
+                       This version of the manual is version
+                       &YOCTO_DOC_VERSION;.
+                       For later releases of the Yocto Project (if they exist),
+                       go to the
+                       <ulink url='&YOCTO_HOME_URL;/documentation'>Yocto Project documentation page</ulink>
+                       and use the drop-down "Active Releases" button
+                       and choose the Yocto Project version for which you want
+                       the manual.
+                       </para></listitem>
+                   <listitem><para>
+                       For an in-development version of the Yocto Project
+                       Reference Manual, see
+                        <ulink url='&YOCTO_DOCS_URL;/latest/ref-manual/ref-manual.html'></ulink>.
+                        </para></listitem>
+                </itemizedlist>
+            </note>
     </legalnotice>
 
     </bookinfo>
@@ -142,6 +162,8 @@
 
     <xi:include href="technical-details.xml"/>
 
+    <xi:include href="ref-release-process.xml"/>
+
     <xi:include href="migration.xml"/>
 
     <xi:include href="ref-structure.xml"/>

documentation/ref-manual/ref-release-process.xml

@@ -0,0 +1,123 @@
+<!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
+"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"
+[<!ENTITY % poky SYSTEM "../poky.ent"> %poky; ] >
+
+<chapter id='ref-release-process'>
+<title>Yocto Project Releases and the Stable Release Process</title>
+
+<para>
+    The Yocto Project release process is predictable and consists of both
+    major and minor (point) releases.
+    This brief chapter provides information on how releases are named, their
+    life cycle, and their stability.
+</para>
+
+<section id='major-and-minor-release-cadence'>
+    <title>Major and Minor Release Cadence</title>
+
+    <para>
+        The Yocto Project delivers major releases (e.g. &DISTRO;) using a six
+        month cadence roughly timed each April and October of the year.
+        Following are examples of some major YP releases with their codenames
+        also shown.
+        See the
+        "<link linkend='major-release-codenames'>Major Release Codenames</link>"
+        section for information on codenames used with major releases.
+        <literallayout class='monospaced'>
+    2.2 (Morty)
+    2.1 (Krogoth)
+    2.0 (Jethro)
+        </literallayout>
+        While the cadence is never perfect, this timescale facilitates
+        regular releases that have strong QA cycles while not overwhelming
+        users with too many new releases.
+        The cadence is predictable and avoids many major holidays in various
+        geographies.
+    </para>
+
+    <para>
+        The Yocto project delivers minor (point) releases on an unscheduled
+        basis and are usually driven by the accumulation of enough significant
+        fixes or enhancements to the associated major release.
+        Following are some example past point releases:
+        <literallayout class='monospaced'>
+    2.1.1
+    2.1.2
+    2.2.1
+        </literallayout>
+        The point release indicates a point in the major release branch where
+        a full QA cycle and release process validates the content of the new
+        branch.
+        <note>
+            Realize that there can be patches merged onto the stable release
+            branches as and when they become available.
+        </note>
+    </para>
+</section>
+
+<section id='major-release-codenames'>
+    <title>Major Release Codenames</title>
+
+    <para>
+        Each major release receives a codename that identifies the release in
+        the
+        <ulink url='&YOCTO_DOCS_DEV_URL;#yocto-project-repositories'>Yocto Project Source Repositories</ulink>.
+        The concept is that branches of
+        <ulink url='&YOCTO_DOCS_DEV_URL;#metadata'>Metadata</ulink>
+        with the same codename are likely to be compatible and thus
+        work together.
+        <note>
+            Codenames are associated with major releases because a Yocto
+            Project release number (e.g. &DISTRO;) could conflict with
+            a given layer or company versioning scheme.
+            Codenames are unique, interesting, and easily identifiable.
+        </note>
+        Releases are given a nominal release version as well but the codename
+        is used in repositories for this reason.
+        You can find information on Yocto Project releases and codenames at
+        <ulink url='https://wiki.yoctoproject.org/wiki/Releases'></ulink>.
+    </para>
+</section>
+
+<section id='stable-release-process'>
+    <title>Stable Release Process</title>
+
+    <para>
+        Once released, the release enters the stable release process at which
+        time a person is assigned as the maintainer for that stable release.
+        This maintainer monitors activity for the release by investigating
+        and handling nominated patches and backport activity.
+        Only fixes and enhancements that have first been applied on the
+        "master" branch (i.e. the current, in-development branch) are
+        considered for backporting to a stable release.
+        <note>
+            The current Yocto Project policy regarding backporting is to
+            consider bug fixes and security fixes only.
+            Policy dictates that features are not backported to a stable
+            release.
+            This policy means generic recipe version upgrades are unlikely to
+            be accepted for backporting.
+            The exception to this policy occurs when a strong reason exists
+            such as the fix happens to also be the preferred upstream approach.
+        </note>
+    </para>
+
+    <para>
+        Stable release branches have strong maintenance for about a year after
+        their initial release.
+        Should significant issues be found for any release regardless of its
+        age, fixes could be backported to older releases.
+        For issues that are not backported given an older release,
+        Community LTS trees and branches exist where
+        community members share patches for older releases.
+        However, these types of patches do not go through the same release
+        process as do point releases.
+        You can find more information about stable branch maintenance at
+        <ulink url='https://wiki.yoctoproject.org/wiki/Stable_branch_maintenance'></ulink>.
+    </para>
+</section>
+
+</chapter>
+<!--
+vim: expandtab tw=80 ts=4
+-->

documentation/ref-manual/ref-tasks.xml

@@ -853,7 +853,10 @@
         <title><filename>do_compile_kernelmodules</filename></title>
 
         <para>
-            Compiles loadable modules for the Linux kernel.
+            Runs the step that builds the kernel modules (if needed).
+            Building a kernel consists of two steps: 1) the kernel
+            (<filename>vmlinux</filename>) is built, and 2) the modules
+            are built (i.e. <filename>make modules</filename>).
         </para>
     </section>
 
@@ -861,9 +864,21 @@
         <title><filename>do_diffconfig</filename></title>
 
         <para>
-            Compares the old and new config files after running the
-            <link linkend='ref-tasks-menuconfig'><filename>do_menuconfig</filename></link>
-            task for the kernel.
+            When invoked by the user, this task creates a file containing the
+            differences between the original config as produced by
+            <link linkend='ref-tasks-kernel_configme'><filename>do_kernel_configme</filename></link>
+            task and the changes made by the user with other methods
+            (i.e. using
+            (<link linkend='ref-tasks-kernel_menuconfig'><filename>do_kernel_menuconfig</filename></link>).
+            Once the file of differences is created, it can be used to create
+            a config fragment that only contains the differences.
+            You can invoke this task from the command line as follows:
+            <literallayout class='monospaced'>
+     $ bitbake linux-yocto -c diffconfig
+            </literallayout>
+            For more information, see the
+            "<ulink url='&YOCTO_DOCS_KERNEL_DEV_URL;#generating-configuration-files'>Generating Configuration Files</ulink>"
+            section in the Yocto Project Linux Kernel Development Manual.
         </para>
     </section>
 
@@ -871,7 +886,12 @@
         <title><filename>do_kernel_checkout</filename></title>
 
         <para>
-            Checks out source/meta branches for a linux-yocto style kernel.
+            Converts the newly unpacked kernel source into a form with which
+            the OpenEmbedded build system can work.
+            Because the kernel source can be fetched in several different ways,
+            the <filename>do_kernel_checkout</filename> task makes sure that
+            subsequent tasks are given a clean working tree copy of the kernel
+            with the correct branches checked out.
         </para>
     </section>
 
@@ -879,7 +899,21 @@
         <title><filename>do_kernel_configcheck</filename></title>
 
         <para>
-            Validates the kernel configuration for a linux-yocto style kernel.
+            Validates the configuration produced by the
+            <link linkend='ref-tasks-kernel_menuconfig'><filename>do_kernel_menuconfig</filename></link>
+            task.
+            The <filename>do_kernel_configcheck</filename> task produces
+            warnings when a requested configuration does not appear in the
+            final <filename>.config</filename> file or when you override a
+            policy configuration in a hardware configuration fragment.
+            You can run this task explicitly and view the output by using
+            the following command:
+            <literallayout class='monospaced'>
+     $ bitbake linux-yocto -c kernel_configcheck -f
+            </literallayout>
+            For more information, see the
+            "<ulink url='&YOCTO_DOCS_KERNEL_DEV_URL;#generating-configuration-files'>Generating Configuration Files</ulink>"
+            section in the Yocto Project Linux Kernel Development Manual.
         </para>
     </section>
 
@@ -887,17 +921,41 @@
         <title><filename>do_kernel_configme</filename></title>
 
         <para>
-            Assembles the kernel configuration for a linux-yocto style kernel.
+            After the kernel is patched by the
+            <link linkend='ref-tasks-patch'><filename>do_patch</filename></link>
+            task, the <filename>do_kernel_configme</filename> task assembles
+            and merges all the kernel config fragments into a merged
+            configuration that can then be passed to the kernel configuration
+            phase proper.
+            This is also the time during which user-specified defconfigs
+            are applied if present, and where configuration modes such as
+            <filename>--allnoconfig</filename> are applied.
         </para>
     </section>
 
-    <section id='ref-tasks-kernel_link_vmlinux'>
-        <title><filename>do_kernel_link_vmlinux</filename></title>
+    <section id='ref-tasks-kernel_menuconfig'>
+        <title><filename>do_kernel_menuconfig</filename></title>
 
         <para>
-            Creates a symbolic link in
-            <filename>arch/$arch/boot</filename> for vmlinux kernel
-            images.
+            Invoked by the user to manipulate the
+            <filename>.config</filename> file used to build a linux-yocto
+            recipe.
+            This task starts the Linux kernel configuration tool, which you
+            then use to modify the kernel configuration.
+            <note>
+                You can also invoke this tool from the command line as
+                follows:
+                <literallayout class='monospaced'>
+     $ bitbake linux-yocto -c menuconfig
+                </literallayout>
+            </note>
+            See the
+            "<ulink url='&YOCTO_DOCS_KERNEL_DEV_URL;#generating-configuration-files'>Generating Configuration Files</ulink>"
+            section in the Yocto Project Linux Kernel Development Manual
+            for more information on this configuration tool.
+            You can also reference the
+            "<ulink url='&YOCTO_DOCS_DEV_URL;#using-menuconfig'>Using <filename>menuconfig</filename></ulink>"
+            section in the Yocto Project Development Manual.
         </para>
     </section>
 
@@ -905,8 +963,16 @@
         <title><filename>do_kernel_metadata</filename></title>
 
         <para>
-            Collects kernel metadata for a
-            <filename>linux-yocto</filename> style kernel.
+            Collects all the features required for a given kernel build,
+            whether the features come from
+            <link linkend='var-SRC_URI'><filename>SRC_URI</filename></link>
+            or from Git repositories.
+            After collection, the <filename>do_kernel_metadata</filename> task
+            processes the features into a series of config fragments and
+            patches, which can then be applied by subsequent tasks such as
+            <link linkend='ref-tasks-patch'><filename>do_patch</filename></link>
+            and
+            <link linkend='ref-tasks-kernel_configme'><filename>do_kernel_configme</filename></link>.
         </para>
     </section>
 
@@ -925,7 +991,17 @@
         <title><filename>do_savedefconfig</filename></title>
 
         <para>
-            Creates a minimal Linux kernel configuration file.
+            When invoked by the user, creates a defconfig file that can be
+            used instead of the default defconfig.
+            The saved defconfig contains the differences between the default
+            defconfig and the changes made by the user using other methods
+            (i.e. the
+            <link linkend='ref-tasks-kernel_menuconfig'><filename>do_kernel_menuconfig</filename></link>
+            task.
+            You can invoke the task using the following command:
+            <literallayout class='monospaced'>
+     $ bitbake linux-yocto -c savedefconfig
+            </literallayout>
         </para>
     </section>
 
@@ -933,7 +1009,14 @@
         <title><filename>do_shared_workdir</filename></title>
 
         <para>
-            Creates the shared working directory for the kernel.
+            After the kernel has been compiled but before the kernel modules
+            have been compiled, this task copies files required for module
+            builds and which are generated from the  kernel build into the
+            shared work directory.
+            With these copies successfully copied, the
+            <link linkend='ref-tasks-compile_kernelmodules'><filename>do_compile_kernelmodules</filename></link>
+            task can successfully build the kernel modules in the next step
+            of the build.
         </para>
     </section>
 
@@ -941,9 +1024,12 @@
         <title><filename>do_sizecheck</filename></title>
 
         <para>
-            Checks the size of the kernel image against
-            <link linkend='var-KERNEL_IMAGE_MAXSIZE'><filename>KERNEL_IMAGE_MAXSIZE</filename></link>
-            when set.
+            After the kernel has been built, this task checks the size of the
+            stripped kernel image against
+            <link linkend='var-KERNEL_IMAGE_MAXSIZE'><filename>KERNEL_IMAGE_MAXSIZE</filename></link>.
+            If that variable was set and the size of the stripped kernel
+            exceeds that size, the kernel build produces a warning to that
+            effect.
         </para>
     </section>
 
@@ -951,15 +1037,13 @@
         <title><filename>do_strip</filename></title>
 
         <para>
-            Strips unneeded sections out of the Linux kernel image.
-        </para>
-    </section>
-
-    <section id='ref-tasks-uboot_mkimage'>
-        <title><filename>do_uboot_mkimage</filename></title>
-
-        <para>
-            Creates a uImage file from the kernel for the U-Boot bootloader.
+            If
+            <filename>KERNEL_IMAGE_STRIP_EXTRA_SECTIONS</filename> is defined,
+            this task strips the sections named in that variable from
+            <filename>vmlinux</filename>.
+            This stripping is typically used to remove nonessential sections
+            such as <filename>.comment</filename> sections from a
+            size-sensitive configuration.
         </para>
     </section>
 
@@ -967,10 +1051,14 @@
         <title><filename>do_validate_branches</filename></title>
 
         <para>
-            Ensures that the source, metadata (or both) branches are on the
-            locations specified by their
-            <link linkend='var-SRCREV'><filename>SRCREV</filename></link>
-            values for a linux-yocto style kernel.
+            After the kernel is unpacked but before it is patched, this task
+            makes sure that the machine and metadata branches as specified
+            by the <link linkend='var-SRCREV'><filename>SRCREV</filename></link>
+            variables actually exist on the specified branches.
+            If these branches do not exist and
+            <link linkend='var-AUTOREV'><filename>AUTOREV</filename></link>
+            is not being used, the <filename>do_validate_branches</filename>
+            task fails during the build.
         </para>
     </section>
 </section>

documentation/ref-manual/ref-variables.xml

@@ -2364,6 +2364,161 @@
             </glossdef>
         </glossentry>
 
+        <glossentry id='var-COPYLEFT_LICENSE_EXCLUDE'><glossterm>COPYLEFT_LICENSE_EXCLUDE</glossterm>
+            <info>
+                COPYLEFT_LICENSE_EXCLUDE[doc] = "Licenses to exclude in the source archived by the archiver class."
+            </info>
+            <glossdef>
+                <para role="glossdeffirst">
+<!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
+                    A space-separated list of licenses to exclude from the
+                    source archived by the
+                    <link linkend='ref-classes-archiver'><filename>archiver</filename></link>
+                    class.
+                    In other words, if a license in a recipe's
+                    <link linkend='var-LICENSE'><filename>LICENSE</filename></link>
+                    value is in the value of
+                    <filename>COPYLEFT_LICENSE_EXCLUDE</filename>, then its
+                    source is not archived by the class.
+                    <note>
+                        The <filename>COPYLEFT_LICENSE_EXCLUDE</filename>
+                        variable takes precedence over the
+                        <link linkend='var-COPYLEFT_LICENSE_INCLUDE'><filename>COPYLEFT_LICENSE_INCLUDE</filename></link>
+                        variable.
+                    </note>
+                    The default value, which is "CLOSED Proprietary", for
+                    <filename>COPYLEFT_LICENSE_EXCLUDE</filename> is set
+                    by the
+                    <link linkend='ref-classes-copyleft_filter'><filename>copyleft_filter</filename></link>
+                    class, which is inherited by the
+                    <filename>archiver</filename> class.
+                </para>
+            </glossdef>
+        </glossentry>
+
+        <glossentry id='var-COPYLEFT_LICENSE_INCLUDE'><glossterm>COPYLEFT_LICENSE_INCLUDE</glossterm>
+            <info>
+                COPYLEFT_LICENSE_INCLUDE[doc] = "Licenses to include in the source archived by the archiver class."
+            </info>
+            <glossdef>
+                <para role="glossdeffirst">
+<!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
+                    A space-separated list of licenses to include in the
+                    source archived by the
+                    <link linkend='ref-classes-archiver'><filename>archiver</filename></link>
+                    class.
+                    In other words, if a license in a recipe's
+                    <link linkend='var-LICENSE'><filename>LICENSE</filename></link>
+                    value is in the value of
+                    <filename>COPYLEFT_LICENSE_INCLUDE</filename>, then its
+                    source is archived by the class.
+                </para>
+
+                <para>
+                    The default value, which is "GPL* LGPL*", for
+                    <filename>COPYLEFT_LICENSE_INCLUDE</filename> is set
+                    by the
+                    <link linkend='ref-classes-copyleft_filter'><filename>copyleft_filter</filename></link>
+                    class, which is inherited by the
+                    <filename>archiver</filename> class.
+                </para>
+            </glossdef>
+        </glossentry>
+
+        <glossentry id='var-COPYLEFT_PN_EXCLUDE'><glossterm>COPYLEFT_PN_EXCLUDE</glossterm>
+            <info>
+                COPYLEFT_PN_EXCLUDE[doc] = "Recipes to exclude in the source archived by the archiver class."
+            </info>
+            <glossdef>
+                <para role="glossdeffirst">
+<!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
+                    A list of recipes to exclude in the source archived
+                    by the
+                    <link linkend='ref-classes-archiver'><filename>archiver</filename></link>
+                    class.
+                    The <filename>COPYLEFT_PN_EXCLUDE</filename> variable
+                    overrides the license inclusion and exclusion caused
+                    through the
+                    <link linkend='var-COPYLEFT_LICENSE_INCLUDE'><filename>COPYLEFT_LICENSE_INCLUDE</filename></link>
+                    and
+                    <link linkend='var-COPYLEFT_LICENSE_EXCLUDE'><filename>COPYLEFT_LICENSE_EXCLUDE</filename></link>
+                    variables, respectively.
+                </para>
+
+                <para>
+                    The default value, which is "" indicating to not explicitly
+                    exclude any recipes by name, for
+                    <filename>COPYLEFT_PN_EXCLUDE</filename> is set
+                    by the
+                    <link linkend='ref-classes-copyleft_filter'><filename>copyleft_filter</filename></link>
+                    class, which is inherited by the
+                    <filename>archiver</filename> class.
+                </para>
+            </glossdef>
+        </glossentry>
+
+        <glossentry id='var-COPYLEFT_PN_INCLUDE'><glossterm>COPYLEFT_PN_INCLUDE</glossterm>
+            <info>
+                COPYLEFT_PN_INCLUDE[doc] = "Recipes to include in the source archived by the archiver class."
+            </info>
+            <glossdef>
+                <para role="glossdeffirst">
+<!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
+                    A list of recipes to include in the source archived
+                    by the
+                    <link linkend='ref-classes-archiver'><filename>archiver</filename></link>
+                    class.
+                    The <filename>COPYLEFT_PN_INCLUDE</filename> variable
+                    overrides the license inclusion and exclusion caused
+                    through the
+                    <link linkend='var-COPYLEFT_LICENSE_INCLUDE'><filename>COPYLEFT_LICENSE_INCLUDE</filename></link>
+                    and
+                    <link linkend='var-COPYLEFT_LICENSE_EXCLUDE'><filename>COPYLEFT_LICENSE_EXCLUDE</filename></link>
+                    variables, respectively.
+                </para>
+
+                <para>
+                    The default value, which is "" indicating to not explicitly
+                    include any recipes by name, for
+                    <filename>COPYLEFT_PN_INCLUDE</filename> is set
+                    by the
+                    <link linkend='ref-classes-copyleft_filter'><filename>copyleft_filter</filename></link>
+                    class, which is inherited by the
+                    <filename>archiver</filename> class.
+                </para>
+            </glossdef>
+        </glossentry>
+
+        <glossentry id='var-COPYLEFT_RECIPE_TYPES'><glossterm>COPYLEFT_RECIPE_TYPES</glossterm>
+            <info>
+                COPYLEFT_RECIPE_TYPES[doc] = "Recipe types to include in the source archived by the archiver class."
+            </info>
+            <glossdef>
+                <para role="glossdeffirst">
+<!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
+                    A space-separated list of recipe types to include
+                    in the source archived by the
+                    <link linkend='ref-classes-archiver'><filename>archiver</filename></link>
+                    class.
+                    Recipe types are <filename>target</filename>,
+                    <filename>native</filename>,
+                    <filename>nativesdk</filename>,
+                    <filename>cross</filename>,
+                    <filename>crosssdk</filename>, and
+                    <filename>cross-canadian</filename>.
+                </para>
+
+                <para>
+                    The default value, which is "target*", for
+                    <filename>COPYLEFT_RECIPE_TYPES</filename> is set
+                    by the
+                    <link linkend='ref-classes-copyleft_filter'><filename>copyleft_filter</filename></link>
+                    class, which is inherited by the
+                    <filename>archiver</filename> class.
+                </para>
+            </glossdef>
+        </glossentry>
+
         <glossentry id='var-COPY_LIC_DIRS'><glossterm>COPY_LIC_DIRS</glossterm>
             <info>
                 COPY_LIC_DIRS[doc] = "If set to "1" along with the COPY_LIC_MANIFEST variable, the OpenEmbedded build system copies into the image the license files, which are located in /usr/share/common-licenses, for each package."
@@ -8332,7 +8487,7 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
                     <filename>local.conf</filename> file or you can attach it to
                     a specific image recipe by using the recipe name override:
                     <literallayout class='monospaced'>
-     NO_RECOMMENDATIONS_pn-<replaceable>target_image</replaceable> = "<replaceable>package_name</replaceable>"
+     NO_RECOMMENDATIONS_pn-<replaceable>target_image</replaceable> = "1"
                     </literallayout>
                 </para>
 
@@ -15590,6 +15745,50 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
             </glossdef>
         </glossentry>
 
+        <glossentry id='var-WKS_FILE_DEPENDS'><glossterm>WKS_FILE_DEPENDS</glossterm>
+            <info>
+               WKS_FILE_DEPENDS[doc] = "Lists a recipe's build-time dependencies specific to Wic."
+            </info>
+            <glossdef>
+                <para role="glossdeffirst">
+                    When placed in the recipe that builds your image, this
+                    variable lists build-time dependencies.
+                    The <filename>WKS_FILE_DEPENDS</filename> variable is only
+                    applicable when Wic images are active (i.e. when
+                    <link linkend='var-IMAGE_FSTYPES'><filename>IMAGE_FSTYPES</filename></link>
+                    contains entries related to Wic).
+                    If your recipe does not create Wic images, the variable
+                    has no effect.
+                </para>
+
+                <para>
+                    The <filename>WKS_FILE_DEPENDS</filename> variable is
+                    similar to the
+                    <link linkend='var-DEPENDS'><filename>DEPENDS</filename></link>
+                    variable.
+                    When you use the variable in your recipe that builds the
+                    Wic image, dependencies you list in the
+                    <filename>WIC_FILE_DEPENDS</filename> variable are added to
+                    the <filename>DEPENDS</filename> variable.
+                </para>
+
+                <para>
+                    With the <filename>WKS_FILE_DEPENDS</filename> variable,
+                    you have the possibility to specify a list of additional
+                    dependencies (e.g. native tools, bootloaders, and so forth),
+                    that are required to build Wic images.
+                    Following is an example:
+                    <literallayout class='monospaced'>
+     WKS_FILE_DEPENDS = "<replaceable>some-native-tool</replaceable>"
+                    </literallayout>
+                    In the previous example,
+                    <replaceable>some-native-tool</replaceable> would be
+                    replaced with an actual native tool on which the build
+                    would depend.
+                </para>
+            </glossdef>
+        </glossentry>
+
         <glossentry id='var-WKS_FILE'><glossterm>WKS_FILE</glossterm>
             <info>
                WKS_FILE[doc] = "Specifies the name of the wic kickstart file."

documentation/ref-manual/technical-details.xml

@@ -1455,12 +1455,22 @@
                          file://licfile2.txt;endline=50;md5=zzzz \
                          ..."
                 </literallayout>
-                <note>
-                    When using "beginline" and "endline", realize that line
-                    numbering begins with one and not zero.
-                    Also, the lines included are inclusive (e.g. lines five
-                    through 29 in the previous example for
-                    <filename>licfile1.txt</filename>.
+                <note><title>Notes</title>
+                    <itemizedlist>
+                        <listitem><para>
+                            When using "beginline" and "endline", realize that
+                            line numbering begins with one and not zero.
+                            Also, the lines included are inclusive (e.g. lines
+                            five through 29 in the previous example for
+                            <filename>licfile1.txt</filename>.
+                            </para></listitem>
+                        <listitem><para>
+                            When a license check fails, the selected license
+                            text is included as part of the QA message.
+                            Using this output, you can determine the exact
+                            start and finish for the needed license text.
+                            </para></listitem>
+                    </itemizedlist>
                 </note>
             </para>
 

documentation/sdk-manual/sdk-manual.xml

@@ -43,7 +43,7 @@
             </revision>
             <revision>
                 <revnumber>2.3</revnumber>
-                <date>Proposed for April 2017</date>
+                <date>May 2017</date>
                 <revremark>Released with the Yocto Project 2.3 Release.</revremark>
             </revision>
        </revhistory>
@@ -58,12 +58,33 @@
         Permission is granted to copy, distribute and/or modify this document under
         the terms of the <ulink type="http" url="http://creativecommons.org/licenses/by-sa/2.0/uk/">Creative Commons Attribution-Share Alike 2.0 UK: England &amp; Wales</ulink> as published by Creative Commons.
       </para>
-      <note>
-          For the latest version of this manual associated with this
-          Yocto Project release, see the
-          <ulink url='&YOCTO_DOCS_SDK_URL;'>Yocto Project Software Development Kit (SDK) Developer's Guide</ulink>
-          from the Yocto Project website.
-      </note>
+           <note><title>Manual Notes</title>
+               <itemizedlist>
+                   <listitem><para>
+                       For the latest version of the Yocto Project Software
+                       Development Kit (SDK) Developer's Guide associated with
+                       this Yocto Project release (version &YOCTO_DOC_VERSION;),
+                       see the Yocto Project Software Development Kit (SDK)
+                       Developer's Guide from the
+                       <ulink url='&YOCTO_HOME_URL;/documentation'>Yocto Project documentation page</ulink>.
+                       </para></listitem>
+                   <listitem><para>
+                       This version of the manual is version
+                       &YOCTO_DOC_VERSION;.
+                       For later releases of the Yocto Project (if they exist),
+                       go to the
+                       <ulink url='&YOCTO_HOME_URL;/documentation'>Yocto Project documentation page</ulink>
+                       and use the drop-down "Active Releases" button
+                       and choose the Yocto Project version for which you want
+                       the manual.
+                       </para></listitem>
+                   <listitem><para>
+                       For an in-development version of the Yocto Project
+                       Software Development Kit (SDK) Developer's Guide, see
+                       <ulink url='&YOCTO_DOCS_URL;/latest/sdk-manual/sdk-manual.html'></ulink>.
+                       </para></listitem>
+               </itemizedlist>
+           </note>
 
     </legalnotice>
 

documentation/toaster-manual/toaster-manual.xml

@@ -53,7 +53,7 @@
             </revision>
             <revision>
                 <revnumber>2.3</revnumber>
-                <date>Proposed for April 2017</date>
+                <date>May 2017</date>
                 <revremark>Released with the Yocto Project 2.3 Release.</revremark>
             </revision>
        </revhistory>
@@ -68,12 +68,32 @@
         Permission is granted to copy, distribute and/or modify this document under
         the terms of the <ulink type="http" url="http://creativecommons.org/licenses/by-sa/2.0/uk/">Creative Commons Attribution-Share Alike 2.0 UK: England &amp; Wales</ulink> as published by Creative Commons.
       </para>
-      <note>
-          For the latest version of this manual associated with this
-          Yocto Project release, see the
-          <ulink url='&YOCTO_DOCS_TOAST_URL;'>Toaster User Manual</ulink>
-          from the Yocto Project website.
-      </note>
+           <note><title>Manual Notes</title>
+               <itemizedlist>
+                   <listitem><para>
+                       For the latest version of the Yocto Project Toaster
+                       User Manual associated with this Yocto Project release
+                       (version &YOCTO_DOC_VERSION;),
+                       see the Yocto Project Toaster User Manual from the
+                       <ulink url='&YOCTO_HOME_URL;/documentation'>Yocto Project documentation page</ulink>.
+                       </para></listitem>
+                   <listitem><para>
+                       This version of the manual is version
+                       &YOCTO_DOC_VERSION;.
+                       For later releases of the Yocto Project (if they exist),
+                       go to the
+                       <ulink url='&YOCTO_HOME_URL;/documentation'>Yocto Project documentation page</ulink>
+                       and use the drop-down "Active Releases" button
+                       and choose the Yocto Project version for which you want
+                       the manual.
+                       </para></listitem>
+                   <listitem><para>
+                       For an in-development version of the Yocto Project
+                       Toaster User Manual, see
+                       <ulink url='&YOCTO_DOCS_URL;/latest/toaster-manual/toaster-manual.html'></ulink>.
+                       </para></listitem>
+               </itemizedlist>
+           </note>
 
     </legalnotice>
 

documentation/yocto-project-qs/yocto-project-qs.xml

@@ -20,15 +20,20 @@
                 <itemizedlist>
                     <listitem><para>
                         For the latest version of the Yocto Project Quick
-                        Start associated with this Yocto Project release,
+                        Start associated with this Yocto Project release
+                        (version &YOCTO_DOC_VERSION;),
                         see the Yocto Project Quick Start from the
-                        <ulink url='&YOCTO_HOME_URL;'>Yocto Project website</ulink>.
+                        <ulink url='&YOCTO_HOME_URL;/documentation'>Yocto Project documentation page</ulink>.
                         </para></listitem>
                     <listitem><para>
-                        Later releases of the Yocto Project might exist and the
-                        associated Yocto Project Quick Start for any later
-                        release can be found at
-                        <ulink url='&YOCTO_HOME_URL;/documentation'></ulink>.
+                        This version of the manual is version
+                        &YOCTO_DOC_VERSION;.
+                        For later releases of the Yocto Project (if they exist),
+                        go to the
+                        <ulink url='&YOCTO_HOME_URL;/documentation'>Yocto Project documentation page</ulink>
+                        and use the drop-down "Active Releases" button
+                        and choose the Yocto Project version for which you want
+                        the manual.
                         </para></listitem>
                     <listitem><para>
                         For an in-development version of the Yocto Project
@@ -327,8 +332,7 @@
                         </para></listitem>
                     <listitem><para><emphasis>OpenSUSE</emphasis>
                         <literallayout class='monospaced'>
-     $ sudo zypper install &OPENSUSE_HOST_PACKAGES_ESSENTIAL; \
-     libSDL-devel xterm
+     $ sudo zypper install &OPENSUSE_HOST_PACKAGES_ESSENTIAL; libSDL-devel xterm
                         </literallayout>
                         </para></listitem>
                     <listitem><para><emphasis>CentOS</emphasis>

meta-poky/conf/distro/include/maintainers.inc

@@ -378,6 +378,7 @@ RECIPE_MAINTAINER_pn-libpam = "Maxin B. John <maxin.john@intel.com>"
 RECIPE_MAINTAINER_pn-libpcap = "Maxin B. John <maxin.john@intel.com>"
 RECIPE_MAINTAINER_pn-libpciaccess = "Alexander Kanavin <alexander.kanavin@intel.com>"
 RECIPE_MAINTAINER_pn-libpcre = "Robert Yang <liezhi.yang@windriver.com>"
+RECIPE_MAINTAINER_pn-libpcre2 = "Armin Kuster  <akuster@mvista.com>"
 RECIPE_MAINTAINER_pn-libpfm4 = "Matthew McClintock <msm@freescale.com>"
 RECIPE_MAINTAINER_pn-libpng = "Maxin B. John <maxin.john@intel.com>"
 RECIPE_MAINTAINER_pn-libpng12 = "Maxin B. John <maxin.john@intel.com>"

meta-poky/conf/distro/poky-tiny.conf

@@ -30,6 +30,7 @@
 
 require conf/distro/poky.conf
 DISTRO = "poky-tiny"
+DISTROOVERRIDES = "poky:poky-tiny"
 TCLIBC = "musl"
 # FIXME: consider adding a new "tiny" feature
 #DISTRO_FEATURES_append = " tiny"
@@ -37,7 +38,7 @@ TCLIBC = "musl"
 # Distro config is evaluated after the machine config, so we have to explicitly
 # set the kernel provider to override a machine config.
 PREFERRED_PROVIDER_virtual/kernel = "linux-yocto-tiny"
-PREFERRED_VERSION_linux-yocto-tiny ?= "4.4%"
+PREFERRED_VERSION_linux-yocto-tiny ?= "4.9%"
 
 # We can use packagegroup-core-boot, but in the future we may need a new packagegroup-core-tiny
 #POKY_DEFAULT_EXTRA_RDEPENDS += "packagegroup-core-boot"

meta-poky/conf/distro/poky.conf

@@ -21,13 +21,13 @@ POKY_DEFAULT_EXTRA_RRECOMMENDS = "kernel-module-af-packet"
 
 DISTRO_FEATURES ?= "${DISTRO_FEATURES_DEFAULT} ${DISTRO_FEATURES_LIBC} ${POKY_DEFAULT_DISTRO_FEATURES}"
 
-PREFERRED_VERSION_linux-yocto ?= "4.8%"
-PREFERRED_VERSION_linux-yocto_qemux86 ?= "4.8%"
-PREFERRED_VERSION_linux-yocto_qemux86-64 ?= "4.8%"
-PREFERRED_VERSION_linux-yocto_qemuarm ?= "4.8%"
-PREFERRED_VERSION_linux-yocto_qemumips ?= "4.8%"
-PREFERRED_VERSION_linux-yocto_qemumips64 ?= "4.8%"
-PREFERRED_VERSION_linux-yocto_qemuppc ?= "4.8%"
+PREFERRED_VERSION_linux-yocto ?= "4.10%"
+PREFERRED_VERSION_linux-yocto_qemux86 ?= "4.10%"
+PREFERRED_VERSION_linux-yocto_qemux86-64 ?= "4.10%"
+PREFERRED_VERSION_linux-yocto_qemuarm ?= "4.10%"
+PREFERRED_VERSION_linux-yocto_qemumips ?= "4.10%"
+PREFERRED_VERSION_linux-yocto_qemumips64 ?= "4.10%"
+PREFERRED_VERSION_linux-yocto_qemuppc ?= "4.10%"
 
 SDK_NAME = "${DISTRO}-${TCLIBC}-${SDK_ARCH}-${IMAGE_BASENAME}-${TUNE_PKGARCH}"
 SDKPATH = "/opt/${DISTRO}/${SDK_VERSION}"
@@ -70,24 +70,17 @@ https://.*/.*    http://downloads.yoctoproject.org/mirror/sources/ \n"
 CONNECTIVITY_CHECK_URIS ?= "https://www.example.com/"
 
 SANITY_TESTED_DISTROS ?= " \
-            poky-1.8 \n \
-            poky-2.0 \n \
             poky-2.1 \n \
             poky-2.2 \n \
-            ubuntu-14.04 \n \
-            ubuntu-14.10 \n \
             ubuntu-15.04 \n \
-            ubuntu-15.10 \n \
             ubuntu-16.04 \n \
             ubuntu-16.10 \n \
-            fedora-22 \n \
-            fedora-23 \n \
             fedora-24 \n \
             fedora-25 \n \
             centos-7 \n \
             debian-8 \n \
-            opensuse-13.2 \n \
             opensuse-42.1 \n \
+            opensuse-42.2 \n \
             "
 #
 # OELAYOUT_ABI allows us to notify users when the format of TMPDIR changes in 

meta-selftest/lib/oeqa/runtime/cases/selftest.py

@@ -1,5 +1,7 @@
 from oeqa.runtime.case import OERuntimeTestCase
 from oeqa.core.decorator.depends import OETestDepends
+from oeqa.runtime.cases.dnf import DnfTest
+from oeqa.utils.httpserver import HTTPService
 
 class Selftest(OERuntimeTestCase):
 
@@ -8,7 +10,7 @@ class Selftest(OERuntimeTestCase):
         """
         Summary: Check basic package installation functionality.
         Expected: 1. Before the test socat must be installed using scp.
-                  2. After the test socat must be unistalled using ssh.
+                  2. After the test socat must be uninstalled using ssh.
                      This can't be checked in this test.
         Product: oe-core
         Author: Mariano Lopez <mariano.lopez@intel.com>
@@ -18,10 +20,10 @@ class Selftest(OERuntimeTestCase):
         self.assertEqual(status, 0, msg="socat is not installed")
 
     @OETestDepends(['selftest.Selftest.test_install_package'])
-    def test_verify_unistall(self):
+    def test_verify_uninstall(self):
         """
         Summary: Check basic package installation functionality.
-        Expected: 1. test_install_package must unistall socat.
+        Expected: 1. test_install_package must uninstall socat.
                      This test is just to verify that.
         Product: oe-core
         Author: Mariano Lopez <mariano.lopez@intel.com>
@@ -29,3 +31,43 @@ class Selftest(OERuntimeTestCase):
 
         (status, output) = self.target.run("socat -V")
         self.assertNotEqual(status, 0, msg="socat is still installed")
+
+
+class DnfSelftest(DnfTest):
+
+    @classmethod
+    def setUpClass(cls):
+        cls.repo_server = HTTPService(os.path.join(cls.tc.td['WORKDIR'], 'oe-rootfs-repo'),
+                                      cls.tc.target.server_ip)
+        cls.repo_server.start()
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.repo_server.stop()
+
+    @OETestDepends(['ssh.SSHTest.test_ssh'])
+    def test_verify_package_feeds(self):
+        """
+        Summary: Check correct setting of PACKAGE_FEED_URIS var
+        Expected: 1. Feeds were correctly set for dnf
+                  2. Update recovers packages from host's repo
+        Author: Humberto Ibarra <humberto.ibarra.lopez@intel.com>
+        Author: Alexander Kanavin <alexander.kanavin@intel.com>
+        """
+        # When we created an image, we had to supply fake ip and port
+        # for the feeds. Now we can patch the real ones into the config file.
+        import tempfile
+        temp_file = tempfile.TemporaryDirectory(prefix="oeqa-remotefeeds-").name
+        self.tc.target.copyFrom("/etc/yum.repos.d/oe-remote-repo.repo", temp_file)
+        fixed_config = open(temp_file, "r").read().replace("bogus_ip", self.tc.target.server_ip).replace("bogus_port", str(self.repo_server.port))
+        open(temp_file, "w").write(fixed_config)
+        self.tc.target.copyTo(temp_file, "/etc/yum.repos.d/oe-remote-repo.repo")
+
+        import re
+        output_makecache = self.dnf('makecache')
+        self.assertTrue(re.match(r".*Metadata cache created", output_makecache, re.DOTALL) is not None, msg = "dnf makecache failed: %s" %(output_makecache))
+
+        output_repoinfo = self.dnf('repoinfo')
+        matchobj = re.match(r".*Repo-pkgs\s*:\s*(?P<n_pkgs>[0-9]+)", output_repoinfo, re.DOTALL)
+        self.assertTrue(matchobj is not None, msg = "Could not find the amount of packages in dnf repoinfo output: %s" %(output_repoinfo))
+        self.assertTrue(int(matchobj.group('n_pkgs')) > 0, msg = "Amount of remote packages is not more than zero: %s\n" %(output_repoinfo))

meta-selftest/recipes-test/container-image/container-image-testpkg.bb

@@ -0,0 +1,8 @@
+LICENSE = "MIT"
+
+INHIBIT_DEFAULT_DEPS = "1"
+
+do_install_append() {
+    install -d ${D}${bindir}
+    touch ${D}${bindir}/theapp
+}

meta-selftest/recipes-test/container-image/container-test-image.bb

@@ -0,0 +1,8 @@
+IMAGE_INSTALL += "container-image-testpkg"
+
+LICENSE = "MIT"
+
+IMAGE_FSTYPES = "container"
+IMAGE_LINGUAS = ""
+
+inherit core-image

meta-selftest/recipes-test/images/oe-selftest-image.bb

@@ -1,6 +1,6 @@
 SUMMARY = "An image used during oe-selftest tests"
 
-IMAGE_INSTALL = "packagegroup-core-boot ${ROOTFS_PKGMANAGE_BOOTSTRAP} dropbear"
+IMAGE_INSTALL = "packagegroup-core-boot dropbear"
 IMAGE_FEATURES = "debug-tweaks"
 
 IMAGE_LINGUAS = " "

meta-selftest/recipes-test/images/wic-image-minimal.bb

@@ -2,11 +2,11 @@ SUMMARY = "An example of partitioned image."
 
 SRC_URI = "file://${FILE_DIRNAME}/${BPN}.wks"
 
-IMAGE_INSTALL = "packagegroup-core-boot ${ROOTFS_PKGMANAGE_BOOTSTRAP}"
+IMAGE_INSTALL = "packagegroup-core-boot"
 
 IMAGE_FSTYPES = "wic"
 
-DEPENDS = "syslinux syslinux-native dosfstools-native mtools-native gptfdisk-native"
+WKS_FILE_DEPENDS = "syslinux syslinux-native dosfstools-native mtools-native gptfdisk-native"
 
 LIC_FILES_CHKSUM = "file://${COREBASE}/meta/COPYING.MIT;md5=3da9cfbcb788c80a0384361b4de20420"
 

meta-selftest/recipes-test/selftest-ed/selftest-ed_0.5.bb

@@ -1,4 +1,4 @@
-SUMMARY = "Line-oriented text editor"
+SUMMARY = "Line-oriented text editor -- selftest GPLv2 version"
 HOMEPAGE = "http://www.gnu.org/software/ed/"
 SECTION = "base"
 LICENSE = "GPLv2+"
@@ -13,6 +13,8 @@ SRC_URI[sha256sum] = "edef2bbde0fbf0d88232782a0eded323f483a0519d6fde9a3b1809056f
 
 inherit autotools texinfo
 
+S = "${WORKDIR}/ed-${PV}"
+
 EXTRA_OECONF = "'CC=${CC}' 'CXX=${CXX}' 'CFLAGS=${CFLAGS}' 'CXXFLAGS=${CXXFLAGS}' 'CPPFLAGS=${CPPFLAGS}' 'LDFLAGS=${LDFLAGS}'"
 
 CONFIGUREOPTS_remove = "--disable-dependency-tracking"

meta-selftest/recipes-test/selftest-ed/selftest-ed_1.14.1.bb

@@ -0,0 +1,35 @@
+SUMMARY = "Line-oriented text editor -- selftest variant"
+HOMEPAGE = "http://www.gnu.org/software/ed/"
+
+LICENSE = "GPLv3+"
+LIC_FILES_CHKSUM = "file://COPYING;md5=0c7051aef9219dc7237f206c5c4179a7 \
+                    file://ed.h;endline=20;md5=4e36b7a40e137f42aee718165590d125 \
+                    file://main.c;endline=17;md5=c5b8f78f115df187af76868a2aead16a"
+
+SECTION = "base"
+
+# LSB states that ed should be in /bin/
+bindir = "${base_bindir}"
+
+SRC_URI = "${GNU_MIRROR}/ed/ed-${PV}.tar.lz"
+
+SRC_URI[md5sum] = "7f4a54fa7f366479f03654b8af645fd0"
+SRC_URI[sha256sum] = "ffb97eb8f2a2b5a71a9b97e3872adce953aa1b8958e04c5b7bf11d556f32552a"
+
+S = "${WORKDIR}/ed-${PV}"
+
+EXTRA_OEMAKE = "-e MAKEFLAGS="
+
+inherit texinfo
+
+do_configure() {
+	${S}/configure
+}
+
+do_install() {
+	oe_runmake 'DESTDIR=${D}' install
+	# Info dir listing isn't interesting at this point so remove it if it exists.
+	if [ -e "${D}${infodir}/dir" ]; then
+		rm -f ${D}${infodir}/dir
+	fi
+}

meta-selftest/wic/test_rawcopy_plugin.wks.in

@@ -0,0 +1,6 @@
+# short-description: This file is used in oe-selftest wic module to test rawcopy plugin
+
+part /boot --active --source bootimg-pcbios
+part / --source rawcopy --sourceparams="file=core-image-minimal-${MACHINE}.ext4" --use-uuid
+
+bootloader  --timeout=0 --append="console=ttyS0,115200n8"

meta-selftest/wic/wictestdisk.wks

@@ -0,0 +1,7 @@
+# short-description: image for use in machine agnostic wic test cases
+
+# /boot is intentionally an empty partition
+part /boot --ondisk sda --label boot --active --align 1024 --size 16
+part / --source rootfs --ondisk sda --fstype=ext4 --label platform --align 1024 --use-uuid
+
+# bootloader is intentionally left out

meta-yocto-bsp/conf/machine/beaglebone.conf

@@ -23,7 +23,7 @@ do_image_wic[depends] += "mtools-native:do_populate_sysroot dosfstools-native:do
 SERIAL_CONSOLE = "115200 ttyO0"
 
 PREFERRED_PROVIDER_virtual/kernel ?= "linux-yocto"
-PREFERRED_VERSION_linux-yocto ?= "4.8%"
+PREFERRED_VERSION_linux-yocto ?= "4.10%"
 
 KERNEL_IMAGETYPE = "zImage"
 KERNEL_DEVICETREE = "am335x-bone.dtb am335x-boneblack.dtb"

meta-yocto-bsp/conf/machine/edgerouter.conf

@@ -11,7 +11,7 @@ KERNEL_ALT_IMAGETYPE = "vmlinux.bin"
 KERNEL_IMAGE_STRIP_EXTRA_SECTIONS  = ".comment"
 
 PREFERRED_PROVIDER_virtual/kernel ?= "linux-yocto"
-PREFERRED_VERSION_linux-yocto ?= "4.8%"
+PREFERRED_VERSION_linux-yocto ?= "4.10%"
 
 SERIAL_CONSOLE = "115200 ttyS0"
 USE_VT ?= "0"

meta-yocto-bsp/conf/machine/genericx86-64.conf

@@ -7,6 +7,6 @@ DEFAULTTUNE ?= "core2-64"
 require conf/machine/include/tune-core2.inc
 require conf/machine/include/genericx86-common.inc
 
-PREFERRED_VERSION_linux-yocto_genericx86-64 ?= "4.8%"
+PREFERRED_VERSION_linux-yocto ?= "4.10%"
 
 SERIAL_CONSOLES_CHECK = "ttyS0"

meta-yocto-bsp/conf/machine/genericx86.conf

@@ -9,4 +9,4 @@ require conf/machine/include/genericx86-common.inc
 
 MACHINE_ESSENTIAL_EXTRA_RRECOMMENDS += "gma500-gfx-check"
 
-PREFERRED_VERSION_linux-yocto_genericx86 ?= "4.8%"
+PREFERRED_VERSION_linux-yocto ?= "4.10%"

meta-yocto-bsp/conf/machine/include/genericx86-common.inc

@@ -3,7 +3,6 @@ require conf/machine/include/qemuboot-x86.inc
 MACHINE_FEATURES += "wifi efi pcbios"
 
 PREFERRED_PROVIDER_virtual/kernel ?= "linux-yocto"
-PREFERRED_VERSION_linux-yocto ?= "4.8%"
 PREFERRED_PROVIDER_virtual/xserver ?= "xserver-xorg"
 XSERVER ?= "${XSERVER_X86_BASE} \
             ${XSERVER_X86_EXT} \

meta-yocto-bsp/conf/machine/mpc8315e-rdb.conf

@@ -8,13 +8,13 @@ require conf/machine/include/tune-ppce300c3.inc
 KERNEL_IMAGETYPE = "uImage"
 
 EXTRA_IMAGEDEPENDS += "u-boot"
-UBOOT_MACHINE_mpc8315e-rdb = "MPC8315ERDB_config"
+UBOOT_MACHINE = "MPC8315ERDB_config"
 
 SERIAL_CONSOLE = "115200 ttyS0"
 
 MACHINE_FEATURES = "keyboard pci ext2 ext3 serial"
 
-PREFERRED_VERSION_linux-yocto ?= "4.8%"
+PREFERRED_VERSION_linux-yocto ?= "4.10%"
 PREFERRED_PROVIDER_virtual/kernel = "linux-yocto"
 
 PREFERRED_PROVIDER_virtual/xserver ?= "xserver-xorg"

meta-yocto-bsp/recipes-kernel/linux/linux-yocto_4.1.bbappend

@@ -7,11 +7,11 @@ KBRANCH_mpc8315e-rdb = "standard/fsl-mpc8315e-rdb"
 KMACHINE_genericx86 ?= "common-pc"
 KMACHINE_genericx86-64 ?= "common-pc-64"
 
-SRCREV_machine_genericx86    ?= "fec49247816d7045aa8abe0047bcd4737af9a853"
-SRCREV_machine_genericx86-64 ?= "fec49247816d7045aa8abe0047bcd4737af9a853"
-SRCREV_machine_edgerouter ?= "fec49247816d7045aa8abe0047bcd4737af9a853"
-SRCREV_machine_beaglebone ?= "938cc4ac8d36f166c9e2e0517d6ffd6d278fe631"
-SRCREV_machine_mpc8315e-rdb ?= "fec49247816d7045aa8abe0047bcd4737af9a853"
+SRCREV_machine_genericx86    ?= "aadda25ee787681666f1d7961cd6c5a641da678d"
+SRCREV_machine_genericx86-64 ?= "aadda25ee787681666f1d7961cd6c5a641da678d"
+SRCREV_machine_edgerouter ?= "2ce56d130ddff67f43ec857cc51cd347666a0078"
+SRCREV_machine_beaglebone ?= "235de25be3f143b91dcc69479b770722c508889e"
+SRCREV_machine_mpc8315e-rdb ?= "2ce56d130ddff67f43ec857cc51cd347666a0078"
 
 COMPATIBLE_MACHINE_genericx86 = "genericx86"
 COMPATIBLE_MACHINE_genericx86-64 = "genericx86-64"
@@ -19,8 +19,8 @@ COMPATIBLE_MACHINE_edgerouter = "edgerouter"
 COMPATIBLE_MACHINE_beaglebone = "beaglebone"
 COMPATIBLE_MACHINE_mpc8315e-rdb = "mpc8315e-rdb"
 
-LINUX_VERSION_genericx86 = "4.1.36"
-LINUX_VERSION_genericx86-64 = "4.1.36"
-LINUX_VERSION_edgerouter = "4.1.36"
-LINUX_VERSION_beaglebone = "4.1.36"
-LINUX_VERSION_mpc8315e-rdb = "4.1.36"
+LINUX_VERSION_genericx86 = "4.1.37"
+LINUX_VERSION_genericx86-64 = "4.1.37"
+LINUX_VERSION_edgerouter = "4.1.38"
+LINUX_VERSION_beaglebone = "4.1.38"
+LINUX_VERSION_mpc8315e-rdb = "4.1.38"

meta-yocto-bsp/recipes-kernel/linux/linux-yocto_4.10.bbappend

@@ -0,0 +1,26 @@
+KBRANCH_genericx86  = "standard/base"
+KBRANCH_genericx86-64  = "standard/base"
+
+KMACHINE_genericx86 ?= "common-pc"
+KMACHINE_genericx86-64 ?= "common-pc-64"
+KBRANCH_edgerouter = "standard/edgerouter"
+KBRANCH_beaglebone = "standard/beaglebone"
+KBRANCH_mpc8315e-rdb = "standard/fsl-mpc8315e-rdb"
+
+SRCREV_machine_genericx86    ?= "b259a5d744e8955a03405c6b59c5a61724755ea8"
+SRCREV_machine_genericx86-64 ?= "b259a5d744e8955a03405c6b59c5a61724755ea8"
+SRCREV_machine_edgerouter ?= "827a1164b155110c65f028a13a6c5699be93bbc3"
+SRCREV_machine_beaglebone ?= "827a1164b155110c65f028a13a6c5699be93bbc3"
+SRCREV_machine_mpc8315e-rdb ?= "fc1a68a9046707a9709075afc99eb3b5a2237c4b"
+
+COMPATIBLE_MACHINE_genericx86 = "genericx86"
+COMPATIBLE_MACHINE_genericx86-64 = "genericx86-64"
+COMPATIBLE_MACHINE_edgerouter = "edgerouter"
+COMPATIBLE_MACHINE_beaglebone = "beaglebone"
+COMPATIBLE_MACHINE_mpc8315e-rdb = "mpc8315e-rdb"
+
+LINUX_VERSION_genericx86 = "4.10"
+LINUX_VERSION_genericx86-64 = "4.10"
+LINUX_VERSION_edgerouter = "4.10.2"
+LINUX_VERSION_beaglebone = "4.10.2"
+LINUX_VERSION_mpc8315e-rdb = "4.10.2"

meta-yocto-bsp/recipes-kernel/linux/linux-yocto_4.4.bbappend

@@ -7,11 +7,11 @@ KBRANCH_edgerouter = "standard/edgerouter"
 KBRANCH_beaglebone = "standard/beaglebone"
 KBRANCH_mpc8315e-rdb = "standard/fsl-mpc8315e-rdb"
 
-SRCREV_machine_genericx86    ?= "35482df5d5ba0807eb8a7c40b554bd657e3f9987"
-SRCREV_machine_genericx86-64 ?= "35482df5d5ba0807eb8a7c40b554bd657e3f9987"
-SRCREV_machine_edgerouter ?= "35482df5d5ba0807eb8a7c40b554bd657e3f9987"
-SRCREV_machine_beaglebone ?= "35482df5d5ba0807eb8a7c40b554bd657e3f9987"
-SRCREV_machine_mpc8315e-rdb ?= "772f071dbdd4b813c921058ddf9cba207237228b"
+SRCREV_machine_genericx86    ?= "ad8b1d659ddd2699ebf7d50ef9de8940b157bfc2"
+SRCREV_machine_genericx86-64 ?= "ad8b1d659ddd2699ebf7d50ef9de8940b157bfc2"
+SRCREV_machine_edgerouter ?= "cebe1ad56aebd89e0de29412e19433fb441bf13c"
+SRCREV_machine_beaglebone ?= "cebe1ad56aebd89e0de29412e19433fb441bf13c"
+SRCREV_machine_mpc8315e-rdb ?= "06c0dbdcba374ca7f92a53d69292d6bb7bc9b0f3"
 
 COMPATIBLE_MACHINE_genericx86 = "genericx86"
 COMPATIBLE_MACHINE_genericx86-64 = "genericx86-64"
@@ -19,8 +19,8 @@ COMPATIBLE_MACHINE_edgerouter = "edgerouter"
 COMPATIBLE_MACHINE_beaglebone = "beaglebone"
 COMPATIBLE_MACHINE_mpc8315e-rdb = "mpc8315e-rdb"
 
-LINUX_VERSION_genericx86 = "4.4.36"
-LINUX_VERSION_genericx86-64 = "4.4.36"
-LINUX_VERSION_edgerouter = "4.4.36"
-LINUX_VERSION_beaglebone = "4.4.36"
-LINUX_VERSION_mpc8315e-rdb = "4.4.36"
+LINUX_VERSION_genericx86 = "4.4.41"
+LINUX_VERSION_genericx86-64 = "4.4.41"
+LINUX_VERSION_edgerouter = "4.4.53"
+LINUX_VERSION_beaglebone = "4.4.53"
+LINUX_VERSION_mpc8315e-rdb = "4.4.53"

meta-yocto-bsp/recipes-kernel/linux/linux-yocto_4.8.bbappend

@@ -7,11 +7,11 @@ KBRANCH_edgerouter = "standard/edgerouter"
 KBRANCH_beaglebone = "standard/beaglebone"
 KBRANCH_mpc8315e-rdb = "standard/fsl-mpc8315e-rdb"
 
-SRCREV_machine_genericx86    ?= "021b4aef55b44597587a1ce5879be642b3dca155"
-SRCREV_machine_genericx86-64 ?= "021b4aef55b44597587a1ce5879be642b3dca155"
-SRCREV_machine_edgerouter ?= "6076f16536329465b62bd2037b8582a5e18f85d1"
-SRCREV_machine_beaglebone ?= "85dc85153cd7e3b72d34f967c4c0edde590c79a8"
-SRCREV_machine_mpc8315e-rdb ?= "f73222eb3bbd07a45564397a88dec554e848da7d"
+SRCREV_machine_genericx86    ?= "9bcb4ea3fa107f1a8790c8c3408eb250e8d1d66e"
+SRCREV_machine_genericx86-64 ?= "9bcb4ea3fa107f1a8790c8c3408eb250e8d1d66e"
+SRCREV_machine_edgerouter ?= "e9912dbfdabfed45b2e15feabfc66a404a165d9b"
+SRCREV_machine_beaglebone ?= "07250a1be0e99eb54bf7d3b629125cd09129d6f5"
+SRCREV_machine_mpc8315e-rdb ?= "38e8cf7d4e6d8546bb631d109c60243d2ee01aef"
 
 COMPATIBLE_MACHINE_genericx86 = "genericx86"
 COMPATIBLE_MACHINE_genericx86-64 = "genericx86-64"
@@ -19,8 +19,8 @@ COMPATIBLE_MACHINE_edgerouter = "edgerouter"
 COMPATIBLE_MACHINE_beaglebone = "beaglebone"
 COMPATIBLE_MACHINE_mpc8315e-rdb = "mpc8315e-rdb"
 
-LINUX_VERSION_genericx86 = "4.8.12"
-LINUX_VERSION_genericx86-64 = "4.8.12"
-LINUX_VERSION_edgerouter = "4.8.12"
-LINUX_VERSION_beaglebone = "4.8.12"
-LINUX_VERSION_mpc8315e-rdb = "4.8.12"
+LINUX_VERSION_genericx86 = "4.8.17"
+LINUX_VERSION_genericx86-64 = "4.8.17"
+LINUX_VERSION_edgerouter = "4.8.18"
+LINUX_VERSION_beaglebone = "4.8.18"
+LINUX_VERSION_mpc8315e-rdb = "4.8.18"

meta-yocto-bsp/recipes-kernel/linux/linux-yocto_4.9.bbappend

@@ -0,0 +1,26 @@
+KBRANCH_genericx86  = "standard/base"
+KBRANCH_genericx86-64  = "standard/base"
+
+KMACHINE_genericx86 ?= "common-pc"
+KMACHINE_genericx86-64 ?= "common-pc-64"
+KBRANCH_edgerouter = "standard/edgerouter"
+KBRANCH_beaglebone = "standard/beaglebone"
+KBRANCH_mpc8315e-rdb = "standard/fsl-mpc8315e-rdb"
+
+SRCREV_machine_genericx86    ?= "95c0a80ee83f1cf8e59d733f36e8a9dfd50a0098"
+SRCREV_machine_genericx86-64 ?= "95c0a80ee83f1cf8e59d733f36e8a9dfd50a0098"
+SRCREV_machine_edgerouter ?= "95c0a80ee83f1cf8e59d733f36e8a9dfd50a0098"
+SRCREV_machine_beaglebone ?= "95c0a80ee83f1cf8e59d733f36e8a9dfd50a0098"
+SRCREV_machine_mpc8315e-rdb ?= "7744d762ee37396dfab60f21a2b974c5b94c47b4"
+
+COMPATIBLE_MACHINE_genericx86 = "genericx86"
+COMPATIBLE_MACHINE_genericx86-64 = "genericx86-64"
+COMPATIBLE_MACHINE_edgerouter = "edgerouter"
+COMPATIBLE_MACHINE_beaglebone = "beaglebone"
+COMPATIBLE_MACHINE_mpc8315e-rdb = "mpc8315e-rdb"
+
+LINUX_VERSION_genericx86 = "4.9.13"
+LINUX_VERSION_genericx86-64 = "4.9.13"
+LINUX_VERSION_edgerouter = "4.9.13"
+LINUX_VERSION_beaglebone = "4.9.13"
+LINUX_VERSION_mpc8315e-rdb = "4.9.13"

meta/classes/archiver.bbclass

@@ -365,7 +365,7 @@ python do_ar_recipe () {
         elif include_re.match(line):
             incfile = include_re.match(line).group(1)
         if incfile:
-            incfile = bb.data.expand(incfile, d)
+            incfile = d.expand(incfile)
             incfile = bb.utils.which(bbpath, incfile)
             if incfile:
                 shutil.copy(incfile, outdir)

meta/classes/autotools.bbclass

@@ -143,7 +143,6 @@ python autotools_aclocals () {
     # Refresh variable with cache files
     d.setVar("CONFIG_SITE", siteinfo_get_files(d, aclocalcache=True))
 }
-autotools_copy_aclocals[vardepsexclude] += "MACHINE SDK_ARCH BUILD_ARCH SDK_OS BB_TASKDEPDATA"
 
 CONFIGURE_FILES = "${S}/configure.in ${S}/configure.ac ${S}/config.h.in ${S}/acinclude.m4 Makefile.am"
 
@@ -194,14 +193,14 @@ autotools_do_configure() {
 		else
 			CONFIGURE_AC=configure.ac
 		fi
-		if grep "^[[:space:]]*AM_GLIB_GNU_GETTEXT" $CONFIGURE_AC >/dev/null; then
-			if grep "sed.*POTFILES" $CONFIGURE_AC >/dev/null; then
+		if grep -q "^[[:space:]]*AM_GLIB_GNU_GETTEXT" $CONFIGURE_AC; then
+			if grep -q "sed.*POTFILES" $CONFIGURE_AC; then
 				: do nothing -- we still have an old unmodified configure.ac
 	    		else
 				bbnote Executing glib-gettextize --force --copy
 				echo "no" | glib-gettextize --force --copy
 			fi
-		elif grep "^[[:space:]]*AM_GNU_GETTEXT" $CONFIGURE_AC >/dev/null; then
+		elif grep -q "^[[:space:]]*AM_GNU_GETTEXT" $CONFIGURE_AC; then
 			# We'd call gettextize here if it wasn't so broken...
 			cp ${STAGING_DATADIR_NATIVE}/gettext/config.rpath ${AUTOTOOLS_AUXDIR}/
 			if [ -d ${S}/po/ ]; then
@@ -213,7 +212,7 @@ autotools_do_configure() {
 			PRUNE_M4="$PRUNE_M4 gettext.m4 iconv.m4 lib-ld.m4 lib-link.m4 lib-prefix.m4 nls.m4 po.m4 progtest.m4"
 		fi
 		mkdir -p m4
-		if grep "^[[:space:]]*[AI][CT]_PROG_INTLTOOL" $CONFIGURE_AC >/dev/null; then
+		if grep -q "^[[:space:]]*[AI][CT]_PROG_INTLTOOL" $CONFIGURE_AC; then
 			if ! echo "${DEPENDS}" | grep -q intltool-native; then
 				bbwarn "Missing DEPENDS on intltool-native"
 			fi

meta/classes/base.bbclass

@@ -10,7 +10,7 @@ inherit utility-tasks
 inherit metadata_scm
 inherit logging
 
-OE_IMPORTS += "os sys time oe.path oe.utils oe.types oe.package oe.packagegroup oe.sstatesig oe.lsb oe.cachedpath"
+OE_IMPORTS += "os sys time oe.path oe.utils oe.types oe.package oe.packagegroup oe.sstatesig oe.lsb oe.cachedpath oe.license"
 OE_IMPORTS[type] = "list"
 
 def oe_import(d):
@@ -119,6 +119,25 @@ def get_lic_checksum_file_list(d):
             bb.fatal(d.getVar('PN') + ": LIC_FILES_CHKSUM contains an invalid URL: " + url)
     return " ".join(filelist)
 
+def setup_hosttools_dir(dest, toolsvar, d, fatal=True):
+    tools = d.getVar(toolsvar).split()
+    origbbenv = d.getVar("BB_ORIGENV", False)
+    path = origbbenv.getVar("PATH")
+    bb.utils.mkdirhier(dest)
+    notfound = []
+    for tool in tools:
+        desttool = os.path.join(dest, tool)
+        if not os.path.exists(desttool):
+            srctool = bb.utils.which(path, tool)
+            if "ccache" in srctool:
+                srctool = bb.utils.which(path, tool, direction=1)
+            if srctool:
+                os.symlink(srctool, desttool)
+            else:
+                notfound.append(tool)
+    if notfound and fatal:
+        bb.fatal("The following required tools (as specified by HOSTTOOLS) appear to be unavailable in PATH, please install them in order to proceed:\n  %s" % " ".join(notfound))
+
 addtask fetch
 do_fetch[dirs] = "${DL_DIR}"
 do_fetch[file-checksums] = "${@bb.fetch.get_checksum_file_list(d)}"
@@ -219,10 +238,12 @@ python base_eventhandler() {
         pkgarch_mapping(e.data)
         oe.utils.features_backfill("DISTRO_FEATURES", e.data)
         oe.utils.features_backfill("MACHINE_FEATURES", e.data)
+        # Works with the line in layer.conf which changes PATH to point here
+        setup_hosttools_dir(d.expand('${TMPDIR}/hosttools'), 'HOSTTOOLS', d)
+        setup_hosttools_dir(d.expand('${TMPDIR}/hosttools'), 'HOSTTOOLS_NONFATAL', d, fatal=False)
 
     if isinstance(e, bb.event.BuildStarted):
         localdata = bb.data.createCopy(e.data)
-        bb.data.update_data(localdata)
         statuslines = []
         for func in oe.data.typed_value('BUILDCFG_FUNCS', localdata):
             g = globals()
@@ -355,7 +376,6 @@ def set_packagetriplet(d):
         localdata = bb.data.createCopy(d)
         overrides = localdata.getVar("OVERRIDES", False) + ":virtclass-multilib-" + item
         localdata.setVar("OVERRIDES", overrides)
-        bb.data.update_data(localdata)
 
         archs.append(localdata.getVar("PACKAGE_ARCHS").split())
         tos.append(localdata.getVar("TARGET_OS"))
@@ -535,7 +555,7 @@ python () {
                         bb.debug(1, "INCLUDING the package " + pkg)
                 elif all_skipped or incompatible_license(d, bad_licenses):
                     bb.debug(1, "SKIPPING recipe %s because it's %s" % (pn, license))
-                    raise bb.parse.SkipPackage("incompatible with license %s" % license)
+                    raise bb.parse.SkipPackage("it has an incompatible license: %s" % license)
             elif pn in whitelist:
                 if pn in incompatwl:
                     bb.note("INCLUDING " + pn + " as buildable despite INCOMPATIBLE_LICENSE because it has been whitelisted")

meta/classes/binconfig.bbclass

@@ -13,14 +13,14 @@ def get_binconfig_mangle(d):
         s += " -e 's:=%s${exec_prefix}/:=\\1OEEXECPREFIX/:'" % optional_quote
         s += " -e 's:-L${libdir}:-LOELIBDIR:;'"
         s += " -e 's:-I${includedir}:-IOEINCDIR:;'"
+        s += " -e 's:-L${WORKDIR}:-LOELIBDIR:'"
+        s += " -e 's:-I${WORKDIR}:-I-IOEINCDIR:'"
         s += " -e 's:OEBASELIBDIR:${STAGING_BASELIBDIR}:;'"
         s += " -e 's:OELIBDIR:${STAGING_LIBDIR}:;'"
         s += " -e 's:OEINCDIR:${STAGING_INCDIR}:;'"
         s += " -e 's:OEDATADIR:${STAGING_DATADIR}:'"
         s += " -e 's:OEPREFIX:${STAGING_DIR_HOST}${prefix}:'"
         s += " -e 's:OEEXECPREFIX:${STAGING_DIR_HOST}${exec_prefix}:'"
-        s += " -e 's:-I${WORKDIR}:-I${STAGING_INCDIR}:'"
-        s += " -e 's:-L${WORKDIR}:-L${STAGING_LIBDIR}:'"
         if d.getVar("OE_BINCONFIG_EXTRA_MANGLE", False):
             s += d.getVar("OE_BINCONFIG_EXTRA_MANGLE")
 

meta/classes/buildhistory.bbclass

@@ -86,6 +86,7 @@ python buildhistory_emit_pkghistory() {
             self.depends = ""
             self.packages = ""
             self.srcrev = ""
+            self.layer = ""
 
 
     class PackageInfo:
@@ -186,6 +187,7 @@ python buildhistory_emit_pkghistory() {
     pe = d.getVar('PE') or "0"
     pv = d.getVar('PV')
     pr = d.getVar('PR')
+    layer = bb.utils.get_file_layer(d.getVar('FILE', True), d)
 
     pkgdata_dir = d.getVar('PKGDATA_DIR')
     packages = ""
@@ -225,6 +227,7 @@ python buildhistory_emit_pkghistory() {
     rcpinfo.pr = pr
     rcpinfo.depends = sortlist(oe.utils.squashspaces(d.getVar('DEPENDS') or ""))
     rcpinfo.packages = packages
+    rcpinfo.layer = layer
     write_recipehistory(rcpinfo, d)
 
     pkgdest = d.getVar('PKGDEST')
@@ -303,6 +306,7 @@ def write_recipehistory(rcpinfo, d):
         f.write(u"PR = %s\n" %  rcpinfo.pr)
         f.write(u"DEPENDS = %s\n" %  rcpinfo.depends)
         f.write(u"PACKAGES = %s\n" %  rcpinfo.packages)
+        f.write(u"LAYER = %s\n" %  rcpinfo.layer)
 
 
 def write_pkghistory(pkginfo, d):
@@ -414,7 +418,7 @@ buildhistory_get_installed() {
 
 	# Produce installed package sizes list
 	oe-pkgdata-util -p ${PKGDATA_DIR} read-value "PKGSIZE" -n -f $pkgcache > $1/installed-package-sizes.tmp
-	cat $1/installed-package-sizes.tmp | awk '{print $2 "\tKiB " $1}' | sort -n -r > $1/installed-package-sizes.txt
+	cat $1/installed-package-sizes.tmp | awk '{print $2 "\tKiB\t" $1}' | sort -n -r > $1/installed-package-sizes.txt
 	rm $1/installed-package-sizes.tmp
 
 	# We're now done with the cache, delete it
@@ -550,7 +554,9 @@ END
 python buildhistory_get_extra_sdkinfo() {
     import operator
     import math
-    if d.getVar('BB_CURRENTTASK') == 'populate_sdk_ext':
+
+    if d.getVar('BB_CURRENTTASK') == 'populate_sdk_ext' and \
+            "sdk" in (d.getVar('BUILDHISTORY_FEATURES') or "").split():
         tasksizes = {}
         filesizes = {}
         for root, _, files in os.walk(d.expand('${SDK_OUTPUT}/${SDKPATH}/sstate-cache')):
@@ -594,7 +600,6 @@ def buildhistory_get_build_id(d):
     if d.getVar('BB_WORKERCONTEXT') != '1':
         return ""
     localdata = bb.data.createCopy(d)
-    bb.data.update_data(localdata)
     statuslines = []
     for func in oe.data.typed_value('BUILDCFG_FUNCS', localdata):
         g = globals()

meta/classes/buildstats.bbclass

@@ -31,6 +31,11 @@ def get_process_cputime(pid):
                 i = f.readline().strip()
                 if not i:
                     break
+                if not ":" in i:
+                    # one more extra line is appended (empty or containing "0")
+                    # most probably due to race condition in kernel while
+                    # updating IO stats
+                    break
                 i = i.split(": ")
                 iostats[i[0]] = i[1]
     resources = resource.getrusage(resource.RUSAGE_SELF)

meta/classes/ccache.bbclass

@@ -2,6 +2,12 @@ CCACHE = "${@bb.utils.which(d.getVar('PATH'), 'ccache') and 'ccache '}"
 export CCACHE_DIR ?= "${TMPDIR}/ccache/${MULTIMACH_HOST_SYS}/${PN}"
 CCACHE_DISABLE[unexport] = "1"
 
+# We need to stop ccache considering the current directory or the
+# debug-prefix-map target directory to be significant when calculating
+# its hash. Without this the cache would be invalidated every time
+# ${PV} or ${PR} change.
+export CCACHE_NOHASHDIR ?= "1"
+
 DEPENDS_append_class-target = " ccache-native"
 DEPENDS[vardepvalueexclude] = " ccache-native"
 

meta/classes/copyleft_filter.bbclass

@@ -6,7 +6,7 @@
 #
 # vi:sts=4:sw=4:et
 
-COPYLEFT_LICENSE_INCLUDE ?= 'GPL* LGPL*'
+COPYLEFT_LICENSE_INCLUDE ?= 'GPL* LGPL* AGPL*'
 COPYLEFT_LICENSE_INCLUDE[type] = 'list'
 COPYLEFT_LICENSE_INCLUDE[doc] = 'Space separated list of globs which include licenses'
 

meta/classes/core-image.bbclass

@@ -24,11 +24,15 @@
 # - hwcodecs            - Install hardware acceleration codecs
 # - package-management  - installs package management tools and preserves the package manager database
 # - debug-tweaks        - makes an image suitable for development, e.g. allowing passwordless root logins
+#   - empty-root-password
+#   - allow-empty-password
+#   - post-install-logging
 # - dev-pkgs            - development packages (headers, etc.) for all installed packages in the rootfs
 # - dbg-pkgs            - debug symbol packages for all installed packages in the rootfs
 # - doc-pkgs            - documentation packages for all installed packages in the rootfs
 # - ptest-pkgs          - ptest packages for all ptest-enabled recipes
 # - read-only-rootfs    - tweaks an image to support read-only rootfs
+# - splash              - bootup splash screen
 #
 FEATURE_PACKAGES_x11 = "packagegroup-core-x11"
 FEATURE_PACKAGES_x11-base = "packagegroup-core-x11-base"