build-appliance: Update SRCREV

(From OE-Core rev: 821037700ae61fff6a63d4cdb68559c5fcc96fde)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

bitbake/bin/bitbake

@@ -41,7 +41,7 @@ from bb import ui
 from bb import server
 from bb import cookerdata
 
-__version__ = "1.19.1"
+__version__ = "1.20.0"
 logger = logging.getLogger("BitBake")
 
 # Python multiprocessing requires /dev/shm
@@ -75,18 +75,6 @@ def get_ui(config):
         sys.exit("FATAL: Invalid user interface '%s' specified.\n"
                  "Valid interfaces: depexp, goggle, ncurses, hob, knotty [default]." % interface)
 
-def gather_extra_cache_data():
-    extra = []
-    interfaces = ['depexp', 'goggle', 'ncurses', 'hob', 'knotty']
-    for i in interfaces:
-        try:
-            ui = __import__("bb.ui." + i, fromlist = [i])
-            if hasattr(ui, "extraCaches"):
-                extra = extra + ui.extraCaches
-            del ui
-        except:
-            pass
-    return extra
 
 # Display bitbake/OE warnings via the BitBake.Warnings logger, ignoring others"""
 warnlog = logging.getLogger("BitBake.Warnings")
@@ -112,59 +100,58 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
     def parseCommandLine(self):
         parser = optparse.OptionParser(
             version = "BitBake Build Tool Core version %s, %%prog version %s" % (bb.__version__, __version__),
-            usage = """%prog [options] [package ...]
+            usage = """%prog [options] [recipename/target ...]
 
-    Executes the specified task (default is 'build') for a given set of BitBake files.
-    It expects that BBFILES is defined, which is a space separated list of files to
-    be executed.  BBFILES does support wildcards.
-    Default BBFILES are the .bb files in the current directory.""")
+    Executes the specified task (default is 'build') for a given set of target recipes (.bb files).
+    It is assumed there is a conf/bblayers.conf available in cwd or in BBPATH which
+    will provide the layer, BBFILES and other configuration information.""")
 
-        parser.add_option("-b", "--buildfile", help = "execute the task against this .bb file, rather than a package from BBFILES. Does not handle any dependencies.",
+        parser.add_option("-b", "--buildfile", help = "Execute tasks from a specific .bb recipe directly. WARNING: Does not handle any dependencies from other recipes.",
                    action = "store", dest = "buildfile", default = None)
 
-        parser.add_option("-k", "--continue", help = "continue as much as possible after an error. While the target that failed, and those that depend on it, cannot be remade, the other dependencies of these targets can be processed all the same.",
+        parser.add_option("-k", "--continue", help = "Continue as much as possible after an error. While the target that failed and anything depending on it cannot be built, as much as possible will be built before stopping.",
                    action = "store_false", dest = "abort", default = True)
 
-        parser.add_option("-a", "--tryaltconfigs", help = "continue with builds by trying to use alternative providers where possible.",
+        parser.add_option("-a", "--tryaltconfigs", help = "Continue with builds by trying to use alternative providers where possible.",
                    action = "store_true", dest = "tryaltconfigs", default = False)
 
-        parser.add_option("-f", "--force", help = "force run of specified cmd, regardless of stamp status",
+        parser.add_option("-f", "--force", help = "Force the specified targets/task to run (invalidating any existing stamp file).",
                    action = "store_true", dest = "force", default = False)
 
-        parser.add_option("-c", "--cmd", help = "Specify task to execute. Note that this only executes the specified task for the providee and the packages it depends on, i.e. 'compile' does not implicitly call stage for the dependencies (IOW: use only if you know what you are doing). Depending on the base.bbclass a listtasks tasks is defined and will show available tasks",
+        parser.add_option("-c", "--cmd", help = "Specify the task to execute. The exact options available depend on the metadata. Some examples might be 'compile' or 'populate_sysroot' or 'listtasks' may give a list of the tasks available.",
                    action = "store", dest = "cmd")
 
-        parser.add_option("-C", "--clear-stamp", help = "Invalidate the stamp for the specified cmd such as 'compile' and run the default task for the specified target(s)",
+        parser.add_option("-C", "--clear-stamp", help = "Invalidate the stamp for the specified task such as 'compile' and then run the default task for the specified target(s).",
                     action = "store", dest = "invalidate_stamp")
 
-        parser.add_option("-r", "--read", help = "read the specified file before bitbake.conf",
+        parser.add_option("-r", "--read", help = "Read the specified file before bitbake.conf.",
                    action = "append", dest = "prefile", default = [])
 
-        parser.add_option("-R", "--postread", help = "read the specified file after bitbake.conf",
+        parser.add_option("-R", "--postread", help = "Read the specified file after bitbake.conf.",
                           action = "append", dest = "postfile", default = [])
 
-        parser.add_option("-v", "--verbose", help = "output more chit-chat to the terminal",
+        parser.add_option("-v", "--verbose", help = "Output more log message data to the terminal.",
                    action = "store_true", dest = "verbose", default = False)
 
         parser.add_option("-D", "--debug", help = "Increase the debug level. You can specify this more than once.",
                    action = "count", dest="debug", default = 0)
 
-        parser.add_option("-n", "--dry-run", help = "don't execute, just go through the motions",
+        parser.add_option("-n", "--dry-run", help = "Don't execute, just go through the motions.",
                    action = "store_true", dest = "dry_run", default = False)
 
-        parser.add_option("-S", "--dump-signatures", help = "don't execute, just dump out the signature construction information",
+        parser.add_option("-S", "--dump-signatures", help = "Don't execute, just dump out the signature construction information.",
                    action = "store_true", dest = "dump_signatures", default = False)
 
-        parser.add_option("-p", "--parse-only", help = "quit after parsing the BB files (developers only)",
+        parser.add_option("-p", "--parse-only", help = "Quit after parsing the BB recipes.",
                    action = "store_true", dest = "parse_only", default = False)
 
-        parser.add_option("-s", "--show-versions", help = "show current and preferred versions of all recipes",
+        parser.add_option("-s", "--show-versions", help = "Show current and preferred versions of all recipes.",
                    action = "store_true", dest = "show_versions", default = False)
 
-        parser.add_option("-e", "--environment", help = "show the global or per-package environment (this is what used to be bbread)",
+        parser.add_option("-e", "--environment", help = "Show the global or per-package environment complete with information about where variables were set/changed.",
                    action = "store_true", dest = "show_environment", default = False)
 
-        parser.add_option("-g", "--graphviz", help = "emit the dependency trees of the specified packages in the dot syntax, and the pn-buildlist to show the build list",
+        parser.add_option("-g", "--graphviz", help = "Save dependency tree information for the specified targets in the dot syntax.",
                     action = "store_true", dest = "dot_graph", default = False)
 
         parser.add_option("-I", "--ignore-deps", help = """Assume these dependencies don't exist and are already provided (equivalent to ASSUME_PROVIDED). Useful to make dependency graphs more appealing""",
@@ -173,34 +160,34 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
         parser.add_option("-l", "--log-domains", help = """Show debug logging for the specified logging domains""",
                     action = "append", dest = "debug_domains", default = [])
 
-        parser.add_option("-P", "--profile", help = "profile the command and print a report",
+        parser.add_option("-P", "--profile", help = "Profile the command and save reports.",
                    action = "store_true", dest = "profile", default = False)
 
-        parser.add_option("-u", "--ui", help = "userinterface to use",
+        parser.add_option("-u", "--ui", help = "The user interface to use (e.g. knotty, hob, depexp).",
                    action = "store", dest = "ui")
 
-        parser.add_option("-t", "--servertype", help = "Choose which server to use, process or xmlrpc",
+        parser.add_option("-t", "--servertype", help = "Choose which server to use, process or xmlrpc.",
                    action = "store", dest = "servertype")
 
-        parser.add_option("", "--revisions-changed", help = "Set the exit code depending on whether upstream floating revisions have changed or not",
+        parser.add_option("", "--revisions-changed", help = "Set the exit code depending on whether upstream floating revisions have changed or not.",
                    action = "store_true", dest = "revisions_changed", default = False)
 
-        parser.add_option("", "--server-only", help = "Run bitbake without UI,  the frontend can connect with bitbake server itself",
+        parser.add_option("", "--server-only", help = "Run bitbake without a UI, only starting a server (cooker) process.",
                    action = "store_true", dest = "server_only", default = False)
 
-        parser.add_option("-B", "--bind", help = "The name/address for the bitbake server to bind to",
+        parser.add_option("-B", "--bind", help = "The name/address for the bitbake server to bind to.",
                    action = "store", dest = "bind", default = False)
 
-        parser.add_option("", "--no-setscene", help = "Do not run any setscene tasks, forces builds",
+        parser.add_option("", "--no-setscene", help = "Do not run any setscene tasks. sstate will be ignored and everything needed, built.",
                    action = "store_true", dest = "nosetscene", default = False)
 
-        parser.add_option("", "--remote-server", help = "Connect to the specified server",
+        parser.add_option("", "--remote-server", help = "Connect to the specified server.",
                    action = "store", dest = "remote_server", default = False)
 
-        parser.add_option("-m", "--kill-server", help = "Terminate the remote server",
+        parser.add_option("-m", "--kill-server", help = "Terminate the remote server.",
                     action = "store_true", dest = "kill_server", default = False)
 
-        parser.add_option("", "--observe-only", help = "Connect to a server as an observing-only client",
+        parser.add_option("", "--observe-only", help = "Connect to a server as an observing-only client.",
                    action = "store_true", dest = "observe_only", default = False)
 
         options, targets = parser.parse_args(sys.argv)
@@ -302,25 +289,22 @@ def main():
     # Clear away any spurious environment variables while we stoke up the cooker
     cleanedvars = bb.utils.clean_environment()
 
-    # Collect all the caches we need
-    if configParams.server_only:
-        configuration.extra_caches = gather_extra_cache_data()
-    else:
-        configuration.extra_caches = getattr(ui_module, "extraCaches", [])
-
     if not configParams.remote_server:
         # we start a server with a given configuration
         server = start_server(servermodule, configParams, configuration)
     else:
-        # we start a stub server that is actually a XMLRPClient to
+        # we start a stub server that is actually a XMLRPClient that connects to a real server
         server = servermodule.BitBakeXMLRPCClient(configParams.observe_only)
         server.saveConnectionDetails(configParams.remote_server)
 
     logger.removeHandler(handler)
 
     if not configParams.server_only:
+        # Collect the feature set for the UI
+        featureset = getattr(ui_module, "featureSet", [])
+
         # Setup a connection to the server (cooker)
-        server_connection = server.establishConnection()
+        server_connection = server.establishConnection(featureset)
 
         # Restore the environment in case the UI needs it
         for k in cleanedvars:

bitbake/bin/bitbake-worker

@@ -147,6 +147,9 @@ def fork_off_task(cfg, data, workerdata, fn, task, taskname, appends, quieterror
                 os.umask(umask)
 
             data.setVar("BB_WORKERCONTEXT", "1")
+            data.setVar("BUILDNAME", workerdata["buildname"])
+            data.setVar("DATE", workerdata["date"])
+            data.setVar("TIME", workerdata["time"])
             bb.parse.siggen.set_taskdata(workerdata["hashes"], workerdata["hash_deps"], workerdata["sigchecksums"])
             ret = 0
             try:

bitbake/contrib/vim/ftdetect/bitbake.vim

@@ -14,7 +14,7 @@ endif
 au BufNewFile,BufRead *.{bb,bbappend,bbclass}	set filetype=bitbake
 
 " .inc
-au BufNewFile,BufRead *.inc 		set filetype=bitbake
+au BufNewFile,BufRead *.inc		set filetype=bitbake
 
 " .conf
 au BufNewFile,BufRead *.conf

bitbake/contrib/vim/ftplugin/bitbake.vim

@@ -1 +1,2 @@
 set sts=4 sw=4 et
+set cms=#%s

bitbake/contrib/vim/plugin/newbb.vim

@@ -15,7 +15,7 @@ if &compatible || v:version < 600
 endif
 
 fun! <SID>GetUserName()
-    let l:user_name = system("git-config --get user.name")
+    let l:user_name = system("git config --get user.name")
     if v:shell_error
         return "Unknow User"
     else
@@ -23,7 +23,7 @@ fun! <SID>GetUserName()
 endfun
 
 fun! <SID>GetUserEmail()
-    let l:user_email = system("git-config --get user.email")
+    let l:user_email = system("git config --get user.email")
     if v:shell_error
         return "unknow@user.org"
     else

bitbake/contrib/vim/syntax/bitbake.vim

@@ -44,8 +44,8 @@ syn match bbArrayBrackets       "[\[\]]" contained
 
 " BitBake strings
 syn match bbContinue            "\\$"
-syn region bbString             matchgroup=bbQuote start=+"+ skip=+\\$+ excludenl end=+"+ contained keepend contains=bbTodo,bbContinue,bbVarDeref,bbVarPyValue,@Spell
-syn region bbString             matchgroup=bbQuote start=+'+ skip=+\\$+ excludenl end=+'+ contained keepend contains=bbTodo,bbContinue,bbVarDeref,bbVarPyValue,@Spell
+syn region bbString             matchgroup=bbQuote start=+"+ skip=+\\$+ end=+"+ contained contains=bbTodo,bbContinue,bbVarDeref,bbVarPyValue,@Spell
+syn region bbString             matchgroup=bbQuote start=+'+ skip=+\\$+ end=+'+ contained contains=bbTodo,bbContinue,bbVarDeref,bbVarPyValue,@Spell
 
 " Vars definition
 syn match bbExport            "^export" nextgroup=bbIdentifier skipwhite
@@ -55,11 +55,11 @@ syn match bbVarDeref            "${[a-zA-Z0-9\-_\.\/\+]\+}" contained
 syn match bbVarEq               "\(:=\|+=\|=+\|\.=\|=\.\|?=\|??=\|=\)" contained nextgroup=bbVarValue
 syn match bbVarDef              "^\(export\s*\)\?\([a-zA-Z0-9\-_\.\/\+]\+\(_[${}a-zA-Z0-9\-_\.\/\+]\+\)\?\)\s*\(:=\|+=\|=+\|\.=\|=\.\|?=\|??=\|=\)\@=" contains=bbExportFlag,bbIdentifier,bbVarDeref nextgroup=bbVarEq
 syn match bbVarValue            ".*$" contained contains=bbString,bbVarDeref,bbVarPyValue
-syn region bbVarPyValue         start=+${@+ skip=+\\$+ excludenl end=+}+ contained contains=@python
+syn region bbVarPyValue         start=+${@+ skip=+\\$+ end=+}+ contained contains=@python
 
 " Vars metadata flags
 syn match bbVarFlagDef          "^\([a-zA-Z0-9\-_\.]\+\)\(\[[a-zA-Z0-9\-_\.]\+\]\)\@=" contains=bbIdentifier nextgroup=bbVarFlagFlag
-syn region bbVarFlagFlag        matchgroup=bbArrayBrackets start="\[" end="\]\s*\(=\)\@=" keepend excludenl contained contains=bbIdentifier nextgroup=bbVarEq
+syn region bbVarFlagFlag        matchgroup=bbArrayBrackets start="\[" end="\]\s*\(=\|+=\|=+\|?=\)\@=" contained contains=bbIdentifier nextgroup=bbVarEq
 
 " Includes and requires
 syn keyword bbInclude           inherit include require contained 
@@ -83,13 +83,16 @@ if exists("b:current_syntax")
   unlet b:current_syntax
 endif
 syn keyword bbShFakeRootFlag    fakeroot contained
-syn match bbShFuncDef           "^\(fakeroot\s*\)\?\([0-9A-Za-z_-]\+\)\(python\)\@<!\(\s*()\s*\)\({\)\@=" contains=bbShFakeRootFlag,bbFunction,bbDelimiter nextgroup=bbShFuncRegion skipwhite
-syn region bbShFuncRegion       matchgroup=bbDelimiter start="{\s*$" end="^}\s*$" keepend contained contains=@shell
+syn match bbShFuncDef           "^\(fakeroot\s*\)\?\([0-9A-Za-z_${}-]\+\)\(python\)\@<!\(\s*()\s*\)\({\)\@=" contains=bbShFakeRootFlag,bbFunction,bbVarDeref,bbDelimiter nextgroup=bbShFuncRegion skipwhite
+syn region bbShFuncRegion       matchgroup=bbDelimiter start="{\s*$" end="^}\s*$" contained contains=@shell
+
+" Python value inside shell functions
+syn region shDeref         start=+${@+ skip=+\\$+ excludenl end=+}+ contained contains=@python
 
 " BitBake python metadata
 syn keyword bbPyFlag            python contained
-syn match bbPyFuncDef           "^\(python\s\+\)\([0-9A-Za-z_-]\+\)\?\(\s*()\s*\)\({\)\@=" contains=bbPyFlag,bbFunction,bbDelimiter nextgroup=bbPyFuncRegion skipwhite 
-syn region bbPyFuncRegion       matchgroup=bbDelimiter start="{\s*$" end="^}\s*$" keepend contained contains=@python
+syn match bbPyFuncDef           "^\(python\s\+\)\([0-9A-Za-z_${}-]\+\)\?\(\s*()\s*\)\({\)\@=" contains=bbPyFlag,bbFunction,bbVarDeref,bbDelimiter nextgroup=bbPyFuncRegion skipwhite
+syn region bbPyFuncRegion       matchgroup=bbDelimiter start="{\s*$" end="^}\s*$" contained contains=@python
 
 " BitBake 'def'd python functions
 syn keyword bbPyDef             def contained

bitbake/doc/manual/usermanual.xml

@@ -137,6 +137,24 @@ share common metadata between many packages.</para></listitem>
                 <para>In this example, <varname>B</varname> is now <literal>bvaladditionaldata</literal> and <varname>C</varname> is <literal>testcval</literal>. In contrast to the above appending and prepending operators, no additional space
 will be introduced.</para>
             </section>
+            <section>
+                <title>Appending and Prepending (override style syntax)</title>
+                    <para><screen><varname>B</varname> = "bval"
+<varname>B_append</varname> = " additional data"
+<varname>C</varname> = "cval"
+<varname>C_prepend</varname> = "additional data "</screen></para>
+                 <para>This example results in <varname>B</varname> becoming <literal>bval additional data</literal>
+and <varname>C</varname> becoming <literal>additional data cval</literal>. Note the spaces in the append.
+Unlike the += operator, additional space is not automatically added.  You must take steps to add space
+yourself.</para>
+            </section>
+            <section>
+                <title>Removing (override style syntax)</title>
+                <para><screen><varname>FOO</varname> = "123 456 789 123456 123 456 123 456"
+<varname>FOO_remove</varname> = "123"
+<varname>FOO_remove</varname> = "456"</screen></para>
+                <para>In this example, <varname>FOO</varname> is now <literal>789 123456</literal>.</para>
+            </section>
             <section>
                 <title>Conditional metadata set</title>
                 <para>OVERRIDES is a <quote>:</quote> separated variable containing each item you want to satisfy conditions.  So, if you have a variable which is conditional on <quote>arm</quote>, and <quote>arm</quote> is in OVERRIDES, then the <quote>arm</quote> specific version of the variable is used rather than the non-conditional version.  Example:</para>
@@ -536,7 +554,7 @@ options:
                 <example>
                     <title>Generating dependency graphs</title>
                     <para>BitBake is able to generate dependency graphs using the dot syntax. These graphs can be converted
-to images using the <application>dot</application> application from <ulink url="http://www.graphviz.org">Graphviz</ulink>. 
+to images using the <application>dot</application> application from <ulink url="http://www.graphviz.org">Graphviz</ulink>.
 Two files will be written into the current working directory, <emphasis>depends.dot</emphasis> containing dependency information at the package level and <emphasis>task-depends.dot</emphasis> containing a breakdown of the dependencies at the task level. To stop depending on common depends, one can use the <prompt>-I depend</prompt> to omit these from the graph. This can lead to more readable graphs.  This way, <varname>DEPENDS</varname> from inherited classes such as base.bbclass can be removed from the graph.</para>
                     <screen><prompt>$ </prompt>bitbake -g blah</screen>
                     <screen><prompt>$ </prompt>bitbake -g -I virtual/whatever -I bloom blah</screen>

bitbake/lib/bb/__init__.py

@@ -21,7 +21,7 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-__version__ = "1.19.1"
+__version__ = "1.20.0"
 
 import sys
 if sys.version_info < (2, 7, 3):

bitbake/lib/bb/build.py

@@ -69,9 +69,12 @@ class FuncFailed(Exception):
 class TaskBase(event.Event):
     """Base class for task events"""
 
-    def __init__(self, t, d ):
+    def __init__(self, t, logfile, d):
         self._task = t
         self._package = d.getVar("PF", True)
+        self.taskfile = d.getVar("FILE", True)
+        self.taskname = self._task
+        self.logfile = logfile
         event.Event.__init__(self)
         self._message = "recipe %s: task %s: %s" % (d.getVar("PF", True), t, self.getDisplayName())
 
@@ -96,16 +99,11 @@ class TaskFailed(TaskBase):
     """Task execution failed"""
 
     def __init__(self, task, logfile, metadata, errprinted = False):
-        self.logfile = logfile
         self.errprinted = errprinted
-        super(TaskFailed, self).__init__(task, metadata)
+        super(TaskFailed, self).__init__(task, logfile, metadata)
 
 class TaskFailedSilent(TaskBase):
     """Task execution failed (silently)"""
-    def __init__(self, task, logfile, metadata):
-        self.logfile = logfile
-        super(TaskFailedSilent, self).__init__(task, metadata)
-
     def getDisplayName(self):
         # Don't need to tell the user it was silent
         return "Failed"
@@ -113,7 +111,7 @@ class TaskFailedSilent(TaskBase):
 class TaskInvalid(TaskBase):
 
     def __init__(self, task, metadata):
-        super(TaskInvalid, self).__init__(task, metadata)
+        super(TaskInvalid, self).__init__(task, None, metadata)
         self._message = "No such task '%s'" % task
 
 
@@ -348,6 +346,14 @@ def _exec_task(fn, task, d, quieterr):
     if not tempdir:
         bb.fatal("T variable not set, unable to build")
 
+    # Change nice level if we're asked to
+    nice = localdata.getVar("BB_TASK_NICE_LEVEL", True)
+    if nice:
+        curnice = os.nice(0)
+        nice = int(nice) - curnice
+        newnice = os.nice(nice)
+        logger.debug(1, "Renice to %s " % newnice)
+
     bb.utils.mkdirhier(tempdir)
 
     # Determine the logfile to generate
@@ -416,7 +422,7 @@ def _exec_task(fn, task, d, quieterr):
     localdata.setVar('BB_LOGFILE', logfn)
     localdata.setVar('BB_RUNTASK', task)
 
-    event.fire(TaskStarted(task, localdata), localdata)
+    event.fire(TaskStarted(task, logfn, localdata), localdata)
     try:
         for func in (prefuncs or '').split():
             exec_func(func, localdata)
@@ -453,7 +459,7 @@ def _exec_task(fn, task, d, quieterr):
             logger.debug(2, "Zero size logfn %s, removing", logfn)
             bb.utils.remove(logfn)
             bb.utils.remove(loglink)
-    event.fire(TaskSucceeded(task, localdata), localdata)
+    event.fire(TaskSucceeded(task, logfn, localdata), localdata)
 
     if not localdata.getVarFlag(task, 'nostamp') and not localdata.getVarFlag(task, 'selfstamp'):
         make_stamp(task, localdata)
@@ -467,7 +473,7 @@ def exec_task(fn, task, d, profile = False):
             quieterr = True
 
         if profile: 
-            profname = "profile-%s.log" % (os.path.basename(fn) + "-" + task)
+            profname = "profile-%s.log" % (d.getVar("PN", True) + "-" + task)
             try:
                 import cProfile as profile
             except:

bitbake/lib/bb/cache.py

@@ -43,7 +43,7 @@ except ImportError:
     logger.info("Importing cPickle failed. "
                 "Falling back to a very slow implementation.")
 
-__cache_version__ = "146"
+__cache_version__ = "147"
 
 def getCacheFile(path, filename, data_hash):
     return os.path.join(path, filename + "." + data_hash)

bitbake/lib/bb/cache_extra.py

@@ -35,6 +35,12 @@ class HobRecipeInfo(RecipeInfoCommon):
     # such as (bb_cache.dat, bb_extracache_hob.dat) 
     cachefile = "bb_extracache_" + classname +".dat"        
 
+    # override this member with the list of extra cache fields
+    # that this class will provide
+    cachefields = ['summary', 'license', 'section',
+            'description', 'homepage', 'bugtracker',
+            'prevision', 'files_info']
+
     def __init__(self, filename, metadata):
 
         self.summary = self.getvar('SUMMARY', metadata)

bitbake/lib/bb/command.py

@@ -117,14 +117,14 @@ class Command:
             return False
 
     def finishAsyncCommand(self, msg=None, code=None):
-        if msg:
+        if msg or msg == "":
             bb.event.fire(CommandFailed(msg), self.cooker.event_data)
         elif code:
             bb.event.fire(CommandExit(code), self.cooker.event_data)
         else:
             bb.event.fire(CommandCompleted(), self.cooker.event_data)
         self.currentAsyncCommand = None
-
+        self.cooker.finishcommand()
 
 class CommandsSync:
     """
@@ -137,13 +137,22 @@ class CommandsSync:
         """
         Trigger cooker 'shutdown' mode
         """
-        command.cooker.shutdown()
+        command.cooker.shutdown(False)
 
-    def stateStop(self, command, params):
+    def stateForceShutdown(self, command, params):
         """
         Stop the cooker
         """
-        command.cooker.stop()
+        command.cooker.shutdown(True)
+
+    def getAllKeysWithFlags(self, command, params):
+        """
+        Returns a dump of the global state. Call with
+        variable flags to be retrieved as params.
+        """
+        flaglist = params[0]
+        return command.cooker.getAllKeysWithFlags(flaglist)
+    getAllKeysWithFlags.readonly = True
 
     def getVariable(self, command, params):
         """

bitbake/lib/bb/cooker.py

@@ -61,7 +61,7 @@ class CollectionError(bb.BBHandledException):
     """
 
 class state:
-    initial, parsing, running, shutdown, stop = range(5)
+    initial, parsing, running, shutdown, forceshutdown, stopped = range(6)
 
 
 class SkippedPackage:
@@ -79,6 +79,29 @@ class SkippedPackage:
         elif reason:
             self.skipreason = reason
 
+
+class CookerFeatures(object):
+    _feature_list = [HOB_EXTRA_CACHES, SEND_DEPENDS_TREE] = range(2)
+
+    def __init__(self):
+        self._features=set()
+
+    def setFeature(self, f):
+        # validate we got a request for a feature we support
+        if f not in CookerFeatures._feature_list:
+            return
+        self._features.add(f)
+
+    def __contains__(self, f):
+        return f in self._features
+
+    def __iter__(self):
+        return self._features.__iter__()
+
+    def next(self):
+        return self._features.next()
+
+
 #============================================================================#
 # BBCooker
 #============================================================================#
@@ -90,25 +113,10 @@ class BBCooker:
     def __init__(self, configuration):
         self.recipecache = None
         self.skiplist = {}
+        self.featureset = CookerFeatures()
 
         self.configuration = configuration
 
-        self.caches_array = []
-
-        caches_name_array = ['bb.cache:CoreRecipeInfo'] + configuration.extra_caches
-
-        # At least CoreRecipeInfo will be loaded, so caches_array will never be empty!
-        # This is the entry point, no further check needed!
-        for var in caches_name_array:
-            try:
-                module_name, cache_name = var.split(':')
-                module = __import__(module_name, fromlist=(cache_name,))
-                self.caches_array.append(getattr(module, cache_name)) 
-            except ImportError as exc:
-                logger.critical("Unable to import extra RecipeInfo '%s' from '%s': %s" % (cache_name, module_name, exc))
-                sys.exit("FATAL: Failed to import extra cache class '%s'." % cache_name)
-
-        self.data = None
         self.loadConfigurationData()
 
         # Take a lock so only one copy of bitbake can run against a given build
@@ -118,13 +126,6 @@ class BBCooker:
         if not self.lock:
             bb.fatal("Only one copy of bitbake should be run against a build directory")
 
-        #
-        # Special updated configuration we use for firing events
-        #
-        self.event_data = bb.data.createCopy(self.data)
-        bb.data.update_data(self.event_data)
-        bb.parse.init_parser(self.event_data)
-
         # TOSTOP must not be set or our children will hang when they output
         fd = sys.stdout.fileno()
         if os.isatty(fd):
@@ -141,11 +142,30 @@ class BBCooker:
         self.parser = None
 
     def initConfigurationData(self):
-        worker = False
-        if not self.configuration.server_register_idlecallback:
-            worker = True
 
-        self.databuilder = bb.cookerdata.CookerDataBuilder(self.configuration, worker)
+        self.state = state.initial
+
+        self.caches_array = []
+
+        all_extra_cache_names = []
+        # We hardcode all known cache types in a single place, here.
+        if CookerFeatures.HOB_EXTRA_CACHES in self.featureset:
+            all_extra_cache_names.append("bb.cache_extra:HobRecipeInfo")
+
+        caches_name_array = ['bb.cache:CoreRecipeInfo'] + all_extra_cache_names
+
+        # At least CoreRecipeInfo will be loaded, so caches_array will never be empty!
+        # This is the entry point, no further check needed!
+        for var in caches_name_array:
+            try:
+                module_name, cache_name = var.split(':')
+                module = __import__(module_name, fromlist=(cache_name,))
+                self.caches_array.append(getattr(module, cache_name))
+            except ImportError as exc:
+                logger.critical("Unable to import extra RecipeInfo '%s' from '%s': %s" % (cache_name, module_name, exc))
+                sys.exit("FATAL: Failed to import extra cache class '%s'." % cache_name)
+
+        self.databuilder = bb.cookerdata.CookerDataBuilder(self.configuration, False)
         self.data = self.databuilder.data
 
     def enableDataTracking(self):
@@ -162,6 +182,13 @@ class BBCooker:
         self.data = self.databuilder.data
         self.data_hash = self.databuilder.data_hash
 
+        #
+        # Special updated configuration we use for firing events
+        #
+        self.event_data = bb.data.createCopy(self.data)
+        bb.data.update_data(self.event_data)
+        bb.parse.init_parser(self.event_data)
+
     def modifyConfigurationVar(self, var, val, default_file, op):
         if op == "append":
             self.appendConfigurationVar(var, val, default_file)
@@ -170,7 +197,7 @@ class BBCooker:
 
     def appendConfigurationVar(self, var, val, default_file):
         #add append var operation to the end of default_file
-        default_file = bb.cookerdata.findConfigFile(default_file)
+        default_file = bb.cookerdata.findConfigFile(default_file, self.data)
 
         with open(default_file, 'r') as f:
             contents = f.readlines()
@@ -252,7 +279,7 @@ class BBCooker:
             self.data.varhistory.del_var_history(var)
 
             #add var to the end of default_file
-            default_file = bb.cookerdata.findConfigFile(default_file)
+            default_file = bb.cookerdata.findConfigFile(default_file, self.data)
 
             with open(default_file, 'r') as f:
                 contents = f.readlines()
@@ -348,13 +375,7 @@ class BBCooker:
             if pkgs_to_build[0] in set(ignore.split()):
                 bb.fatal("%s is in ASSUME_PROVIDED" % pkgs_to_build[0])
 
-            localdata = data.createCopy(self.data)
-            bb.data.update_data(localdata)
-            bb.data.expandKeys(localdata)
-
-            taskdata = bb.taskdata.TaskData(self.configuration.abort)
-            taskdata.add_provider(localdata, self.recipecache, pkgs_to_build[0])
-            taskdata.add_unresolved(localdata, self.recipecache)
+            taskdata, runlist, pkgs_to_build = self.buildTaskData(pkgs_to_build, None, self.configuration.abort)
 
             targetid = taskdata.getbuild_id(pkgs_to_build[0])
             fnid = taskdata.build_targets[targetid][0]
@@ -386,34 +407,44 @@ class BBCooker:
             if data.getVarFlag( e, 'python', envdata ):
                 logger.plain("\npython %s () {\n%s}\n", e, data.getVar(e, envdata, 1))
 
-    def prepareTreeData(self, pkgs_to_build, task):
+
+    def buildTaskData(self, pkgs_to_build, task, abort):
         """
         Prepare a runqueue and taskdata object for iteration over pkgs_to_build
         """
         bb.event.fire(bb.event.TreeDataPreparationStarted(), self.data)
 
-        # If we are told to do the None task then query the default task
-        if (task == None):
+        # A task of None means use the default task
+        if task is None:
             task = self.configuration.cmd
 
-        pkgs_to_build = self.checkPackages(pkgs_to_build)
+        fulltargetlist = self.checkPackages(pkgs_to_build)
 
         localdata = data.createCopy(self.data)
         bb.data.update_data(localdata)
         bb.data.expandKeys(localdata)
+        taskdata = bb.taskdata.TaskData(abort, skiplist=self.skiplist)
+
+        current = 0
+        runlist = []
+        for k in fulltargetlist:
+            taskdata.add_provider(localdata, self.recipecache, k)
+            current += 1
+            runlist.append([k, "do_%s" % task])
+            bb.event.fire(bb.event.TreeDataPreparationProgress(current, len(fulltargetlist)), self.data)
+        taskdata.add_unresolved(localdata, self.recipecache)
+        bb.event.fire(bb.event.TreeDataPreparationCompleted(len(fulltargetlist)), self.data)
+        return taskdata, runlist, fulltargetlist
+
+    def prepareTreeData(self, pkgs_to_build, task):
+        """
+        Prepare a runqueue and taskdata object for iteration over pkgs_to_build
+        """
+
         # We set abort to False here to prevent unbuildable targets raising
         # an exception when we're just generating data
-        taskdata = bb.taskdata.TaskData(False, skiplist=self.skiplist)
+        taskdata, runlist, pkgs_to_build = self.buildTaskData(pkgs_to_build, task, False)
 
-        runlist = []
-        current = 0
-        for k in pkgs_to_build:
-            taskdata.add_provider(localdata, self.recipecache, k)
-            runlist.append([k, "do_%s" % task])
-            current += 1
-            bb.event.fire(bb.event.TreeDataPreparationProgress(current, len(pkgs_to_build)), self.data)
-        taskdata.add_unresolved(localdata, self.recipecache)
-        bb.event.fire(bb.event.TreeDataPreparationCompleted(len(pkgs_to_build)), self.data)
         return runlist, taskdata
     
     ######## WARNING : this function requires cache_extra to be enabled ########
@@ -426,7 +457,10 @@ class BBCooker:
         runlist, taskdata = self.prepareTreeData(pkgs_to_build, task)
         rq = bb.runqueue.RunQueue(self, self.data, self.recipecache, taskdata, runlist)
         rq.rqdata.prepare()
+        return self.buildDependTree(rq, taskdata)
 
+
+    def buildDependTree(self, rq, taskdata):
         seen_fnids = []
         depend_tree = {}
         depend_tree["depends"] = {}
@@ -447,6 +481,19 @@ class BBCooker:
                 depend_tree["pn"][pn] = {}
                 depend_tree["pn"][pn]["filename"] = fn
                 depend_tree["pn"][pn]["version"] = version
+
+                # if we have extra caches, list all attributes they bring in
+                extra_info = []
+                for cache_class in self.caches_array:
+                    if type(cache_class) is type and issubclass(cache_class, bb.cache.RecipeInfoCommon) and hasattr(cache_class, 'cachefields'):
+                        cachefields = getattr(cache_class, 'cachefields', [])
+                        extra_info = extra_info + cachefields
+
+                # for all attributes stored, add them to the dependency tree
+                for ei in extra_info:
+                    depend_tree["pn"][pn][ei] = vars(self.recipecache)[ei][fn]
+
+
             for dep in rq.rqdata.runq_depends[task]:
                 depfn = taskdata.fn_index[rq.rqdata.runq_fnid[dep]]
                 deppn = self.recipecache.pkg_fn[depfn]
@@ -509,35 +556,30 @@ class BBCooker:
         depend_tree["rdepends-pkg"] = {}
         depend_tree["rrecs-pkg"] = {}
 
+        # if we have extra caches, list all attributes they bring in
+        extra_info = []
+        for cache_class in self.caches_array:
+            if type(cache_class) is type and issubclass(cache_class, bb.cache.RecipeInfoCommon) and hasattr(cache_class, 'cachefields'):
+                cachefields = getattr(cache_class, 'cachefields', [])
+                extra_info = extra_info + cachefields
+
         for task in xrange(len(tasks_fnid)):
             fnid = tasks_fnid[task]
             fn = taskdata.fn_index[fnid]
             pn = self.recipecache.pkg_fn[fn]
-            version  = "%s:%s-%s" % self.recipecache.pkg_pepvpr[fn]
-            summary = self.recipecache.summary[fn]
-            lic = self.recipecache.license[fn]
-            section = self.recipecache.section[fn]
-            description = self.recipecache.description[fn]
-            homepage = self.recipecache.homepage[fn]
-            bugtracker = self.recipecache.bugtracker[fn]
-            files_info = self.recipecache.files_info[fn]
-            rdepends = self.recipecache.rundeps[fn]
-            rrecs = self.recipecache.runrecs[fn]
-            prevision = self.recipecache.prevision[fn]
-            inherits = self.recipecache.inherits.get(fn, None)
+
             if pn not in depend_tree["pn"]:
                 depend_tree["pn"][pn] = {}
                 depend_tree["pn"][pn]["filename"] = fn
+                version  = "%s:%s-%s" % self.recipecache.pkg_pepvpr[fn]
                 depend_tree["pn"][pn]["version"] = version
-                depend_tree["pn"][pn]["summary"] = summary
-                depend_tree["pn"][pn]["license"] = lic
-                depend_tree["pn"][pn]["section"] = section
-                depend_tree["pn"][pn]["description"] = description
-                depend_tree["pn"][pn]["inherits"] = inherits
-                depend_tree["pn"][pn]["homepage"] = homepage
-                depend_tree["pn"][pn]["bugtracker"] = bugtracker
-                depend_tree["pn"][pn]["files_info"] = files_info
-                depend_tree["pn"][pn]["revision"] = prevision
+                rdepends = self.recipecache.rundeps[fn]
+                rrecs = self.recipecache.runrecs[fn]
+                depend_tree["pn"][pn]["inherits"] = self.recipecache.inherits.get(fn, None)
+
+                # for all extra attributes stored, add them to the dependency tree
+                for ei in extra_info:
+                    depend_tree["pn"][pn][ei] = vars(self.recipecache)[ei][fn]
 
             if fnid not in seen_fnids:
                 seen_fnids.append(fnid)
@@ -697,7 +739,7 @@ class BBCooker:
         Find the location on disk of configfile and if it exists and was parsed by BitBake
         emit the ConfigFilePathFound event with the path to the file.
         """
-        path = bb.cookerdata.findConfigFile(configfile)
+        path = bb.cookerdata.findConfigFile(configfile, self.data)
         if not path:
             return
 
@@ -1047,7 +1089,7 @@ class BBCooker:
 
         def buildFileIdle(server, rq, abort):
 
-            if abort or self.state == state.stop:
+            if abort or self.state == state.forceshutdown:
                 rq.finish_runqueue(True)
             elif self.state == state.shutdown:
                 rq.finish_runqueue(False)
@@ -1076,15 +1118,8 @@ class BBCooker:
         Attempt to build the targets specified
         """
 
-        # If we are told to do the NULL task then query the default task
-        if (task == None):
-            task = self.configuration.cmd
-
-        universe = ('universe' in targets)
-        targets = self.checkPackages(targets)
-
         def buildTargetsIdle(server, rq, abort):
-            if abort or self.state == state.stop:
+            if abort or self.state == state.forceshutdown:
                 rq.finish_runqueue(True)
             elif self.state == state.shutdown:
                 rq.finish_runqueue(False)
@@ -1108,27 +1143,32 @@ class BBCooker:
 
         self.buildSetVars()
 
+        taskdata, runlist, fulltargetlist = self.buildTaskData(targets, task, self.configuration.abort)
+
         buildname = self.data.getVar("BUILDNAME")
-        bb.event.fire(bb.event.BuildStarted(buildname, targets), self.data)
-
-        localdata = data.createCopy(self.data)
-        bb.data.update_data(localdata)
-        bb.data.expandKeys(localdata)
-
-        taskdata = bb.taskdata.TaskData(self.configuration.abort, skiplist=self.skiplist)
-
-        runlist = []
-        for k in targets:
-            taskdata.add_provider(localdata, self.recipecache, k)
-            runlist.append([k, "do_%s" % task])
-        taskdata.add_unresolved(localdata, self.recipecache)
+        bb.event.fire(bb.event.BuildStarted(buildname, fulltargetlist), self.data)
 
         rq = bb.runqueue.RunQueue(self, self.data, self.recipecache, taskdata, runlist)
-        if universe:
+        if 'universe' in targets:
             rq.rqdata.warn_multi_bb = True
 
         self.configuration.server_register_idlecallback(buildTargetsIdle, rq)
 
+
+    def getAllKeysWithFlags(self, flaglist):
+        dump = {}
+        for k in self.data.keys():
+            try:
+                v = self.data.getVar(k, True)
+                if not k.startswith("__") and not isinstance(v, bb.data_smart.DataSmart):
+                    dump[k] = { 'v' : v }
+                    for d in flaglist:
+                        dump[k][d] = self.data.getVarFlag(k, d)
+            except Exception as e:
+                print(e)
+        return dump
+
+
     def generateNewImage(self, image, base_image, package_queue, timestamp, description):
         '''
         Create a new image with a "require"/"inherit" base_image statement
@@ -1173,9 +1213,9 @@ class BBCooker:
         if self.state == state.running:
             return
 
-        if self.state in (state.shutdown, state.stop):
+        if self.state in (state.shutdown, state.forceshutdown):
             self.parser.shutdown(clean=False, force = True)
-            sys.exit(1)
+            raise bb.BBHandledException()
 
         if self.state != state.parsing:
             self.parseConfiguration ()
@@ -1197,7 +1237,7 @@ class BBCooker:
         if not self.parser.parse_next():
             collectlog.debug(1, "parsing complete")
             if self.parser.error:
-                sys.exit(1)
+                raise bb.BBHandledException()
             self.show_appends_with_no_recipes()
             self.handlePrefProviders()
             self.recipecache.bbfile_priority = self.collection.collection_priorities(self.recipecache.pkg_fn)
@@ -1208,6 +1248,9 @@ class BBCooker:
 
     def checkPackages(self, pkgs_to_build):
 
+        # Return a copy, don't modify the original
+        pkgs_to_build = pkgs_to_build[:]
+
         if len(pkgs_to_build) == 0:
             raise NothingToBuild
 
@@ -1243,18 +1286,19 @@ class BBCooker:
         prserv.serv.auto_shutdown(self.data)
         bb.event.fire(CookerExit(), self.event_data)
 
-    def shutdown(self):
-        self.state = state.shutdown
+    def shutdown(self, force = False):
+        if force:
+            self.state = state.forceshutdown
+        else:
+            self.state = state.shutdown
 
-    def stop(self):
-        self.state = state.stop
+    def finishcommand(self):
+        self.state = state.initial
 
     def initialize(self):
-        self.state = state.initial
         self.initConfigurationData()
 
     def reset(self):
-        self.state = state.initial
         self.loadConfigurationData()
 
 def server_main(cooker, func, *args):
@@ -1482,7 +1526,7 @@ class Feeder(multiprocessing.Process):
                 continue
 
 class Parser(multiprocessing.Process):
-    def __init__(self, jobs, results, quit, init):
+    def __init__(self, jobs, results, quit, init, profile):
         self.jobs = jobs
         self.results = results
         self.quit = quit
@@ -1490,8 +1534,28 @@ class Parser(multiprocessing.Process):
         multiprocessing.Process.__init__(self)
         self.context = bb.utils.get_context().copy()
         self.handlers = bb.event.get_class_handlers().copy()
+        self.profile = profile
 
     def run(self):
+
+        if not self.profile:
+            self.realrun()
+            return
+
+        try:
+            import cProfile as profile
+        except:
+            import profile
+        prof = profile.Profile()
+        try:
+            profile.Profile.runcall(prof, self.realrun)
+        finally:
+            logfile = "profile-parse-%s.log" % multiprocessing.current_process().name
+            prof.dump_stats(logfile)
+            bb.utils.process_profilelog(logfile)
+            print("Raw profiling information saved to %s and processed statistics to %s.processed" % (logfile, logfile))
+
+    def realrun(self):
         if self.init:
             self.init()
 
@@ -1592,7 +1656,7 @@ class CookerParser(object):
             self.feeder = Feeder(self.willparse, self.jobs, self.feeder_quit)
             self.feeder.start()
             for i in range(0, self.num_processes):
-                parser = Parser(self.jobs, self.result_queue, self.parser_quit, init)
+                parser = Parser(self.jobs, self.result_queue, self.parser_quit, init, self.cooker.configuration.profile)
                 parser.start()
                 self.processes.append(parser)
 

bitbake/lib/bb/cookerdata.py

@@ -127,7 +127,6 @@ class CookerConfiguration(object):
         self.dump_signatures = False
         self.dry_run = False
         self.tracking = False
-        self.extra_caches = []
 
         self.env = {}
 
@@ -177,14 +176,21 @@ def _inherit(bbclass, data):
     bb.parse.BBHandler.inherit(bbclass, "configuration INHERITs", 0, data)
     return data
 
-def findConfigFile(configfile):
+def findConfigFile(configfile, data):
+    search = []
+    bbpath = data.getVar("BBPATH", True)
+    if bbpath:
+        for i in bbpath.split(":"):
+            search.append(os.path.join(i, "conf", configfile))
     path = os.getcwd()
     while path != "/":
-        confpath = os.path.join(path, "conf", configfile)
-        if os.path.exists(confpath):
-            return confpath
-
+        search.append(os.path.join(path, "conf", configfile))
         path, _ = os.path.split(path)
+
+    for i in search:
+        if os.path.exists(i):
+            return i
+
     return None
 
 class CookerDataBuilder(object):
@@ -225,8 +231,8 @@ class CookerDataBuilder(object):
             logger.exception("Error parsing configuration files")
             sys.exit(1)
 
-    def _findLayerConf(self):
-        return findConfigFile("bblayers.conf")
+    def _findLayerConf(self, data):
+        return findConfigFile("bblayers.conf", data)
 
     def parseConfigurationFiles(self, prefiles, postfiles):
         data = self.data
@@ -236,9 +242,12 @@ class CookerDataBuilder(object):
         for f in prefiles:
             data = parse_config_file(f, data)
 
-        layerconf = self._findLayerConf()
+        layerconf = self._findLayerConf(data)
         if layerconf:
             parselog.debug(2, "Found bblayers.conf (%s)", layerconf)
+            # By definition bblayers.conf is in conf/ of TOPDIR.
+            # We may have been called with cwd somewhere else so reset TOPDIR
+            data.setVar("TOPDIR", os.path.dirname(os.path.dirname(layerconf)))
             data = parse_config_file(layerconf, data)
 
             layers = (data.getVar('BBLAYERS', True) or "").split()

bitbake/lib/bb/data.py

@@ -59,7 +59,7 @@ def init():
 def init_db(parent = None):
     """Return a new object representing the Bitbake data,
     optionally based on an existing object"""
-    if parent:
+    if parent is not None:
         return parent.createCopy()
     else:
         return _dict_type()
@@ -148,7 +148,7 @@ def expandKeys(alterdata, readdata = None):
         readdata = alterdata
 
     todolist = {}
-    for key in keys(alterdata):
+    for key in alterdata:
         if not '${' in key:
             continue
 
@@ -285,20 +285,24 @@ def update_data(d):
     """Performs final steps upon the datastore, including application of overrides"""
     d.finalize(parent = True)
 
-def build_dependencies(key, keys, shelldeps, vardepvals, d):
+def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
     deps = set()
-    vardeps = d.getVarFlag(key, "vardeps", True)
     try:
         if key[-1] == ']':
             vf = key[:-1].split('[')
             value = d.getVarFlag(vf[0], vf[1], False)
-        else:
-            value = d.getVar(key, False)
+            parser = d.expandWithRefs(value, key)
+            deps |= parser.references
+            deps = deps | (keys & parser.execs)
+            return deps, value
+        varflags = d.getVarFlags(key, ["vardeps", "vardepvalue", "vardepsexclude"]) or {}
+        vardeps = varflags.get("vardeps")
+        value = d.getVar(key, False)
 
-        if key in vardepvals:
-           value =  d.getVarFlag(key, "vardepvalue", True)
-        elif d.getVarFlag(key, "func"):
-            if d.getVarFlag(key, "python"):
+        if "vardepvalue" in varflags:
+           value = varflags.get("vardepvalue")
+        elif varflags.get("func"):
+            if varflags.get("python"):
                 parsedvar = d.expandWithRefs(value, key)
                 parser = bb.codeparser.PythonParser(key, logger)
                 if parsedvar.value and "\t" in parsedvar.value:
@@ -320,19 +324,16 @@ def build_dependencies(key, keys, shelldeps, vardepvals, d):
             deps = deps | (keys & parser.execs)
 
         # Add varflags, assuming an exclusion list is set
-        varflagsexcl = d.getVar('BB_SIGNATURE_EXCLUDE_FLAGS', True)
         if varflagsexcl:
             varfdeps = []
-            varflags = d.getVarFlags(key)
-            if varflags:
-                for f in varflags:
-                    if f not in varflagsexcl:
-                        varfdeps.append('%s[%s]' % (key, f))
+            for f in varflags:
+                if f not in varflagsexcl:
+                    varfdeps.append('%s[%s]' % (key, f))
             if varfdeps:
                 deps |= set(varfdeps)
 
         deps |= set((vardeps or "").split())
-        deps -= set((d.getVarFlag(key, "vardepsexclude", True) or "").split())
+        deps -= set(varflags.get("vardepsexclude", "").split())
     except Exception as e:
         raise bb.data_smart.ExpansionError(key, None, e)
     return deps, value
@@ -341,16 +342,16 @@ def build_dependencies(key, keys, shelldeps, vardepvals, d):
 
 def generate_dependencies(d):
 
-    keys = set(key for key in d.keys() if not key.startswith("__"))
-    shelldeps = set(key for key in keys if d.getVarFlag(key, "export") and not d.getVarFlag(key, "unexport"))
-    vardepvals = set(key for key in keys if d.getVarFlag(key, "vardepvalue"))
+    keys = set(key for key in d if not key.startswith("__"))
+    shelldeps = set(key for key in d.getVar("__exportlist", False) if d.getVarFlag(key, "export") and not d.getVarFlag(key, "unexport"))
+    varflagsexcl = d.getVar('BB_SIGNATURE_EXCLUDE_FLAGS', True)
 
     deps = {}
     values = {}
 
     tasklist = d.getVar('__BBTASKS') or []
     for task in tasklist:
-        deps[task], values[task] = build_dependencies(task, keys, shelldeps, vardepvals, d)
+        deps[task], values[task] = build_dependencies(task, keys, shelldeps, varflagsexcl, d)
         newdeps = deps[task]
         seen = set()
         while newdeps:
@@ -359,7 +360,7 @@ def generate_dependencies(d):
             newdeps = set()
             for dep in nextdeps:
                 if dep not in deps:
-                    deps[dep], values[dep] = build_dependencies(dep, keys, shelldeps, vardepvals, d)
+                    deps[dep], values[dep] = build_dependencies(dep, keys, shelldeps, varflagsexcl, d)
                 newdeps |=  deps[dep]
             newdeps -= seen
         #print "For %s: %s" % (task, str(deps[task]))

bitbake/lib/bb/data_smart.py

@@ -40,7 +40,7 @@ logger = logging.getLogger("BitBake.Data")
 
 __setvar_keyword__ = ["_append", "_prepend", "_remove"]
 __setvar_regexp__ = re.compile('(?P<base>.*?)(?P<keyword>_append|_prepend|_remove)(_(?P<add>.*))?$')
-__expand_var_regexp__ = re.compile(r"\${[^{}]+}")
+__expand_var_regexp__ = re.compile(r"\${[^{}@\n\t ]+}")
 __expand_python_regexp__ = re.compile(r"\${@.+?}")
 
 def infer_caller_details(loginfo, parent = False, varval = True):
@@ -94,9 +94,13 @@ class VariableParse:
             if self.varname and key:
                 if self.varname == key:
                     raise Exception("variable %s references itself!" % self.varname)
-            var = self.d.getVar(key, True)
+            if key in self.d.expand_cache:
+                varparse = self.d.expand_cache[key]
+                var = varparse.value
+            else:
+                var = self.d.getVar(key, True)
+            self.references.add(key)
             if var is not None:
-                self.references.add(key)
                 return var
             else:
                 return match.group()
@@ -505,12 +509,7 @@ class DataSmart(MutableMapping):
             self._seen_overrides[override].add( var )
 
     def getVar(self, var, expand=False, noweakdefault=False):
-        value = self.getVarFlag(var, "_content", False, noweakdefault)
-
-        # Call expand() separately to make use of the expand cache
-        if expand and value:
-            return self.expand(value, var)
-        return value
+        return self.getVarFlag(var, "_content", expand, noweakdefault)
 
     def renameVar(self, key, newkey, **loginfo):
         """
@@ -578,17 +577,30 @@ class DataSmart(MutableMapping):
         if flag == "defaultval" and '_' in var:
             self._setvar_update_overrides(var)
 
+        if flag == "unexport" or flag == "export":
+            if not "__exportlist" in self.dict:
+                self._makeShadowCopy("__exportlist")
+            if not "_content" in self.dict["__exportlist"]:
+                self.dict["__exportlist"]["_content"] = set()
+            self.dict["__exportlist"]["_content"].add(var)
+
     def getVarFlag(self, var, flag, expand=False, noweakdefault=False):
         local_var = self._findVar(var)
         value = None
-        if local_var:
+        if local_var is not None:
             if flag in local_var:
                 value = copy.copy(local_var[flag])
             elif flag == "_content" and "defaultval" in local_var and not noweakdefault:
                 value = copy.copy(local_var["defaultval"])
         if expand and value:
-            value = self.expand(value, None)
-        if value and flag == "_content" and local_var and "_removeactive" in local_var:
+            # Only getvar (flag == _content) hits the expand cache
+            cachename = None
+            if flag == "_content":
+                cachename = var
+            else:
+                cachename = var + "[" + flag + "]"
+            value = self.expand(value, cachename)
+        if value is not None and flag == "_content" and local_var is not None and "_removeactive" in local_var:
             filtered = filter(lambda v: v not in local_var["_removeactive"],
                               value.split(" "))
             value = " ".join(filtered)
@@ -636,16 +648,17 @@ class DataSmart(MutableMapping):
             self.varhistory.record(**loginfo)
             self.dict[var][i] = flags[i]
 
-    def getVarFlags(self, var):
+    def getVarFlags(self, var, expand = False, internalflags=False):
         local_var = self._findVar(var)
         flags = {}
 
         if local_var:
             for i in local_var:
-                if i.startswith("_"):
+                if i.startswith("_") and not internalflags:
                     continue
                 flags[i] = local_var[i]
-
+                if expand and i in expand:
+                    flags[i] = self.expand(flags[i], var + "[" + i + "]")
         if len(flags) == 0:
             return None
         return flags
@@ -751,13 +764,16 @@ class DataSmart(MutableMapping):
         for key in keys:
             if key in config_whitelist:
                 continue
+
             value = d.getVar(key, False) or ""
             data.update({key:value})
 
-            varflags = d.getVarFlags(key)
+            varflags = d.getVarFlags(key, internalflags = True)
             if not varflags:
                 continue
             for f in varflags:
+                if f == "_content":
+                    continue
                 data.update({'%s[%s]' % (key, f):varflags[f]})
 
         for key in ["__BBTASKS", "__BBANONFUNCS", "__BBHANDLERS"]:

bitbake/lib/bb/event.py

@@ -589,6 +589,16 @@ class PackageInfo(Event):
         Event.__init__(self)
         self._pkginfolist = pkginfolist
 
+class MetadataEvent(Event):
+    """
+    Generic event that target for OE-Core classes
+    to report information during asynchrous execution
+    """
+    def __init__(self, eventtype, eventdata):
+        Event.__init__(self)
+        self.type = eventtype
+        self.data = eventdata
+
 class SanityCheck(Event):
     """
     Event to issue sanity check

bitbake/lib/bb/providers.py

@@ -341,7 +341,7 @@ def filterProvidersRunTime(providers, item, cfgData, dataCache):
     if numberPreferred > 1:
         logger.error("Trying to resolve runtime dependency %s resulted in conflicting PREFERRED_PROVIDER entries being found.\nThe providers found were: %s\nThe PREFERRED_PROVIDER entries resulting in this conflict were: %s", item, preferred, preferred_vars)
 
-    logger.debug(1, "sorted providers for %s are: %s", item, eligible)
+    logger.debug(1, "sorted runtime providers for %s are: %s", item, eligible)
 
     return eligible, numberPreferred
 

bitbake/lib/bb/runqueue.py

@@ -217,6 +217,15 @@ class RunQueueData:
             ret.extend([nam])
         return ret
 
+    def get_task_name(self, task):
+        return self.runq_task[task]
+
+    def get_task_file(self, task):
+        return self.taskData.fn_index[self.runq_fnid[task]]
+
+    def get_task_hash(self, task):
+        return self.runq_hash[task]
+
     def get_user_idstring(self, task, task_name_suffix = ""):
         fn = self.taskData.fn_index[self.runq_fnid[task]]
         taskname = self.runq_task[task] + task_name_suffix
@@ -696,6 +705,14 @@ class RunQueueData:
                     prov_list[prov].append(fn)
         for prov in prov_list:
             if len(prov_list[prov]) > 1 and prov not in self.multi_provider_whitelist:
+                seen_pn = []
+                # If two versions of the same PN are being built its fatal, we don't support it.
+                for fn in prov_list[prov]:
+                    pn = self.dataCache.pkg_fn[fn]
+                    if pn not in seen_pn:
+                        seen_pn.append(pn)
+                    else:
+                        bb.fatal("Multiple versions of %s are due to be built (%s). Only one version of a given PN should be built in any given build. You likely need to set PREFERRED_VERSION_%s to select the correct version or don't depend on multiple versions." % (pn, " ".join(prov_list[prov]), pn))
                 msg = "Multiple .bb files are due to be built which each provide %s (%s)." % (prov, " ".join(prov_list[prov]))
                 if self.warn_multi_bb:
                     logger.warn(msg)
@@ -703,7 +720,6 @@ class RunQueueData:
                     msg += "\n This usually means one provides something the other doesn't and should."
                     logger.error(msg)
 
-
         # Create a whitelist usable by the stamp checks
         stampfnwhitelist = []
         for entry in self.stampwhitelist.split():
@@ -852,6 +868,9 @@ class RunQueue:
             "logdefaultverboselogs" : bb.msg.loggerVerboseLogs,
             "logdefaultdomain" : bb.msg.loggerDefaultDomains,
             "prhost" : self.cooker.prhost,
+            "buildname" : self.cfgData.getVar("BUILDNAME", True),
+            "date" : self.cfgData.getVar("DATE", True),
+            "time" : self.cfgData.getVar("TIME", True),
         }
 
         worker.stdin.write("<cookerconfig>" + pickle.dumps(self.cooker.configuration) + "</cookerconfig>")
@@ -895,6 +914,14 @@ class RunQueue:
         if self.fakeworkerpipe:
             self.fakeworkerpipe.read()
 
+    def active_fds(self):
+        fds = []
+        if self.workerpipe:
+            fds.append(self.workerpipe.input)
+        if self.fakeworkerpipe:
+            fds.append(self.fakeworkerpipe.input)
+        return fds
+
     def check_stamp_task(self, task, taskname = None, recurse = False, cache = None):
         def get_timestamp(f):
             try:
@@ -972,7 +999,7 @@ class RunQueue:
         (if the abort on failure configuration option isn't set)
         """
 
-        retval = 0.5
+        retval = True
 
         if self.state is runQueuePrepare:
             self.rqexe = RunQueueExecuteDummy(self)
@@ -981,6 +1008,11 @@ class RunQueue:
             else:
                 self.state = runQueueSceneInit
 
+                # we are ready to run, see if any UI client needs the dependency info
+                if bb.cooker.CookerFeatures.SEND_DEPENDS_TREE in self.cooker.featureset:
+                    depgraph = self.cooker.buildDependTree(self, self.rqdata.taskData)
+                    bb.event.fire(bb.event.DepTreeGenerated(depgraph), self.cooker.data)
+
         if self.state is runQueueSceneInit:
             if self.cooker.configuration.dump_signatures:
                 self.dump_signatures()
@@ -1307,9 +1339,10 @@ class RunQueueExecuteTasks(RunQueueExecute):
         if self.rqdata.taskData.abort:
             self.rq.state = runQueueCleanUp
 
-    def task_skip(self, task):
+    def task_skip(self, task, reason):
         self.runq_running[task] = 1
         self.runq_buildable[task] = 1
+        bb.event.fire(runQueueTaskSkipped(task, self.stats, self.rq, reason), self.cfgData)
         self.task_completeoutright(task)
         self.stats.taskCompleted()
         self.stats.taskSkipped()
@@ -1334,13 +1367,13 @@ class RunQueueExecuteTasks(RunQueueExecute):
             if task in self.rq.scenequeue_covered:
                 logger.debug(2, "Setscene covered task %s (%s)", task,
                                 self.rqdata.get_user_idstring(task))
-                self.task_skip(task)
+                self.task_skip(task, "covered")
                 return True
 
             if self.rq.check_stamp_task(task, taskname, cache=self.stampcache):
                 logger.debug(2, "Stamp current task %s (%s)", task,
                                 self.rqdata.get_user_idstring(task))
-                self.task_skip(task)
+                self.task_skip(task, "existing")
                 return True
 
             taskdep = self.rqdata.dataCache.task_deps[fn]
@@ -1375,7 +1408,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
 
         if self.stats.active > 0:
             self.rq.read_workers()
-            return 0.5
+            return self.rq.active_fds()
 
         if len(self.failed_fnids) != 0:
             self.rq.state = runQueueFailed
@@ -1627,6 +1660,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
 
     def task_complete(self, task):
         self.stats.taskCompleted()
+        bb.event.fire(sceneQueueTaskCompleted(task, self.stats, self.rq), self.cfgData)
         self.task_completeoutright(task)
 
     def task_fail(self, task, result):
@@ -1715,7 +1749,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
 
         if self.stats.active > 0:
             self.rq.read_workers()
-            return 0.5
+            return self.rq.active_fds()
 
         # Convert scenequeue_covered task numbers into full taskgraph ids
         oldcovered = self.scenequeue_covered
@@ -1761,6 +1795,9 @@ class runQueueEvent(bb.event.Event):
     def __init__(self, task, stats, rq):
         self.taskid = task
         self.taskstring = rq.rqdata.get_user_idstring(task)
+        self.taskname = rq.rqdata.get_task_name(task)
+        self.taskfile = rq.rqdata.get_task_file(task)
+        self.taskhash = rq.rqdata.get_task_hash(task)
         self.stats = stats.copy()
         bb.event.Event.__init__(self)
 
@@ -1772,6 +1809,9 @@ class sceneQueueEvent(runQueueEvent):
         runQueueEvent.__init__(self, task, stats, rq)
         realtask = rq.rqdata.runq_setscene[task]
         self.taskstring = rq.rqdata.get_user_idstring(realtask, "_setscene")
+        self.taskname = rq.rqdata.get_task_name(realtask) + "_setscene"
+        self.taskfile = rq.rqdata.get_task_file(realtask)
+        self.taskhash = rq.rqdata.get_task_hash(task)
 
 class runQueueTaskStarted(runQueueEvent):
     """
@@ -1810,6 +1850,19 @@ class runQueueTaskCompleted(runQueueEvent):
     Event notifing a task completed
     """
 
+class sceneQueueTaskCompleted(sceneQueueEvent):
+    """
+    Event notifing a setscene task completed
+    """
+
+class runQueueTaskSkipped(runQueueEvent):
+    """
+    Event notifing a task was skipped
+    """
+    def __init__(self, task, stats, rq, reason):
+        runQueueEvent.__init__(self, task, stats, rq)
+        self.reason = reason
+
 class runQueuePipe():
     """
     Abstraction for a pipe between a worker thread and the server

bitbake/lib/bb/server/__init__.py

@@ -89,7 +89,7 @@ class BitBakeBaseServer(object):
     def detach(self):
         return
 
-    def establishConnection(self):
+    def establishConnection(self, featureset):
         raise   "Must redefine the %s.establishConnection()" % self.__class__.__name__
 
     def endSession(self):

bitbake/lib/bb/server/process.py

@@ -29,8 +29,9 @@ import os
 import signal
 import sys
 import time
+import select
 from Queue import Empty
-from multiprocessing import Event, Process, util, Queue, Pipe, queues
+from multiprocessing import Event, Process, util, Queue, Pipe, queues, Manager
 
 from . import BitBakeBaseServer, BitBakeBaseServerConnection, BaseImplServer
 
@@ -77,12 +78,13 @@ class ProcessServer(Process, BaseImplServer):
     profile_filename = "profile.log"
     profile_processed_filename = "profile.log.processed"
 
-    def __init__(self, command_channel, event_queue):
+    def __init__(self, command_channel, event_queue, featurelist):
         BaseImplServer.__init__(self)
-        Process.__init__(self)
+        Process.__init__(self, args=(featurelist))
         self.command_channel = command_channel
         self.event_queue = event_queue
         self.event = EventAdapter(event_queue)
+        self.featurelist = featurelist
         self.quit = False
 
         self.keep_running = Event()
@@ -93,6 +95,14 @@ class ProcessServer(Process, BaseImplServer):
         for event in bb.event.ui_queue:
             self.event_queue.put(event)
         self.event_handle.value = bb.event.register_UIHhandler(self)
+
+        # process any feature changes based on what UI requested
+        original_featureset = list(self.cooker.featureset)
+        while len(self.featurelist)> 0:
+            self.cooker.featureset.setFeature(self.featurelist.pop())
+        if (original_featureset != list(self.cooker.featureset)):
+            self.cooker.reset()
+
         bb.cooker.server_main(self.cooker, self.main)
 
     def main(self):
@@ -105,17 +115,17 @@ class ProcessServer(Process, BaseImplServer):
                     command = self.command_channel.recv()
                     self.runCommand(command)
 
-                self.idle_commands(.1)
+                self.idle_commands(.1, [self.event_queue._reader, self.command_channel])
             except Exception:
                 logger.exception('Running command %s', command)
 
         self.event_queue.close()
         bb.event.unregister_UIHhandler(self.event_handle.value)
         self.command_channel.close()
-        self.cooker.stop()
+        self.cooker.shutdown(True)
         self.idle_commands(.1)
 
-    def idle_commands(self, delay):
+    def idle_commands(self, delay, fds = []):
         nextsleep = delay
 
         for function, data in self._idlefuns.items():
@@ -127,15 +137,15 @@ class ProcessServer(Process, BaseImplServer):
                     nextsleep = None
                 elif nextsleep is None:
                     continue
-                elif retval < nextsleep:
-                    nextsleep = retval
+                else:
+                    fds = fds + retval
             except SystemExit:
                 raise
             except Exception:
                 logger.exception('Running idle function')
 
         if nextsleep is not None:
-            time.sleep(nextsleep)
+            select.select(fds,[],[],nextsleep)
 
     def runCommand(self, command):
         """
@@ -197,13 +207,17 @@ class BitBakeServer(BitBakeBaseServer):
         #
         self.ui_channel, self.server_channel = Pipe()
         self.event_queue = ProcessEventQueue(0)
-        self.serverImpl = ProcessServer(self.server_channel, self.event_queue)
+        manager = Manager()
+        self.featurelist = manager.list()
+        self.serverImpl = ProcessServer(self.server_channel, self.event_queue, self.featurelist)
 
     def detach(self):
         self.serverImpl.start()
         return
 
-    def establishConnection(self):
+    def establishConnection(self, featureset):
+        for f in featureset:
+            self.featurelist.append(f)
         self.connection = BitBakeProcessServerConnection(self.serverImpl, self.ui_channel, self.event_queue)
         signal.signal(signal.SIGTERM, lambda i, s: self.connection.terminate())
         return self.connection

bitbake/lib/bb/server/xmlrpc.py

@@ -78,7 +78,7 @@ class BBTransport(xmlrpclib.Transport):
             h.putheader("Bitbake-token", self.connection_token)
         xmlrpclib.Transport.send_content(self, h, body)
 
-def _create_server(host, port, timeout = 5):
+def _create_server(host, port, timeout = 60):
     t = BBTransport(timeout)
     s = xmlrpclib.Server("http://%s:%d/" % (host, port), transport=t, allow_none=True)
     return s, t
@@ -89,12 +89,23 @@ class BitBakeServerCommands():
         self.server = server
         self.has_client = False
 
-    def registerEventHandler(self, host, port):
+    def registerEventHandler(self, host, port, featureset = []):
         """
         Register a remote UI Event Handler
         """
         s, t = _create_server(host, port)
 
+        # we don't allow connections if the cooker is running
+        if (self.cooker.state in [bb.cooker.state.parsing, bb.cooker.state.running]):
+            return None
+
+        original_featureset = list(self.cooker.featureset)
+        for f in featureset:
+            self.cooker.featureset.setFeature(f)
+
+        if (original_featureset != list(self.cooker.featureset)):
+            self.cooker.reset()
+
         self.event_handle = bb.event.register_UIHhandler(s)
         return self.event_handle
 
@@ -133,6 +144,8 @@ class BitBakeServerCommands():
         if self.has_client:
             self.server.set_connection_token(None)
             self.has_client = False
+            if self.server.single_use:
+                self.server.quit = True
 
 # This request handler checks if the request has a "Bitbake-token" header
 # field (this comes from the client side) and compares it with its internal
@@ -167,51 +180,6 @@ class BitBakeXMLRPCRequestHandler(SimpleXMLRPCRequestHandler):
         self.end_headers()
         self.wfile.write(response)
 
-class BitBakeUIEventServer(threading.Thread):
-    class EventAdapter():
-        """
-        Adapter to wrap our event queue since the caller (bb.event) expects to
-        call a send() method, but our actual queue only has put()
-        """
-        def __init__(self, notify):
-            self.queue = []
-            self.notify = notify
-            self.qlock = threading.Lock()
-
-        def send(self, event):
-            self.qlock.acquire()
-            self.queue.append(event)
-            self.qlock.release()
-            self.notify.set()
-
-        def get(self):
-            self.qlock.acquire()
-            if len(self.queue) == 0:
-                self.qlock.release()
-                return None
-            e = self.queue.pop(0)
-            if len(self.queue) == 0:
-                self.notify.clear()
-            self.qlock.release()
-            return e
-
-    def __init__(self, connection):
-        self.connection = connection
-        self.notify = threading.Event()
-        self.event = BitBakeUIEventServer.EventAdapter(self.notify)
-        self.quit = False
-        threading.Thread.__init__(self)
-
-    def terminateServer(self):
-        self.quit = True
-
-    def run(self):
-        while not self.quit:
-            self.notify.wait(0.1)
-            evt = self.event.get()
-            if evt:
-                self.connection.event.sendpickle(pickle.dumps(evt))
-
 
 class XMLRPCProxyServer(BaseImplServer):
     """ not a real working server, but a stub for a proxy server connection
@@ -239,6 +207,9 @@ class XMLRPCServer(SimpleXMLRPCServer, BaseImplServer):
         self.commands = BitBakeServerCommands(self)
         self.autoregister_all_functions(self.commands, "")
         self.interface = interface
+        self.single_use = False
+        if (interface[1] == 0):     # anonymous port, not getting reused
+            self.single_use = True
 
     def addcooker(self, cooker):
         BaseImplServer.addcooker(self, cooker)
@@ -264,12 +235,9 @@ class XMLRPCServer(SimpleXMLRPCServer, BaseImplServer):
         Serve Requests. Overloaded to honor a quit command
         """
         self.quit = False
-        self.timeout = 0 # Run Idle calls for our first callback
         while not self.quit:
-            #print "Idle queue length %s" % len(self._idlefuns)
-            self.handle_request()
-            #print "Idle timeout, running idle functions"
-            nextsleep = None
+            fds = [self]
+            nextsleep = 0.1
             for function, data in self._idlefuns.items():
                 try:
                     retval = function(self, data, False)
@@ -277,21 +245,22 @@ class XMLRPCServer(SimpleXMLRPCServer, BaseImplServer):
                         del self._idlefuns[function]
                     elif retval is True:
                         nextsleep = 0
-                    elif nextsleep is 0:
-                        continue
-                    elif nextsleep is None:
-                        nextsleep = retval
-                    elif retval < nextsleep:
-                        nextsleep = retval
+                    else:
+                        fds = fds + retval
                 except SystemExit:
                     raise
                 except:
                     import traceback
                     traceback.print_exc()
                     pass
-            if nextsleep is None and len(self._idlefuns) > 0:
-                nextsleep = 0
-            self.timeout = nextsleep
+
+            socktimeout = self.socket.gettimeout() or nextsleep
+            socktimeout = min(socktimeout, nextsleep)
+            # Mirror what BaseServer handle_request would do
+            fd_sets = select.select(fds, [], [], socktimeout)
+            if fd_sets[0] and self in fd_sets[0]:
+                self._handle_request_noblock()
+
         # Tell idle functions we're exiting
         for function, data in self._idlefuns.items():
             try:
@@ -305,11 +274,12 @@ class XMLRPCServer(SimpleXMLRPCServer, BaseImplServer):
         self.connection_token = token
 
 class BitBakeXMLRPCServerConnection(BitBakeBaseServerConnection):
-    def __init__(self, serverImpl, clientinfo=("localhost", 0), observer_only = False):
+    def __init__(self, serverImpl, clientinfo=("localhost", 0), observer_only = False, featureset = []):
         self.connection, self.transport = _create_server(serverImpl.host, serverImpl.port)
         self.clientinfo = clientinfo
         self.serverImpl = serverImpl
         self.observer_only = observer_only
+        self.featureset = featureset
 
     def connect(self):
         if not self.observer_only:
@@ -319,7 +289,8 @@ class BitBakeXMLRPCServerConnection(BitBakeBaseServerConnection):
         if token is None:
             return None
         self.transport.set_connection_token(token)
-        self.events = uievent.BBUIEventQueue(self.connection, self.clientinfo)
+
+        self.events = uievent.BBUIEventQueue(self.connection, self.clientinfo, self.featureset)
         for event in bb.event.ui_queue:
             self.events.queue_event(event)
         return self
@@ -343,14 +314,15 @@ class BitBakeXMLRPCServerConnection(BitBakeBaseServerConnection):
 
 class BitBakeServer(BitBakeBaseServer):
     def initServer(self, interface = ("localhost", 0)):
+        self.interface = interface
         self.serverImpl = XMLRPCServer(interface)
 
     def detach(self):
         daemonize.createDaemon(self.serverImpl.serve_forever, "bitbake-cookerdaemon.log")
         del self.cooker
 
-    def establishConnection(self):
-        self.connection = BitBakeXMLRPCServerConnection(self.serverImpl)
+    def establishConnection(self, featureset):
+        self.connection = BitBakeXMLRPCServerConnection(self.serverImpl, self.interface, False, featureset)
         return self.connection.connect()
 
     def set_connection_token(self, token):
@@ -360,12 +332,13 @@ class BitBakeXMLRPCClient(BitBakeBaseServer):
 
     def __init__(self, observer_only = False):
         self.observer_only = observer_only
+        # if we need extra caches, just tell the server to load them all
         pass
 
     def saveConnectionDetails(self, remote):
         self.remote = remote
 
-    def establishConnection(self):
+    def establishConnection(self, featureset):
         # The format of "remote" must be "server:port"
         try:
             [host, port] = self.remote.split(":")
@@ -382,7 +355,7 @@ class BitBakeXMLRPCClient(BitBakeBaseServer):
         except:
             return None
         self.serverImpl = XMLRPCProxyServer(host, port)
-        self.connection = BitBakeXMLRPCServerConnection(self.serverImpl, (ip, 0), self.observer_only)
+        self.connection = BitBakeXMLRPCServerConnection(self.serverImpl, (ip, 0), self.observer_only, featureset)
         return self.connection.connect()
 
     def endSession(self):

bitbake/lib/bb/siggen.py

@@ -91,8 +91,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
         basehash = {}
 
         for task in tasklist:
-            data = d.getVar(task, False)
-            lookupcache[task] = data
+            data = lookupcache[task]
 
             if data is None:
                 bb.error("Task %s from %s seems to be empty?!" % (task, fn))
@@ -115,16 +114,8 @@ class SignatureGeneratorBasic(SignatureGenerator):
             alldeps = sorted(seen)
             for dep in alldeps:
                 data = data + dep
-                if dep in lookupcache:
-                    var = lookupcache[dep]
-                elif dep[-1] == ']':
-                    vf = dep[:-1].split('[')
-                    var = d.getVarFlag(vf[0], vf[1], False)
-                    lookupcache[dep] = var
-                else:
-                    var = d.getVar(dep, False)
-                    lookupcache[dep] = var
-                if var:
+                var = lookupcache[dep]
+                if var is not None:
                     data = data + str(var)
             self.basehash[fn + "." + task] = hashlib.md5(data).hexdigest()
             taskdeps[task] = alldeps

bitbake/lib/bb/tests/data.py

@@ -27,21 +27,21 @@ import bb.data
 class DataExpansions(unittest.TestCase):
     def setUp(self):
         self.d = bb.data.init()
-        self.d["foo"] = "value of foo"
-        self.d["bar"] = "value of bar"
-        self.d["value of foo"] = "value of 'value of foo'"
+        self.d["foo"] = "value_of_foo"
+        self.d["bar"] = "value_of_bar"
+        self.d["value_of_foo"] = "value_of_'value_of_foo'"
 
     def test_one_var(self):
         val = self.d.expand("${foo}")
-        self.assertEqual(str(val), "value of foo")
+        self.assertEqual(str(val), "value_of_foo")
 
     def test_indirect_one_var(self):
         val = self.d.expand("${${foo}}")
-        self.assertEqual(str(val), "value of 'value of foo'")
+        self.assertEqual(str(val), "value_of_'value_of_foo'")
 
     def test_indirect_and_another(self):
         val = self.d.expand("${${foo}} ${bar}")
-        self.assertEqual(str(val), "value of 'value of foo' value of bar")
+        self.assertEqual(str(val), "value_of_'value_of_foo' value_of_bar")
 
     def test_python_snippet(self):
         val = self.d.expand("${@5*12}")
@@ -49,11 +49,11 @@ class DataExpansions(unittest.TestCase):
 
     def test_expand_in_python_snippet(self):
         val = self.d.expand("${@'boo ' + '${foo}'}")
-        self.assertEqual(str(val), "boo value of foo")
+        self.assertEqual(str(val), "boo value_of_foo")
 
     def test_python_snippet_getvar(self):
         val = self.d.expand("${@d.getVar('foo', True) + ' ${bar}'}")
-        self.assertEqual(str(val), "value of foo value of bar")
+        self.assertEqual(str(val), "value_of_foo value_of_bar")
 
     def test_python_snippet_syntax_error(self):
         self.d.setVar("FOO", "${@foo = 5}")
@@ -70,7 +70,7 @@ class DataExpansions(unittest.TestCase):
 
     def test_value_containing_value(self):
         val = self.d.expand("${@d.getVar('foo', True) + ' ${bar}'}")
-        self.assertEqual(str(val), "value of foo value of bar")
+        self.assertEqual(str(val), "value_of_foo value_of_bar")
 
     def test_reference_undefined_var(self):
         val = self.d.expand("${undefinedvar} meh")
@@ -109,7 +109,7 @@ class DataExpansions(unittest.TestCase):
 
     def test_rename(self):
         self.d.renameVar("foo", "newfoo")
-        self.assertEqual(self.d.getVar("newfoo"), "value of foo")
+        self.assertEqual(self.d.getVar("newfoo"), "value_of_foo")
         self.assertEqual(self.d.getVar("foo"), None)
 
     def test_deletion(self):
@@ -118,17 +118,17 @@ class DataExpansions(unittest.TestCase):
 
     def test_keys(self):
         keys = self.d.keys()
-        self.assertEqual(keys, ['value of foo', 'foo', 'bar'])
+        self.assertEqual(keys, ['value_of_foo', 'foo', 'bar'])
 
 class TestNestedExpansions(unittest.TestCase):
     def setUp(self):
         self.d = bb.data.init()
         self.d["foo"] = "foo"
         self.d["bar"] = "bar"
-        self.d["value of foobar"] = "187"
+        self.d["value_of_foobar"] = "187"
 
     def test_refs(self):
-        val = self.d.expand("${value of ${foo}${bar}}")
+        val = self.d.expand("${value_of_${foo}${bar}}")
         self.assertEqual(str(val), "187")
 
     #def test_python_refs(self):
@@ -154,11 +154,11 @@ class TestNestedExpansions(unittest.TestCase):
     #    self.assertEqual(str(val), str(depth + 1))
 
     def test_mixed(self):
-        val = self.d.expand("${value of ${@('${foo}'+'bar')[0:3]}${${@'BAR'.lower()}}}")
+        val = self.d.expand("${value_of_${@('${foo}'+'bar')[0:3]}${${@'BAR'.lower()}}}")
         self.assertEqual(str(val), "187")
 
     def test_runtime(self):
-        val = self.d.expand("${${@'value of' + ' f'+'o'+'o'+'b'+'a'+'r'}}")
+        val = self.d.expand("${${@'value_of' + '_f'+'o'+'o'+'b'+'a'+'r'}}")
         self.assertEqual(str(val), "187")
 
 class TestMemoize(unittest.TestCase):

bitbake/lib/bb/ui/crumbs/hig/saveimagedialog.py

@@ -70,9 +70,11 @@ class SaveImageDialog (CrumbsDialog):
         sub_label.set_alignment(0, 0)
         sub_label.set_markup("The description should be less than 150 characters long.")
         self.description_entry = gtk.TextView()
-        self.description_entry.get_buffer().set_text(self.description_field)
+        description_buffer = self.description_entry.get_buffer()
+        description_buffer.set_text(self.description_field)
+        description_buffer.connect("insert-text", self.limit_description_length)
         self.description_entry.set_wrap_mode(gtk.WRAP_WORD)
-        self.description_entry.set_size_request(350,150)
+        self.description_entry.set_size_request(350,50)
         sub_vbox.pack_start(label, expand=False, fill=False)
         sub_vbox.pack_start(sub_label, expand=False, fill=False)
         sub_vbox.pack_start(self.description_entry, expand=False, fill=False)
@@ -108,6 +110,13 @@ class SaveImageDialog (CrumbsDialog):
 
         self.show_all()
 
+    def limit_description_length(self, textbuffer, iter, text, length):
+        buffer_bounds = textbuffer.get_bounds()
+        entire_text = textbuffer.get_text(*buffer_bounds)
+        entire_text += text
+        if len(entire_text)>150 or text=="\n":
+            textbuffer.emit_stop_by_name("insert-text")
+
     def name_entry_changed(self, entry):
         text = entry.get_text()
         if text == '':

bitbake/lib/bb/ui/crumbs/hobeventhandler.py

@@ -166,13 +166,11 @@ class HobHandler(gobject.GObject):
             self.clear_busy()
             self.building = True
             targets = [self.image]
-            if self.package_queue:
-                self.set_var_in_file("LINGUAS_INSTALL", "", "local.conf")
-                self.set_var_in_file("PACKAGE_INSTALL", " ".join(self.package_queue), "local.conf")
             if self.toolchain_packages:
                 self.set_var_in_file("TOOLCHAIN_TARGET_TASK", " ".join(self.toolchain_packages), "local.conf")
                 targets.append(self.toolchain)
             if targets[0] == "hob-image":
+                self.set_var_in_file("LINGUAS_INSTALL", "", "local.conf")
                 hobImage = self.runCommand(["matchFile", "hob-image.bb"])
                 if self.base_image != "Start with an empty image recipe":
                     baseImage = self.runCommand(["matchFile", self.base_image + ".bb"])
@@ -442,12 +440,12 @@ class HobHandler(gobject.GObject):
         self.building = False
 
     def cancel_parse(self):
-        self.runCommand(["stateStop"])
+        self.runCommand(["stateForceShutdown"])
 
     def cancel_build(self, force=False):
         if force:
             # Force the cooker to stop as quickly as possible
-            self.runCommand(["stateStop"])
+            self.runCommand(["stateForceShutdown"])
         else:
             # Wait for tasks to complete before shutting down, this helps
             # leave the workdir in a usable state

bitbake/lib/bb/ui/crumbs/hoblistmodel.py

@@ -690,7 +690,7 @@ class RecipeListModel(gtk.ListStore):
             inherits = event_model["pn"][item]["inherits"]
             summary = event_model["pn"][item]["summary"]
             version = event_model["pn"][item]["version"]
-            revision = event_model["pn"][item]["revision"]
+            revision = event_model["pn"][item]["prevision"]
             homepage = event_model["pn"][item]["homepage"]
             bugtracker = event_model["pn"][item]["bugtracker"]
             filename = event_model["pn"][item]["filename"]

bitbake/lib/bb/ui/crumbs/imageconfigurationpage.py

@@ -387,6 +387,10 @@ class ImageConfigurationPage (HobPage):
                 selected_image = self.__dummy_image__
                 self.update_image_combo(self.builder.recipe_model, None)
             dialog.destroy()
+        else:
+            if self.custom_image_selected:
+                self.custom_image_selected = None
+                self.update_image_combo(self.builder.recipe_model, selected_image)
 
         if not selected_image or (selected_image == self.__dummy_image__):
             self.builder.window_sensitive(True)

bitbake/lib/bb/ui/crumbs/imagedetailspage.py

@@ -635,6 +635,14 @@ class ImageDetailsPage (HobPage):
         images_dir = topdir + "/recipes/images/"
         self.builder.ensure_dir(images_dir)
 
+        self.name_field_template = self.builder.image_configuration_page.custom_image_selected
+        if self.name_field_template:
+            image_path = self.builder.recipe_model.pn_path[self.name_field_template]
+            image_iter = self.builder.recipe_model.get_iter(image_path)
+            self.description_field_template = self.builder.recipe_model.get_value(image_iter, self.builder.recipe_model.COL_DESC)
+        else:
+            self.name_field_template = ""
+
         dialog = SaveImageDialog(images_dir, self.name_field_template, self.description_field_template,
                  "Save image recipe", self.builder, gtk.DIALOG_MODAL | gtk.DIALOG_DESTROY_WITH_PARENT)
         response = dialog.run()

bitbake/lib/bb/ui/depexp.py

@@ -314,7 +314,7 @@ def main(server, eventHandler, params):
                 break
             if shutdown == 1:
                 print("\nSecond Keyboard Interrupt, stopping...\n")
-                _, error = server.runCommand(["stateStop"])
+                _, error = server.runCommand(["stateForceShutdown"])
                 if error:
                     print('Unable to cleanly stop: %s' % error)
             if shutdown == 0:

bitbake/lib/bb/ui/goggle.py

@@ -117,5 +117,5 @@ def main (server, eventHandler, params):
     except KeyboardInterrupt:
         pass
     finally:
-        server.runCommand(["stateStop"])
+        server.runCommand(["stateForceShutdown"])
 

bitbake/lib/bb/ui/hob.py

@@ -46,7 +46,7 @@ from bb.ui.crumbs.hoblistmodel import RecipeListModel, PackageListModel
 from bb.ui.crumbs.hobeventhandler import HobHandler
 from bb.ui.crumbs.builder import Builder
 
-extraCaches = ['bb.cache_extra:HobRecipeInfo']
+featureSet = [bb.cooker.CookerFeatures.HOB_EXTRA_CACHES]
 
 def event_handle_idle_func(eventHandler, hobHandler):
     # Consume as many messages as we can in the time available to us

bitbake/lib/bb/ui/knotty.py

@@ -306,8 +306,10 @@ def main(server, eventHandler, params, tf = TerminalFilter):
 
     while True:
         try:
-            termfilter.updateFooter()
-            event = eventHandler.waitEvent(0.25)
+            event = eventHandler.waitEvent(0)
+            if event is None:
+                termfilter.updateFooter()
+                event = eventHandler.waitEvent(0.25)
             if event is None:
                 if main.shutdown > 1:
                     break
@@ -403,8 +405,9 @@ def main(server, eventHandler, params, tf = TerminalFilter):
 
             if isinstance(event, bb.command.CommandFailed):
                 return_value = event.exitcode
-                errors = errors + 1
-                logger.error("Command execution failed: %s", event.error)
+                if event.error:
+                    errors = errors + 1
+                    logger.error("Command execution failed: %s", event.error)
                 main.shutdown = 2
                 continue
             if isinstance(event, bb.command.CommandExit):
@@ -469,8 +472,12 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                              event.taskid, event.taskstring, event.exitcode)
                 continue
 
+            if isinstance(event, bb.event.DepTreeGenerated):
+                continue
+
             # ignore
             if isinstance(event, (bb.event.BuildBase,
+                                  bb.event.MetadataEvent,
                                   bb.event.StampUpdate,
                                   bb.event.ConfigParsed,
                                   bb.event.RecipeParsed,
@@ -497,7 +504,7 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                 main.shutdown = 2
             if not params.observe_only and main.shutdown == 1:
                 print("\nSecond Keyboard Interrupt, stopping...\n")
-                _, error = server.runCommand(["stateStop"])
+                _, error = server.runCommand(["stateForceShutdown"])
                 if error:
                     logger.error("Unable to cleanly stop: %s" % error)
             if not params.observe_only and main.shutdown == 0:
@@ -518,7 +525,7 @@ def main(server, eventHandler, params, tf = TerminalFilter):
     if warnings:
         summary += pluralise("\nSummary: There was %s WARNING message shown.",
                              "\nSummary: There were %s WARNING messages shown.", warnings)
-    if return_value:
+    if return_value and errors:
         summary += pluralise("\nSummary: There was %s ERROR message shown, returning a non-zero exit code.",
                              "\nSummary: There were %s ERROR messages shown, returning a non-zero exit code.", errors)
     if summary:

bitbake/lib/bb/ui/ncurses.py

@@ -350,7 +350,7 @@ class NCursesUI:
                     exitflag = True
                 if shutdown == 1:
                     mw.appendText("Second Keyboard Interrupt, stopping...\n")
-                    _, error = server.runCommand(["stateStop"])
+                    _, error = server.runCommand(["stateForceShutdown"])
                     if error:
                         print("Unable to cleanly stop: %s" % error)
                 if shutdown == 0:

bitbake/lib/bb/ui/uievent.py

@@ -28,7 +28,7 @@ import socket, threading, pickle
 from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
 
 class BBUIEventQueue:
-    def __init__(self, BBServer, clientinfo=("localhost, 0")):
+    def __init__(self, BBServer, clientinfo=("localhost, 0"), featureset=[]):
 
         self.eventQueue = []
         self.eventQueueLock = threading.Lock()
@@ -44,7 +44,10 @@ class BBUIEventQueue:
         server.register_function( self.send_event, "event.sendpickle" )
         server.socket.settimeout(1)
 
-        self.EventHandle = self.BBServer.registerEventHandler(self.host, self.port)
+        self.EventHandle = self.BBServer.registerEventHandler(self.host, self.port, featureset)
+
+        if (self.EventHandle == None):
+            bb.fatal("Could not register UI event handler")
 
         self.server = server
 

bitbake/lib/bb/utils.py

@@ -532,6 +532,8 @@ def approved_variables():
     Determine and return the list of whitelisted variables which are approved
     to remain in the envrionment.
     """
+    if 'BB_PRESERVE_ENV' in os.environ:
+        return os.environ.keys()
     approved = []
     if 'BB_ENV_WHITELIST' in os.environ:
         approved = os.environ['BB_ENV_WHITELIST'].split()

bitbake/lib/prserv/db.py

@@ -2,6 +2,7 @@ import logging
 import os.path
 import errno
 import prserv
+import time
 
 try:
     import sqlite3
@@ -32,16 +33,20 @@ class PRTable(object):
 
     def _execute(self, *query):
         """Execute a query, waiting to acquire a lock if necessary"""
-        count = 0
+        start = time.time()
+        end = start + 20
         while True:
             try:
                 return self.conn.execute(*query)
             except sqlite3.OperationalError as exc:
-                if 'database is locked' in str(exc) and count < 500:
-                    count = count + 1
+                if 'is locked' in str(exc) and end > time.time():
                     continue
                 raise exc
 
+    def sync(self):
+        self.conn.commit()
+        self._execute("BEGIN EXCLUSIVE TRANSACTION")
+
     def _getValueHist(self, version, pkgarch, checksum):
         data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
                            (version, pkgarch, checksum))
@@ -51,11 +56,9 @@ class PRTable(object):
         else:
             #no value found, try to insert
             try:
-                self._execute("BEGIN")
-                self._execute("INSERT OR ROLLBACK INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1,0) from %s where version=? AND pkgarch=?));"
+                self._execute("INSERT INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1,0) from %s where version=? AND pkgarch=?));"
                            % (self.table,self.table),
                            (version,pkgarch, checksum,version, pkgarch))
-                self.conn.commit()
             except sqlite3.IntegrityError as exc:
                 logger.error(str(exc))
 
@@ -79,11 +82,9 @@ class PRTable(object):
         else:
             #no value found, try to insert
             try:
-                self._execute("BEGIN")
                 self._execute("INSERT OR REPLACE INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1,0) from %s where version=? AND pkgarch=?));"
                                % (self.table,self.table),
                                (version, pkgarch, checksum, version, pkgarch))
-                self.conn.commit()
             except sqlite3.IntegrityError as exc:
                 logger.error(str(exc))
                 self.conn.rollback()
@@ -112,10 +113,8 @@ class PRTable(object):
         else:
             #no value found, try to insert
             try:
-                self._execute("BEGIN")
-                self._execute("INSERT OR ROLLBACK INTO %s VALUES (?, ?, ?, ?);"  % (self.table),
+                self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);"  % (self.table),
                            (version, pkgarch, checksum, value))
-                self.conn.commit()
             except sqlite3.IntegrityError as exc:
                 logger.error(str(exc))
 
@@ -129,18 +128,14 @@ class PRTable(object):
     def _importNohist(self, version, pkgarch, checksum, value):
         try:
             #try to insert
-            self._execute("BEGIN")
-            self._execute("INSERT OR ROLLBACK INTO %s VALUES (?, ?, ?, ?);"  % (self.table),
+            self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);"  % (self.table),
                            (version, pkgarch, checksum,value))
-            self.conn.commit()
         except sqlite3.IntegrityError as exc:
             #already have the record, try to update
             try:
-                self._execute("BEGIN")
                 self._execute("UPDATE %s SET value=? WHERE version=? AND pkgarch=? AND checksum=? AND value<?"  
                               % (self.table),
                                (value,version,pkgarch,checksum,value))
-                self.conn.commit()
             except sqlite3.IntegrityError as exc:
                 logger.error(str(exc))
 
@@ -223,7 +218,7 @@ class PRData(object):
         except OSError as e:
             if e.errno != errno.EEXIST:
                 raise e
-        self.connection=sqlite3.connect(self.filename, isolation_level="DEFERRED", check_same_thread = False)
+        self.connection=sqlite3.connect(self.filename, isolation_level="EXCLUSIVE", check_same_thread = False)
         self.connection.row_factory=sqlite3.Row
         self._tables={}
 

bitbake/lib/prserv/serv.py

@@ -2,6 +2,8 @@ import os,sys,logging
 import signal, time, atexit, threading
 from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
 import xmlrpclib
+import threading
+import Queue
 
 try:
     import sqlite3
@@ -31,14 +33,11 @@ class Handler(SimpleXMLRPCRequestHandler):
 PIDPREFIX = "/tmp/PRServer_%s_%s.pid"
 singleton = None
 
-import SocketServer
-class SimpleThreadedXMLRPCServer(SocketServer.ThreadingMixIn, SimpleXMLRPCServer):
-    pass
 
-class PRServer(SimpleThreadedXMLRPCServer):
+class PRServer(SimpleXMLRPCServer):
     def __init__(self, dbfile, logfile, interface, daemon=True):
         ''' constructor '''
-        SimpleThreadedXMLRPCServer.__init__(self, interface,
+        SimpleXMLRPCServer.__init__(self, interface,
                                     logRequests=False, allow_none=True)
         self.dbfile=dbfile
         self.daemon=daemon
@@ -54,19 +53,41 @@ class PRServer(SimpleThreadedXMLRPCServer):
         self.register_function(self.importone, "importone")
         self.register_introspection_functions()
 
+        self.db = prserv.db.PRData(self.dbfile)
+        self.table = self.db["PRMAIN"]
+
+        self.requestqueue = Queue.Queue()
+        self.handlerthread = threading.Thread(target = self.process_request_thread)
+        self.handlerthread.daemon = False
+
+    def process_request_thread(self):
+        """Same as in BaseServer but as a thread.
+
+        In addition, exception handling is done here.
+
+        """
+        while True:
+            (request, client_address) = self.requestqueue.get()
+            try:
+                self.finish_request(request, client_address)
+                self.shutdown_request(request)
+            except:
+                self.handle_error(request, client_address)
+                self.shutdown_request(request)
+                self.table.sync()
+
+    def process_request(self, request, client_address):
+        self.requestqueue.put((request, client_address))
+
     def export(self, version=None, pkgarch=None, checksum=None, colinfo=True):
         try:
-            db = prserv.db.PRData(self.dbfile)
-            table = db["PRMAIN"]
-            return table.export(version, pkgarch, checksum, colinfo)
+            return self.table.export(version, pkgarch, checksum, colinfo)
         except sqlite3.Error as exc:
             logger.error(str(exc))
             return None
 
     def importone(self, version, pkgarch, checksum, value):
-        db = prserv.db.PRData(self.dbfile)
-        table = db["PRMAIN"]
-        return table.importone(version, pkgarch, checksum, value)
+        return self.table.importone(version, pkgarch, checksum, value)
 
     def ping(self):
         return not self.quit
@@ -76,9 +97,7 @@ class PRServer(SimpleThreadedXMLRPCServer):
 
     def getPR(self, version, pkgarch, checksum):
         try:
-            db = prserv.db.PRData(self.dbfile)
-            table = db["PRMAIN"]
-            return table.getValue(version, pkgarch, checksum)
+            return self.table.getValue(version, pkgarch, checksum)
         except prserv.NotFoundError:
             logger.error("can not find value for (%s, %s)",version, checksum)
             return None
@@ -97,9 +116,11 @@ class PRServer(SimpleThreadedXMLRPCServer):
         logger.info("Started PRServer with DBfile: %s, IP: %s, PORT: %s, PID: %s" %
                      (self.dbfile, self.host, self.port, str(os.getpid())))
 
+        self.handlerthread.start()
         while not self.quit:
             self.handle_request()
 
+        self.table.sync()
         logger.info("PRServer: stopping...")
         self.server_close()
         return

documentation/Makefile

@@ -200,7 +200,8 @@ TARFILES = mega-manual.html mega-style.css figures/yocto-environment.png figures
 	figures/yocto-environment-ref.png figures/user-configuration.png figures/source-input.png \
 	figures/package-feeds.png figures/layer-input.png figures/images.png figures/sdk.png \
 	figures/source-fetching.png figures/patching.png figures/configuration-compile-autoreconf.png \
-	figures/analysis-for-package-splitting.png
+	figures/analysis-for-package-splitting.png figures/image-generation.png \
+	figures/sdk-generation.png
 	endif
 
 MANUALS = $(DOC)/$(DOC).html
@@ -219,7 +220,8 @@ TARFILES = ref-manual.html ref-style.css figures/poky-title.png \
 	figures/user-configuration.png figures/yocto-environment-ref.png \
 	figures/images.png figures/sdk.png figures/source-fetching.png \
 	figures/patching.png figures/configuration-compile-autoreconf.png \
-	figures/analysis-for-package-splitting.png
+	figures/analysis-for-package-splitting.png figures/image-generation.png \
+	figures/sdk-generation.png
 MANUALS = $(DOC)/$(DOC).html $(DOC)/$(DOC).pdf $(DOC)/eclipse
 FIGURES = figures
 STYLESHEET = $(DOC)/*.css
@@ -381,8 +383,8 @@ publish:
             echo " "; \
             echo "******** Publishing "$(DOC)".html"; \
             echo " "; \
-            scp -r $(MANUALS) $(STYLESHEET) www.yoctoproject.org:/srv/www/www.yoctoproject.org-docs/$(VER)/$(DOC); \
-            cd $(DOC); scp -r $(FIGURES) www.yoctoproject.org:/srv/www/www.yoctoproject.org-docs/$(VER)/$(DOC); \
+            scp -r $(MANUALS) $(STYLESHEET) docs.yp:/var/www/www.yoctoproject.org-docs/$(VER)/$(DOC); \
+            cd $(DOC); scp -r $(FIGURES) docs.yp:/var/www/www.yoctoproject.org-docs/$(VER)/$(DOC); \
 	else \
           echo " "; \
           echo $(DOC)".html missing.  Generate the file first then try again."; \

documentation/adt-manual/adt-prepare.xml

@@ -91,7 +91,9 @@
             <para>
                 If you use BitBake to generate the ADT Installer tarball, you must
                 <filename>source</filename> the environment setup script
-                (<ulink url='&YOCTO_DOCS_REF_URL;#structure-core-script'><filename>&OE_INIT_FILE;</filename></ulink>)
+                (<ulink url='&YOCTO_DOCS_REF_URL;#structure-core-script'><filename>&OE_INIT_FILE;</filename></ulink>
+                or
+                <ulink url='&YOCTO_DOCS_REF_URL;#structure-memres-core-script'><filename>oe-init-build-env-memres</filename></ulink>)
                 located in the Source Directory before running the
                 BitBake command that creates the tarball.
             </para>
@@ -266,7 +268,7 @@
                             target, go into the <filename>x86_64</filename>
                             folder and download the following installer:
                             <literallayout class='monospaced'>
-     poky-eglibc-x86_64-core-image-sato-i586.sh
+     poky-eglibc-x86_64-core-image-sato-i586-&DISTRO;.sh
                             </literallayout></para></listitem>
                         <listitem><para>Build your own toolchain installer.
                             For cases where you cannot use an installer
@@ -285,7 +287,7 @@
                     The example assumes the toolchain installer is located
                     in <filename>~/Downloads/</filename>.
                     <literallayout class='monospaced'>
-     $ ~/Downloads/poky-eglibc-x86_64-core-image-sato-i586.sh
+     $ ~/Downloads/poky-eglibc-x86_64-core-image-sato-i586-&DISTRO;.sh
                     </literallayout>
                     <note>
                         If you do not have write permissions for the directory
@@ -374,10 +376,11 @@
     </para>
 
     <para>
-        Be sure to run the environment setup script that matches the architecture for
-        which you are developing.
-        Environment setup scripts begin with the string "<filename>environment-setup</filename>"
-        and include as part of their name the architecture.
+        Be sure to run the environment setup script that matches the
+        architecture for which you are developing.
+        Environment setup scripts begin with the string
+        "<filename>environment-setup</filename>" and include as part of their
+        name the architecture.
         For example, the toolchain environment setup script for a 64-bit
         IA-based architecture installed in the default installation directory
         would be the following:
@@ -540,7 +543,7 @@
 
     <para>
         A final method of building the toolchain installer exists that has
-        significant advantages over the previous two methods.
+        significant advantages over the previous method.
         This method results in a toolchain installer that contains the sysroot
         that matches your target root filesystem.
         To build this installer, use the
@@ -549,8 +552,12 @@
 
     <para>
          Remember, before using any <filename>bitbake</filename> command, you
-         must source the <filename>&OE_INIT_PATH;</filename> script located in
-         the Source Directory and you must make sure your
+         must source the build environment setup script
+         (i.e.
+         <ulink url='&YOCTO_DOCS_REF_URL;#structure-core-script'><filename>&OE_INIT_FILE;</filename></ulink>
+         or
+         <ulink url='&YOCTO_DOCS_REF_URL;#structure-memres-core-script'><filename>oe-init-build-env-memres</filename></ulink>)
+         located in the Source Directory and you must make sure your
          <filename>conf/local.conf</filename> variables are correct.
          In particular, you need to be sure the
          <ulink url='&YOCTO_DOCS_REF_URL;#var-MACHINE'><filename>MACHINE</filename></ulink>

documentation/dev-manual/dev-manual-common-tasks.xml

@@ -228,65 +228,192 @@
                             </note></para></listitem>
                 </orderedlist>
             </para>
+        </section>
+
+        <section id='best-practices-to-follow-when-creating-layers'>
+            <title>Best Practices to Follow When Creating Layers</title>
 
             <para>
-                To create layers that are easier to maintain, you should
-                consider the following:
-                <itemizedlist>
-                    <listitem><para>Avoid "overlaying" entire recipes from
-                        other layers in your configuration.
-                        In other words, do not copy an entire recipe into your
-                        layer and then modify it.
-                        Use <filename>.bbappend</filename> files to override the
-                        parts of the recipe you need to modify.
-                        </para></listitem>
-                    <listitem><para>Avoid duplicating include files.
-                        Use <filename>.bbappend</filename> files for each recipe
-                        that uses an include file.
-                        Or, if you are introducing a new recipe that requires
-                        the included file, use the path relative to the original
-                        layer directory to refer to the file.
-                        For example, use
-                        <filename>require recipes-core/somepackage/somefile.inc</filename>
-                        instead of <filename>require somefile.inc</filename>.
-                        If you're finding you have to overlay the include file,
-                        it could indicate a deficiency in the include file in
-                        the layer to which it originally belongs.
-                        If this is the case, you need to address that deficiency
-                        instead of overlaying the include file.
-                        For example, consider how Qt 4 database support plug-ins
-                        are configured.
-                        The Source Directory does not have MySQL or PostgreSQL,
-                        however OpenEmbedded's layer
-                        <filename>meta-oe</filename> does.
-                        Consequently, <filename>meta-oe</filename> uses
-                        <filename>.bbappend</filename> files to modify the
-                        <filename>QT_SQL_DRIVER_FLAGS</filename> variable to
-                        enable the appropriate plugins.
-                        This variable was added to the
-                        <filename>qt4.inc</filename> include file in the Source
-                        Directory specifically to allow the
-                        <filename>meta-oe</filename> layer to be able to control
-                        which plugins are built.</para></listitem>
-                </itemizedlist>
+                To create layers that are easier to maintain and that will
+                not impact builds for other machines, you should consider the
+                information in the following sections.
             </para>
 
-            <para>
-                We also recommend the following:
-                <itemizedlist>
-                    <listitem><para>Store custom layers in a Git repository
-                        that uses the
-                        <filename>meta-&lt;layer_name&gt;</filename> format.
-                        </para></listitem>
-                    <listitem><para>Clone the repository alongside other
-                        <filename>meta</filename> directories in the
-                        <link linkend='source-directory'>Source Directory</link>.
-                        </para></listitem>
-                 </itemizedlist>
-                 Following these recommendations keeps your Source Directory and
-                 its configuration entirely inside the Yocto Project's core
-                 base.
-            </para>
+            <section id='avoid-overlaying-entire-recipes'>
+                <title>Avoid "Overlaying" Entire Recipes</title>
+
+                <para>
+                    Avoid "overlaying" entire recipes from other layers in your
+                    configuration.
+                    In other words, do not copy an entire recipe into your
+                    layer and then modify it.
+                    Use <filename>.bbappend</filename> files to override the
+                    parts of the recipe you need to modify.
+                </para>
+            </section>
+
+            <section id='avoid-duplicating-include-files'>
+                <title>Avoid Duplicating Include Files</title>
+
+                <para>
+                    Avoid duplicating include files.
+                    Use <filename>.bbappend</filename> files for each recipe
+                    that uses an include file.
+                    Or, if you are introducing a new recipe that requires
+                    the included file, use the path relative to the original
+                    layer directory to refer to the file.
+                    For example, use
+                    <filename>require recipes-core/somepackage/somefile.inc</filename>
+                    instead of <filename>require somefile.inc</filename>.
+                    If you're finding you have to overlay the include file,
+                    it could indicate a deficiency in the include file in
+                    the layer to which it originally belongs.
+                    If this is the case, you need to address that deficiency
+                    instead of overlaying the include file.
+                    For example, consider how support plug-ins for the Qt 4
+                    database are configured.
+                    The Source Directory does not have MySQL or PostgreSQL.
+                    However, OpenEmbedded's layer <filename>meta-oe</filename>
+                    does.
+                    Consequently, <filename>meta-oe</filename> uses
+                    <filename>.bbappend</filename> files to modify the
+                    <filename>QT_SQL_DRIVER_FLAGS</filename> variable to
+                    enable the appropriate plug-ins.
+                    This variable was added to the <filename>qt4.inc</filename>
+                    include file in the Source Directory specifically to allow
+                    the <filename>meta-oe</filename> layer to be able to control
+                    which plug-ins are built.
+                </para>
+            </section>
+
+            <section id='structure-your-layers'>
+                <title>Structure Your Layers</title>
+
+                <para>
+                    Proper use of overrides within append files and placement
+                    of machine-specific files within your layer can ensure that
+                    a build is not using the wrong Metadata and negatively
+                    impacting a build for a different machine.
+                    Following are some examples:
+                    <itemizedlist>
+                        <listitem><para><emphasis>Modifying Variables to support
+                            a different machine:</emphasis>
+                            Suppose you have a layer named
+                            <filename>meta-one</filename> that adds support
+                            for building machine "one".
+                            To do so, you use an append file named
+                            <filename>base-files.bbappend</filename> and
+                            create a dependency on "foo" by altering the
+                            <ulink url='&YOCTO_DOCS_REF_URL;#var-DEPENDS'><filename>DEPENDS</filename></ulink>
+                            variable:
+                            <literallayout class='monospaced'>
+     DEPENDS = "foo"
+                            </literallayout>
+                            The dependency is created during any build that
+                            includes the layer
+                            <filename>meta-one</filename>.
+                            However, you might not want this dependency
+                            for all machines.
+                            For example, suppose you are building for
+                            machine "two" but your
+                            <filename>bblayers.conf</filename> file has the
+                            <filename>meta-one</filename> layer included.
+                            During the build, the
+                            <filename>base-files</filename> for machine
+                            "two" will also have the dependency on
+                            <filename>foo</filename>.</para>
+                            <para>To make sure your changes apply only when
+                            building machine "one", use a machine override
+                            with the <filename>DEPENDS</filename> statement:
+                            <literallayout class='monospaced'>
+     DEPENDS_one = "foo"
+                            </literallayout>
+                            You should follow the same strategy when using
+                            <filename>_append</filename> and
+                            <filename>_prepend</filename> operations:
+                            <literallayout class='monospaced'>
+     DEPENDS_append_one = " foo"
+     DEPENDS_prepend_one = "foo "
+                            </literallayout>
+                            <note>
+                                Avoiding "+=" and "=+" and using
+                                machine-specific
+                                <filename>_append</filename>
+                                and <filename>_prepend</filename> operations
+                                is recommended as well.
+                            </note></para></listitem>
+                        <listitem><para><emphasis>Place Machine-Specific Files
+                            in Machine-Specific Locations:</emphasis>
+                            When you have a base recipe, such as
+                            <filename>base-files.bb</filename>, that
+                            contains a
+                            <ulink url='&YOCTO_DOCS_REF_URL;#var-SRC_URI'><filename>SRC_URI</filename></ulink>
+                            statement to a file, you can use an append file
+                            to cause the build to use your own version of
+                            the file.
+                            For example, an append file in your layer at
+                            <filename>/meta-one/recipes-core/base-files/base-files.bbappend</filename>
+                            could extend
+                            <ulink url='&YOCTO_DOCS_REF_URL;#var-FILESPATH'><filename>FILESPATH</filename></ulink>
+                            using
+                            <ulink url='&YOCTO_DOCS_REF_URL;#var-FILESEXTRAPATHS'><filename>FILESEXTRAPATHS</filename></ulink>
+                            as follows:
+                            <literallayout class='monospaced'>
+     FILESEXTRAPATHS_prepend := "${THISDIR}/${BPN}:"
+                            </literallayout>
+                            The build for machine "one" will pick up your
+                            machine-specific file as long as you have the
+                            file in
+                            <filename>/meta-one/recipes-core/base-files/base-files/</filename>.
+                            However, if you are building for a different
+                            machine and the
+                            <filename>bblayers.conf</filename> file includes
+                            the <filename>meta-one</filename> layer and
+                            the location of your machine-specific file is
+                            the first location where that file is found
+                            according to <filename>FILESPATH</filename>,
+                            builds for all machines will also use that
+                            machine-specific file.</para>
+                            <para>You can make sure that a machine-specific
+                            file is used for a particular machine by putting
+                            the file in a subdirectory specific to the
+                            machine.
+                            For example, rather than placing the file in
+                            <filename>/meta-one/recipes-core/base-files/base-files/</filename>
+                            as shown above, put it in
+                            <filename>/meta-one/recipes-core/base-files/base-files/one/</filename>.
+                            Not only does this make sure the file is used
+                            only when building for machine "one" but the
+                            build process locates the file more quickly.</para>
+                            <para>In summary, you need to place all files
+                            referenced from <filename>SRC_URI</filename>
+                            in a machine-specific subdirectory within the
+                            layer in order to restrict those files to
+                            machine-specific builds.</para></listitem>
+                    </itemizedlist>
+                </para>
+            </section>
+
+            <section id='other-recommendations'>
+                <title>Other Recommendations</title>
+
+                <para>
+                    We also recommend the following:
+                    <itemizedlist>
+                        <listitem><para>Store custom layers in a Git repository
+                            that uses the
+                            <filename>meta-&lt;layer_name&gt;</filename> format.
+                            </para></listitem>
+                        <listitem><para>Clone the repository alongside other
+                            <filename>meta</filename> directories in the
+                            <link linkend='source-directory'>Source Directory</link>.
+                            </para></listitem>
+                     </itemizedlist>
+                     Following these recommendations keeps your Source Directory and
+                     its configuration entirely inside the Yocto Project's core
+                     base.
+                </para>
+            </section>
         </section>
 
         <section id='enabling-your-layer'>
@@ -1812,6 +1939,8 @@
                 environment, you must build the tool using BitBake.
                 Thus, the environment must be set up using the
                 <ulink url='&YOCTO_DOCS_REF_URL;#structure-core-script'><filename>&OE_INIT_FILE;</filename></ulink>
+                or
+                <ulink url='&YOCTO_DOCS_REF_URL;#structure-memres-core-script'><filename>oe-init-build-env-memres</filename></ulink>
                 script found in the
                 <link linkend='build-directory'>Build Directory</link>.
                 The following commands build and invoke <filename>menuconfig</filename> assuming the
@@ -2467,9 +2596,12 @@
                             If you have distro-specific configuration files
                             that are included by an existing recipe, you should
                             add a <filename>.bbappend</filename> for those.
-                            For general information on how to add recipes to
-                            your layer, see the "<link linkend='creating-your-own-layer'>Creating Your Own Layer</link>"
-                            section.</para></listitem>
+                            For general information and recommendations
+                            on how to add recipes to your layer, see the
+                            "<link linkend='creating-your-own-layer'>Creating Your Own Layer</link>"
+                            and
+                            "<link linkend='best-practices-to-follow-when-creating-layers'>Best Practices to Follow When Creating Layers</link>"
+                            sections.</para></listitem>
                         <listitem><para>Add any image recipes that are specific
                             to your distribution.</para></listitem>
                         <listitem><para>Add a <filename>psplash</filename>
@@ -2924,6 +3056,8 @@
         <para>
             This section describes a few tasks that involve packages:
             <itemizedlist>
+                <listitem><para>Excluding packages from an image
+                    </para></listitem>
                 <listitem><para>Incrementing a package revision number
                     </para></listitem>
                 <listitem><para>Handling a package name alias
@@ -2938,6 +3072,50 @@
             </itemizedlist>
         </para>
 
+        <section id='excluding-packages-from-an-image'>
+            <title>Excluding Packages from an Image</title>
+
+            <para>
+                You might find it necessary to prevent specific packages
+                from being installed into an image.
+                If so, you can use several variables to direct the build
+                system to essentially ignore installing recommended packages
+                or to not install a package at all.
+            </para>
+
+            <para>
+                The following list introduces variables you can use to
+                prevent packages from being installed into your image.
+                Each of these variables only works with IPK and RPM
+                package types.
+                Support for Debian packages does not exist.
+                Also, you can use these variables from your
+                <filename>local.conf</filename> file or attach them to a
+                specific image recipe by using a recipe name override.
+                For more detail on the variables, see the descriptions in the
+                Yocto Project Reference Manual's glossary chapter.
+                <itemizedlist>
+                    <listitem><para><ulink url='&YOCTO_DOCS_REF_URL;#var-BAD_RECOMMENDATIONS'><filename>BAD_RECOMMENDATIONS</filename></ulink>:
+                        Use this variable to specify "recommended-only"
+                        packages that you do not want installed.
+                        </para></listitem>
+                    <listitem><para><ulink url='&YOCTO_DOCS_REF_URL;#var-NO_RECOMMENDATIONS'><filename>NO_RECOMMENDATIONS</filename></ulink>:
+                        Use this variable to prevent all "recommended-only"
+                        packages from being installed.
+                        </para></listitem>
+                    <listitem><para><ulink url='&YOCTO_DOCS_REF_URL;#var-PACKAGE_EXCLUDE'><filename>PACKAGE_EXCLUDE</filename></ulink>:
+                        Use this variable to prevent specific packages from
+                        being installed regardless of whether they are
+                        "recommended-only" or not.
+                        You need to realize that the build process could
+                        fail with an error when you
+                        prevent the installation of a package whose presence
+                        is required by an installed package.
+                        </para></listitem>
+                </itemizedlist>
+            </para>
+        </section>
+
         <section id='incrementing-a-package-revision-number'>
             <title>Incrementing a Package Revision Number</title>
 
@@ -3215,7 +3393,7 @@
 
             <para>
                 Many pieces of software split functionality into optional
-                modules (or plugins) and the plugins that are built
+                modules (or plug-ins) and the plug-ins that are built
                 might depend on configuration options.
                 To avoid having to duplicate the logic that determines what
                 modules are available in your recipe or to avoid having
@@ -4074,6 +4252,420 @@
         </section>
     </section>
 
+    <section id="running-and-writing-tests-for-a-qemu-image">
+        <title>Running and Writing Tests for a QEMU Image</title>
+
+        <para>
+            The OpenEmbedded build system makes available a series of automated
+            tests for QEMU images.
+            These tests are commands that run on the target system over
+            <filename>ssh</filename> and are written in Python, thus making
+            use of the <filename>unittest</filename> module.
+        </para>
+
+        <para>
+            This section describes how you set up the environment to use these
+            tests, run available tests, and write and add your own tests.
+            You can the same information in a different form and with example
+            log files of test runs on the
+            <ulink url='https://wiki.yoctoproject.org/wiki/Image_tests'>Image Tests</ulink>
+            Wiki page.
+        </para>
+
+        <section id="qemu-image-enabling-tests">
+            <title>Enabling Tests</title>
+
+            <para>
+                In order to run tests, you need to do the following:
+                <itemizedlist>
+                    <listitem><para><emphasis>Ensure you run the test
+                        automatically with no interaction and under
+                        <filename>sudo</filename>:</emphasis>
+                        To accomplish this, you must do one of the
+                        following:
+                        <itemizedlist>
+                            <listitem><para>Add
+                                <filename>NOPASSWD</filename> for your user
+                                in <filename>/etc/sudoers</filename> either for
+                                ALL commands or just for
+                                <filename>runqemu-ifup</filename>.
+                                You must provide the full path as that can
+                                change if you are using multiple clones of the
+                                source repository.
+                                <note>
+                                    On some distributions, you also need to
+                                    comment out "Defaults requiretty" in
+                                    <filename>/etc/sudoers</filename>.
+                                </note></para></listitem>
+                            <listitem><para>Manually configure a tap interface
+                                for your system.</para></listitem>
+                            <listitem><para>Run as root the script in
+                                <filename>scripts/runqemu-gen-tapdev</filename>,
+                                which should generate a list of tap devices.
+                                List generation is usually done in environments
+                                similar to AutoBuilder.</para></listitem>
+                        </itemizedlist></para></listitem>
+                    <listitem><para><emphasis>Set up the
+                        <filename>DISPLAY</filename> variable:</emphasis>
+                        You need to set this variable so that you have an X
+                        server available (e.g. start
+                        <filename>vncserver</filename> for a headless machine).
+                        </para></listitem>
+                    <listitem><para><emphasis>Be sure your host's firewall
+                        accepts incoming connections from
+                        192.168.7.0/24:</emphasis>
+                        Some of the tests (in particular smart tests) start a
+                        HTTP server on a random high number port, which is
+                        used to serve files to the target.
+                        The smart module serves
+                        <filename>${DEPLOY_DIR}/rpm</filename> so it can run
+                        smart channel commands. That means your host's firewall
+                        must accept incoming connections from 192.168.7.0/24,
+                        which is the default class used for
+                        <filename>tap0</filename> devices by
+                        <filename>runqemu</filename>.</para></listitem>
+                    <listitem><para><emphasis>Have your QEMU image built:</emphasis>
+                        The QEMU image on which you want to run tests needs to
+                        be built.
+                        For example, the following command would build the
+                        <filename>core-image-sato</filename> image when
+                        <filename>MACHINE</filename> is set to "qemu".
+                        <literallayout class='monospaced'>
+    bitbake core-image-sato
+                        </literallayout></para></listitem>
+                    <listitem><para><emphasis>Globally inherit
+                        <ulink url='&YOCTO_DOCS_REF_URL;#ref-classes-testimage'><filename>testimage.class</filename>:</ulink></emphasis>
+                        Edit your <filename>local.conf</filename> file as
+                        follows:
+                        <literallayout class='monospaced'>
+    INHERIT += "testimage"
+                        </literallayout></para></listitem>
+                </itemizedlist>
+            </para>
+
+            <note>
+                If you built your QEMU image using <filename>rm_work</filename>,
+                most of the tests will fail with errors because they rely on
+                <filename>${WORKDIR}/installed_pkgs.txt</filename>.
+            </note>
+        </section>
+
+        <section id="qemu-image-running-tests">
+            <title>Running Tests</title>
+
+            <para>
+                After setting up to run the tests, use BitBake to start
+                the standard suite of tests:
+                <literallayout class='monospaced'>
+     bitbake core-image-sato -c testimage
+                </literallayout>
+            </para>
+
+            <para>
+                Once you start the tests, the following happens:
+                <itemizedlist>
+                    <listitem><para>A copy of the root filesystem is written
+                        to <filename>${WORKDIR}/testimage</filename>.
+                        </para></listitem>
+                    <listitem><para>The image is booted under QEMU using the
+                        standard <filename>runqemu</filename> script.
+                        </para></listitem>
+                    <listitem><para>A default timeout of 500 seconds occurs
+                        to allow for the boot process to reach the login prompt.
+                        You can change the timeout period by setting
+                        <ulink url='&YOCTO_DOCS_REF_URL;#var-TEST_QEMUBOOT_TIMEOUT'><filename>TEST_QEMUBOOT_TIMEOUT</filename></ulink>
+                        in the <filename>local.conf</filename> file.
+                        </para></listitem>
+                    <listitem><para>Once the boot process is reached and the
+                        login prompt appears, the tests run.
+                        The full boot log is written to
+                        <filename>${WORKDIR}/testimage/qemu_boot_log</filename>.
+                        </para></listitem>
+                    <listitem><para>Each test module loads in the order found
+                        in <filename>TEST_SUITES</filename>.
+                        You can find the full output of the commands run over
+                        <filename>ssh</filename> in
+                        <filename>${WORKDIR}/testimgage/ssh_target_log</filename>.
+                        </para></listitem>
+                    <listitem><para>If no failures occur, the task running the
+                        tests ends successfully.
+                        You can find the output from the
+                        <filename>unittest</filename> in the task log at
+                        <filename>${WORKDIR}/temp/log.do_testimage</filename>.
+                        </para></listitem>
+                </itemizedlist>
+            </para>
+
+            <para>
+                All test files reside in
+                <filename>meta/lib/oeqa/runtime</filename> in the
+                <link linkend='source-directory'>Source Directory</link>.
+                Tests (also referred to as modules) have a one-to-one
+                relationship with the filenames in <filename>runtime</filename>.
+                Modules can have multiple classes and test methods and are
+                usually grouped together by the area tested (e.g tests for
+                <filename>systemd</filename> reside in
+                <filename>meta/lib/oeqa/runtime/systemd.py</filename>).
+            </para>
+
+            <para>
+                You can add tests to any layer provided you place them in the
+                proper area and you extend
+                <ulink url='&YOCTO_DOCS_REF_URL;#var-BBPATH'><filename>BBPATH</filename></ulink>
+                in the <filename>local.conf</filename> file.
+                Be sure that tests reside in
+                <filename>&lt;layer&gt;/lib/oeqa/runtime</filename>.
+                <note>
+                    Be sure that module names do not collide with module names
+                    provided by the Yocto Project.
+                </note>
+            </para>
+
+            <para>
+                You can change the set of tests run by appending or overriding
+                <ulink url='&YOCTO_DOCS_REF_URL;#var-TEST_SUITES'><filename>TEST_SUITES</filename></ulink>
+                variable in <filename>local.conf</filename>.
+                Each name in <filename>TEST_SUITES</filename> represents a
+                required test for the image.
+                Module skipping is not allowed even if a test is not suitable
+                for an image (e.g. running the <filename>rpm</filename> tests on
+                an image without <filename>rpm</filename>).
+                Appending "auto" to <filename>TEST_SUITES</filename> causes the
+                build system to try to run all tests that are suitable for the
+                image (i.e. the build system decides whether to run each test).
+            </para>
+
+            <para>
+                The order you list tests in <filename>TEST_SUITES</filename>
+                is important.
+                The order influences test dependencies.
+                Consequently, tests that depend on other tests should be added
+                after the test on which they depend.
+                For example, since <filename>ssh</filename> depends on the
+                <filename>ping</filename> test, <filename>ssh</filename>
+                needs to come after <filename>ping</filename> in the list.
+                The test class provides no re-ordering or dependency handling.
+                <note>
+                    Each module can have multiple classes with multiple test
+                    methods.
+                    And, Python <filename>unittest</filename> rules apply.
+                </note>
+            </para>
+
+            <para>
+                The following list summarizes how to run the tests:
+                <itemizedlist>
+                    <listitem><para>Run the default set of tests simply by
+                        running the following:
+                        <literallayout class='monospaced'>
+     bitbake &lt;qemu_image&gt; -c testimage
+                        </literallayout></para></listitem>
+                    <listitem><para>The default tests for the image are defined
+                        as:
+                        <literallayout class='monospaced'>
+     DEFAULT_TEST_SUITES_pn-&lt;qemu_image&gt; = "ping ssh df connman syslog xorg scp vnc date rpm smart dmesg"
+                        </literallayout></para></listitem>
+                    <listitem><para>Add your own test to the list of the
+                        by using the following:
+                        <literallayout class='monospaced'>
+     TEST_SUITES_append = " mytest"
+                        </literallayout></para></listitem>
+                    <listitem><para>Run a specific list of tests as follows:
+                        <literallayout class='monospaced'>
+     TEST_SUITES = "test1 test2 test3"
+                        </literallayout>
+                        Remember, order is important.
+                        Be sure to place a test that is dependent on another test
+                        later in the order.</para></listitem>
+                </itemizedlist>
+            </para>
+        </section>
+
+        <section id="qemu-image-writing-new-tests">
+            <title>Writing New Tests</title>
+
+            <para>
+                As mentioned previously, all new test files need to be in the
+                proper place for the build system to find them.
+                New tests can go in the
+                <link linkend='source-directory'>Source Directory</link> at
+                <filename>meta/lib/oeqa/runtime</filename>.
+                Alternatively, a layer can add its own tests in
+                <filename>&lt;layer&gt;/lib/oeqa/runtime</filename> as long
+                as you extend <filename>BBPATH</filename>.
+                Just remember that filenames need to map directory to test
+                (module) names and you not use module names that collide with
+                existing core tests.
+            </para>
+
+            <para>
+                To create a new test, start by copying an existing module
+                (e.g. <filename>syslog.py</filename> or
+                <filename>gcc.py</filename> are good ones to use).
+                Test modules can use code from
+                <filename>meta/lib/oeqa/utils</filename>, which are helper
+                classes.
+            </para>
+
+            <note>
+                Structure shell commands such that you rely on them and they
+                return a single code for success.
+                Be aware that sometimes you will need to parse the output.
+                See the <filename>df.py</filename> and
+                <filename>date.py</filename> modules for examples.
+            </note>
+
+            <para>
+                You will notice that all test classes inherit
+                <filename>oeRuntimeTest</filename>, which is found in
+                <filename>meta/lib/oetest.py</filename>.
+                This base class offers some helper attributes, which are
+                described in the following sections:
+            </para>
+
+            <section id='qemu-image-writing-tests-class-methods'>
+                <title>Class Methods</title>
+
+                <para>
+                    Class methods are as follows:
+                    <itemizedlist>
+                        <listitem><para><emphasis><filename>hasPackage(pkg)</filename>:</emphasis>
+                            Returns "True" if <filename>pkg</filename> is in the
+                            installed package list of the image, which is based
+                            on
+                            <filename>${WORKDIR}/installed_pkgs.txt</filename>
+                            that is generated during the
+                            <filename>do.rootfs</filename> task.
+                            </para></listitem>
+                        <listitem><para><emphasis><filename>hasFeature(feature)</filename>:</emphasis>
+                            Returns "True" if the feature is in
+                            <ulink url='&YOCTO_DOCS_REF_URL;#var-IMAGE_FEATURES'><filename>IMAGE_FEATURES</filename></ulink>
+                            or
+                            <ulink url='&YOCTO_DOCS_REF_URL;#var-DISTRO_FEATURES'><filename>DISTRO_FEATURES</filename></ulink>.
+                            </para></listitem>
+                        <listitem><para><emphasis><filename>restartTarget(params)</filename>:</emphasis>
+                            Restarts the QEMU image optionally passing
+                            <filename>params</filename> to the
+                            <filename>runqemu</filename> script's
+                            <filename>qemuparams</filename> list (e.g "-m 1024" for
+                            more memory).</para></listitem>
+                    </itemizedlist>
+                </para>
+            </section>
+
+            <section id='qemu-image-writing-tests-class-attributes'>
+                <title>Class Attributes</title>
+
+                <para>
+                    Class attributes are as follows:
+                    <itemizedlist>
+                        <listitem><para><emphasis><filename>pscmd</filename>:</emphasis>
+                            Equals "ps -ef" if <filename>procps</filename> is
+                            installed in the image.
+                            Otherwise, <filename>pscmd</filename> equals
+                            "ps" (busybox).
+                            </para></listitem>
+                        <listitem><para><emphasis><filename>tc</filename>:</emphasis>
+                            The called text context, which gives access to the
+                            following attributes:
+                            <itemizedlist>
+                                <listitem><para><emphasis><filename>d</filename>:</emphasis>
+                                    The BitBake data store, which allows you to
+                                    use stuff such as
+                                    <filename>oeRuntimeTest.tc.d.getVar("VIRTUAL-RUNTIME_init_manager")</filename>.
+                                    </para></listitem>
+                                <listitem><para><emphasis><filename>testslist</filename> and <filename>testsrequired</filename>:</emphasis>
+                                    Used internally.
+                                    The tests do not need these.
+                                    </para></listitem>
+                                <listitem><para><emphasis><filename>filesdir</filename>:</emphasis>
+                                    The absolute path to
+                                    <filename>meta/lib/oeqa/runtime/files</filename>,
+                                    which contains helper files for tests meant
+                                    for copying on the target such as small
+                                    files written in C for compilation.
+                                    </para></listitem>
+                                <listitem><para><emphasis><filename>qemu</filename>:</emphasis>
+                                    Provides access to the
+                                    <filename>QemuRunner</filename> object,
+                                    which is the class that boots the image.
+                                    The <filename>qemu</filename> attribute
+                                    provides the following useful attributes:
+                                    <itemizedlist>
+                                        <listitem><para><emphasis><filename>ip</filename>:</emphasis>
+                                            The machine's IP address.
+                                            </para></listitem>
+                                        <listitem><para><emphasis><filename>host_ip</filename>:</emphasis>
+                                            The host IP address, which is only
+                                            used by smart tests.
+                                            </para></listitem>
+                                    </itemizedlist></para></listitem>
+                                <listitem><para><emphasis><filename>target</filename>:</emphasis>
+                                    The <filename>SSHControl</filename> object,
+                                    which is used for running the following
+                                    commands on the image:
+                                    <itemizedlist>
+                                        <listitem><para><emphasis><filename>host</filename>:</emphasis>
+                                            Used internally.
+                                            The tests do not use this command.
+                                            </para></listitem>
+                                        <listitem><para><emphasis><filename>timeout</filename>:</emphasis>
+                                            A global timeout for commands run on
+                                            the target for the instance of a
+                                            test.
+                                            The default is 300 seconds.
+                                            </para></listitem>
+                                        <listitem><para><emphasis><filename>run(cmd, timeout=None)</filename>:</emphasis>
+                                            The single, most used method.
+                                            This command is a wrapper for:
+                                            <filename>ssh root@host "cmd"</filename>.
+                                            The command returns a tuple:
+                                            (status, output), which are what
+                                            their names imply - the return code
+                                            of 'cmd' and whatever output
+                                            it produces.
+                                            The optional timeout argument
+                                            represents the number of seconds the
+                                            test should wait for 'cmd' to
+                                            return.
+                                            If the argument is "None", the
+                                            test uses the default instance's
+                                            timeout period, which is 300
+                                            seconds.
+                                            If the argument is "0", the test
+                                            runs until the command returns.
+                                            </para></listitem>
+                                        <listitem><para><emphasis><filename>copy_to(localpath, remotepath)</filename>:</emphasis>
+                                            <filename>scp localpath root@ip:remotepath</filename>.
+                                            </para></listitem>
+                                        <listitem><para><emphasis><filename>copy_from(remotepath, localpath)</filename>:</emphasis>
+                                            <filename>scp root@host:remotepath localpath</filename>.
+                                            </para></listitem>
+                                    </itemizedlist></para></listitem>
+                            </itemizedlist></para></listitem>
+                    </itemizedlist>
+                </para>
+            </section>
+
+            <section id='qemu-image-writing-tests-instance-attributes'>
+                <title>Instance Attributes</title>
+
+                <para>
+                    A single instance attribute exists, which is
+                    <filename>target</filename>.
+                    The <filename>target</filename> instance attribute is
+                    identical to the class attribute of the same name, which
+                    is described in the previous section.
+                    This attribute exists as both an instance and class
+                    attribute so tests can use
+                    <filename>self.target.run(cmd)</filename> in instance
+                    methods instead of
+                    <filename>oeRuntimeTest.tc.target.run(cmd)</filename>.
+                </para>
+            </section>
+        </section>
+    </section>
+
     <section id="platdev-gdb-remotedebug">
         <title>Debugging With the GNU Project Debugger (GDB) Remotely</title>
 
@@ -4323,6 +4915,50 @@
         </section>
     </section>
 
+    <section id="examining-builds-using-toaster">
+        <title>Examining Builds Using the Toaster API</title>
+
+        <para>
+            Toaster is an Application Programming Interface (API) to the
+            OpenEmbedded build system, which uses BitBake.
+            The interface is a Representational State Transfer (REST) API
+            that queries for and returns build information using
+            <filename>GET</filename> and <filename>JSON</filename>.
+        </para>
+
+        <para>
+            Through the API you can do the following:
+            <itemizedlist>
+                <listitem><para>See information about the tasks executed
+                    and reused during the build.</para></listitem>
+                <listitem><para>See what is built (recipes and
+                    packages) and what packages were installed into the final
+                    image.</para></listitem>
+                <listitem><para>See performance-related information such
+                    as build time, CPU usage, and disk I/O.</para></listitem>
+                <listitem><para>Examine error, warning and trace messages
+                    to aid in debugging.</para></listitem>
+            </itemizedlist>
+        </para>
+
+        <para>
+            In summary, the search operation retrieves a set of objects from
+            a data store used to collect build information.
+            The result contains all the data for the objects being returned.
+            You can order the results of the search by key and the search
+            parameters are consistent for all object types.
+        </para>
+
+        <para>
+            For complete information on the API and its search operation
+            URI, parameters, and reposes, see the
+            <ulink url='https://wiki.yoctoproject.org/wiki/REST_API_Contracts'>REST API Contracts</ulink>
+            Wiki page.
+        </para>
+    </section>
+
+<!--  Commenting out the Toaster GUI stuff as it did not make it for 1.5
+
     <section id="examining-builds-using-toaster">
         <title>Examining Builds using Toaster</title>
 
@@ -4462,7 +5098,7 @@
                 execution time, CPU usage and disk I/O per task.
             </para>
 
-<!--
+            <para>
                 The home page of the interface into the database organizes
                 builds into areas:
                 <itemizedlist>
@@ -4475,6 +5111,8 @@
                     <listitem><para>All builds, which appear in row format at
                         the end of the page.</para></listitem>
                 </itemizedlist>
+            </para>
+
             <para>
                 Each entry is linked to more detail on the particular build
                 or recipe.
@@ -4492,7 +5130,7 @@
                 task, and package information along with directory structure,
                 build time, CPU usage, and disk I/O information.
             </para>
--->
+
         </section>
 
         <section id='stopping-toaster'>
@@ -4507,6 +5145,7 @@
             </para>
         </section>
     </section>
+-->
 
     <section id="platdev-oprofile">
         <title>Profiling with OProfile</title>
@@ -4980,15 +5619,15 @@
      BBFILES ?= ""
 
      BBLAYERS ?= " \
-       ##COREBASE##/meta \
-       ##COREBASE##/meta-yocto \
-       ##COREBASE##/meta-yocto-bsp \
-       ##COREBASE##/meta-mylayer \
+       ##OEROOT##/meta \
+       ##OEROOT##/meta-yocto \
+       ##OEROOT##/meta-yocto-bsp \
+       ##OEROOT##/meta-mylayer \
        "
 
      BBLAYERS_NON_REMOVABLE ?= " \
-       ##COREBASE##/meta \
-       ##COREBASE##/meta-yocto \
+       ##OEROOT##/meta \
+       ##OEROOT##/meta-yocto \
        "
                 </literallayout>
                 Creating and providing an archive of the

documentation/dev-manual/dev-manual-newbie.xml

@@ -509,7 +509,9 @@
                 This term refers to the area used by the OpenEmbedded build system for builds.
                 The area is created when you <filename>source</filename> the setup
                 environment script that is found in the Source Directory
-                (i.e. <ulink url='&YOCTO_DOCS_REF_URL;#structure-core-script'><filename>&OE_INIT_FILE;</filename></ulink>).
+                (i.e. <ulink url='&YOCTO_DOCS_REF_URL;#structure-core-script'><filename>&OE_INIT_FILE;</filename></ulink>
+                or
+                <ulink url='&YOCTO_DOCS_REF_URL;#structure-memres-core-script'><filename>oe-init-build-env-memres</filename></ulink>).
                 The <ulink url='&YOCTO_DOCS_REF_URL;#var-TOPDIR'><filename>TOPDIR</filename></ulink>
                 variable points to the Build Directory.</para>
 

documentation/kernel-dev/kernel-dev-advanced.xml

@@ -296,6 +296,20 @@
      $ git commit --allow-empty -m "Create orphan meta branch"
             </literallayout>
         </para>
+
+        <para>
+            If you modify the Metadata in the linux-yocto 
+            <filename>meta</filename> branch, you must not forget to update 
+            the 
+            <ulink url='&YOCTO_DOCS_REF_URL;#var-SRCREV'><filename>SRCREV</filename></ulink>
+            statements in the kernel's recipe.
+            In particular, you need to update the 
+            <filename>SRCREV_meta</filename> variable to match the commit in 
+            the <filename>KMETA</filename> branch you wish to use.
+            Changing the data in these branches and not updating the 
+            <filename>SRCREV</filename> statements to match will cause the 
+            build to fetch an older commit.
+        </para>
     </section>
 </section>
 

documentation/poky.ent

@@ -1,6 +1,6 @@
 <!ENTITY DISTRO "1.5">
 <!ENTITY DISTRO_COMPRESSED "15">
-<!ENTITY DISTRO_NAME "tbd">
+<!ENTITY DISTRO_NAME "dora">
 <!ENTITY YOCTO_DOC_VERSION "1.5">
 <!ENTITY POKYVERSION "10.0.0">
 <!ENTITY POKYVERSION_COMPRESSED "1000">
@@ -26,6 +26,7 @@
 <!ENTITY ECLIPSE_UPDATES_URL "&ECLIPSE_DL_URL;/tm/updates/3.3">
 <!ENTITY ECLIPSE_INDIGO_URL "&ECLIPSE_DL_URL;/releases/indigo">
 <!ENTITY ECLIPSE_JUNO_URL "&ECLIPSE_DL_URL;/releases/juno">
+<!ENTITY ECLIPSE_KEPLER_URL "&ECLIPSE_DL_URL;/releases/kepler">
 <!ENTITY ECLIPSE_INDIGO_CDT_URL "&ECLIPSE_DL_URL;tools/cdt/releases/indigo">
 <!ENTITY YOCTO_DOCS_URL "&YOCTO_HOME_URL;/docs">
 <!ENTITY YOCTO_SOURCES_URL "&YOCTO_HOME_URL;/sources/">

documentation/profile-manual/profile-manual-usage.xml

@@ -3094,7 +3094,7 @@
                 If you already have an LTTng trace on a remote target and
                 would like to view it in Eclipse on the host, you can easily
                 copy it from the target to the host and import it into
-                Eclipse to view it using the LTTng Eclipse plugin already
+                Eclipse to view it using the LTTng Eclipse plug-in already
                 bundled in the Eclipse (Juno SR1 or greater).
             </para>
 
@@ -3172,7 +3172,7 @@
 
             <para>
                 You can access extensive help information on how to use
-                the LTTng plugin to search and analyze captured traces via
+                the LTTng plug-in to search and analyze captured traces via
                 the Eclipse help system:
                 <literallayout class='monospaced'>
      Help | Help Contents | LTTng Plug-in User Guide

documentation/ref-manual/faq.xml

@@ -682,9 +682,11 @@
             <para>
                 Yes - you can easily do this.
                 When you use BitBake to build an image, all the build output
-                goes into the directory created when you source the
+                goes into the directory created when you run the 
+                build environment setup script (i.e.
                 <link linkend='structure-core-script'><filename>&OE_INIT_FILE;</filename></link>
-                setup script.
+                or
+                <link linkend='structure-memres-core-script'><filename>oe-init-build-env-memres</filename></link>).
                 By default, this <ulink url='&YOCTO_DOCS_DEV_URL;#build-directory'>Build Directory</ulink>
                 is named <filename>build</filename> but can be named
                 anything you want.

documentation/ref-manual/migration.xml

@@ -230,6 +230,30 @@
             </para>
         </section>
     </section>
+
+    <section id='1.3-linux-kernel-naming'>
+        <title>Linux Kernel Naming</title>
+
+        <para>
+            The naming scheme for kernel output binaries has been changed to
+            now include
+            <link linkend='var-PE'><filename>PE</filename></link> as part of the
+            filename:
+            <literallayout class='monospaced'>
+     KERNEL_IMAGE_BASE_NAME ?= "${KERNEL_IMAGETYPE}-${PE}-${PV}-${PR}-${MACHINE}-${DATETIME}"
+            </literallayout>
+        </para>
+
+        <para>
+            Because the <filename>PE</filename> variable is not set by default,
+            these binary files could result with names that include two dash
+            characters.
+            Here is an example:
+            <literallayout class='monospaced'>
+     bzImage--3.10.9+git0+cd502a8814_7144bcc4b8-r0-qemux86-64-20130830085431.bin
+            </literallayout>
+        </para>
+    </section>
 </section>
 
 <section id='moving-to-the-yocto-project-1.4-release'>

documentation/ref-manual/ref-bitbake.xml

@@ -322,10 +322,10 @@
 $ bitbake --help
 Usage: bitbake [options] [package ...]
 
-Executes the specified task (default is 'build') for a given set of BitBake files.
-It expects that BBFILES is defined, which is a space separated list of files to
-be executed.  BBFILES does support wildcards.
-Default BBFILES are the .bb files in the current directory.
+    Executes the specified task (default is 'build') for a given set of BitBake files.
+    It expects that BBFILES is defined, which is a space separated list of files to
+    be executed.  BBFILES does support wildcards.
+    Default BBFILES are the .bb files in the current directory.
 
 Options:
   --version             show program's version number and exit
@@ -348,6 +348,10 @@ Options:
                         what you are doing). Depending on the base.bbclass a
                         listtasks tasks is defined and will show available
                         tasks
+  -C INVALIDATE_STAMP, --clear-stamp=INVALIDATE_STAMP
+                        Invalidate the stamp for the specified cmd such as
+                        'compile' and run the default task for the specified
+                        target(s)
   -r PREFILE, --read=PREFILE
                         read the specified file before bitbake.conf
   -R POSTFILE, --postread=POSTFILE
@@ -360,11 +364,12 @@ Options:
                         don't execute, just dump out the signature
                         construction information
   -p, --parse-only      quit after parsing the BB files (developers only)
-  -s, --show-versions   show current and preferred versions of all packages
+  -s, --show-versions   show current and preferred versions of all recipes
   -e, --environment     show the global or per-package environment (this is
                         what used to be bbread)
   -g, --graphviz        emit the dependency trees of the specified packages in
-                        the dot syntax
+                        the dot syntax, and the pn-buildlist to show the build
+                        list
   -I EXTRA_ASSUME_PROVIDED, --ignore-deps=EXTRA_ASSUME_PROVIDED
                         Assume these dependencies don't exist and are already
                         provided (equivalent to ASSUME_PROVIDED). Useful to
@@ -374,9 +379,17 @@ Options:
   -P, --profile         profile the command and print a report
   -u UI, --ui=UI        userinterface to use
   -t SERVERTYPE, --servertype=SERVERTYPE
-                        Choose which server to use, none, process or xmlrpc
+                        Choose which server to use, process or xmlrpc
   --revisions-changed   Set the exit code depending on whether upstream
                         floating revisions have changed or not
+  --server-only         Run bitbake without UI,  the frontend can connect with
+                        bitbake server itself
+  -B BIND, --bind=BIND  The name/address for the bitbake server to bind to
+  --no-setscene         Do not run any setscene tasks, forces builds
+  --remote-server=REMOTE_SERVER
+                        Connect to the specified server
+  -m, --kill-server     Terminate the remote server
+  --observe-only        Connect to a server as an observing-only client
         </screen>
     </section>
 

documentation/ref-manual/ref-classes.xml

@@ -935,6 +935,33 @@
     </para>
 </section>
 
+<section id='ref-classes-testimage'>
+    <title><filename>testimage.bbclass</filename></title>
+
+    <para>
+        You can use this class to enable running a series of automated tests
+        for QEMU images.
+        The class handles loading the tests and starting the QEMU image.
+    </para>
+
+    <para>
+        To use the class, you need to perform steps to set up the
+        environment.
+        The tests are commands that run on the target system over
+        <filename>ssh</filename>.
+        they are written in Python and make use of the
+        <filename>unittest modulebuild</filename> software.
+    </para>
+
+    <para>
+        For information on how to enable, run, and create new tests, see the
+        "<ulink url='running-and-writing-tests-for-a-qemu-image'>Running and Writing Tests for a QEMU Image</ulink>"
+        section and the
+        <ulink url='https://wiki.yoctoproject.org/wiki/Image_tests'>Image Tests</ulink>
+        Wiki page.
+    </para>
+</section>
+
 <section id='ref-classes-others'>
     <title>Other Classes</title>
 

documentation/ref-manual/ref-features.xml

@@ -194,7 +194,7 @@
                 <listitem><para><emphasis>doc-pkgs:</emphasis> Installs documentation packages for all packages
                     installed in a given image.</para></listitem>
                 <listitem><para><emphasis>nfs-server:</emphasis> Installs an NFS server.</para></listitem>
-                <listitem><para><emphasis>read-only-fsroot:</emphasis> Creates
+                <listitem><para><emphasis>read-only-rootfs:</emphasis> Creates
                     an image whose root filesystem is read-only.
                     See the
                     "<ulink url='&YOCTO_DOCS_DEV_URL;#creating-a-read-only-root-filesystem'>Creating a Read-Only Root Filesystem</ulink>"

documentation/ref-manual/ref-manual.xml

@@ -99,6 +99,8 @@
 
     <xi:include href="usingpoky.xml"/>
 
+    <xi:include href="closer-look.xml"/>
+
     <xi:include href="technical-details.xml"/>
 
     <xi:include href="migration.xml"/>

documentation/ref-manual/ref-structure.xml

@@ -74,7 +74,11 @@
             the source tree is combined with the output.
             The <ulink url='&YOCTO_DOCS_DEV_URL;#build-directory'>Build Directory</ulink>
             is created initially when you <filename>source</filename>
-            the OpenEmbedded build environment setup script <filename>&OE_INIT_FILE;</filename>.
+            the OpenEmbedded build environment setup script 
+            (i.e.
+            <link linkend='structure-core-script'><filename>&OE_INIT_FILE;</filename></link>
+            or
+            <link linkend='structure-memres-core-script'><filename>oe-init-build-env-memres</filename></link>).
         </para>
 
         <para>
@@ -85,8 +89,10 @@
             the setup script.
             For information on separating output from your local
             Source Directory files, see the
-            "<link linkend='structure-core-script'><filename>&OE_INIT_FILE;</filename></link>"
-            section.
+            "<link linkend='structure-core-script'><filename>&OE_INIT_FILE;</filename></link>
+            and
+            "<link linkend='structure-memres-core-script'><filename>oe-init-build-env-memres</filename></link>"
+            sections.
         </para>
     </section>
 
@@ -143,7 +149,7 @@
             which is a Yocto Project build user interface.
             For more information on the Hob, see the
             <ulink url='&YOCTO_HOME_URL;/tools-resources/projects/hob'>Hob Project</ulink>
-            webpage.
+            web page.
         </para>
     </section>
 
@@ -162,7 +168,9 @@
             This directory contains various integration scripts that implement
             extra functionality in the Yocto Project environment (e.g. QEMU scripts).
             The <link linkend="structure-core-script"><filename>&OE_INIT_FILE;</filename></link>
-            script appends this directory to the shell's
+            and
+            <link linkend='structure-memres-core-script'><filename>oe-init-build-env-memres</filename></link>
+            scripts append this directory to the shell's
             <filename>PATH</filename> environment variable.
         </para>
 
@@ -177,11 +185,19 @@
         <title><filename>&OE_INIT_FILE;</filename></title>
 
         <para>
-            This script sets up the OpenEmbedded build environment.
+            This script is one of two scripts that set up the OpenEmbedded build 
+            environment.
+            For information on the other script, see the 
+            "<link linkend='structure-memres-core-script'><filename>oe-init-build-env-memres</filename></link>"
+            section.
+        </para>
+
+        <para>
             Running this script with the <filename>source</filename> command in
             a shell makes changes to <filename>PATH</filename> and sets other
             core BitBake variables based on the current working directory.
-            You need to run this script before running BitBake commands.
+            You need to run an environment setup script before running BitBake 
+            commands.
             The script uses other scripts within the
             <filename>scripts</filename> directory to do the bulk of the work.
         </para>
@@ -191,7 +207,7 @@
             <ulink url='&YOCTO_DOCS_DEV_URL;#build-directory'>Build Directory</ulink>
             argument creates the <filename>build</filename> directory.
             If you provide a Build Directory argument when you
-            <filename>source</filename> the script, you direct OpenEmbedded
+            <filename>source</filename> the script, you direct the OpenEmbedded
             build system to create a Build Directory of your choice.
             For example, the following command creates a Build Directory named
             <filename>mybuilds</filename> that is outside of the
@@ -211,6 +227,81 @@
         </para>
     </section>
 
+    <section id='structure-memres-core-script'>
+        <title><filename>oe-init-build-env-memres</filename></title>
+
+        <para>
+            This script is one of two scripts that set up the OpenEmbedded build 
+            environment.
+            Setting up the environment with this script uses a 
+            memory-resident BitBake.
+            For information on the other setup script, see the 
+            "<link linkend='structure-core-script'><filename>&OE_INIT_FILE;</filename></link>"
+            section.
+        </para>
+
+        <para>
+            Memory-resident BitBake resides in memory until you specifically
+            remove it using the following BitBake command:
+            <literallayout class='monospaced'>
+     $ bitbake -m
+            </literallayout>
+        </para>
+
+        <para>
+            Running this script with the <filename>source</filename> command in
+            a shell makes changes to <filename>PATH</filename> and sets other
+            core BitBake variables based on the current working directory.
+            One of these variables is the 
+            <link linkend='var-BBSERVER'><filename>BBSERVER</filename></link>
+            variable, which allows the OpenEmbedded build system to locate 
+            the server that is running BitBake.
+        </para>
+
+        <para>
+            You need to run an environment setup script before running BitBake 
+            commands.
+            Following is the script syntax:
+            <literallayout class='monospaced'>
+     $ source oe-init-build-env-memres &lt;port_number&gt; &lt;build_dir&gt;
+            </literallayout>
+            The script uses other scripts within the
+            <filename>scripts</filename> directory to do the bulk of the work.
+        </para>
+
+        <para>
+            If you do not provide a port number with the script, the default
+            port "12345" is used.
+        </para>
+
+        <para>
+            By default, running this script without a
+            <ulink url='&YOCTO_DOCS_DEV_URL;#build-directory'>Build Directory</ulink>
+            argument creates the <filename>build</filename> directory.
+            If you provide a Build Directory argument when you
+            <filename>source</filename> the script, you direct the OpenEmbedded
+            build system to create a Build Directory of your choice.
+            For example, the following command uses the default port number
+            "12345" and creates a Build Directory named
+            <filename>mybuilds</filename> that is outside of the
+            <ulink url='&YOCTO_DOCS_DEV_URL;#source-directory'>Source Directory</ulink>:
+            <literallayout class='monospaced'>
+     $ source oe-init-build-env-memres ~/mybuilds
+            </literallayout>
+            <note>
+                The OpenEmbedded build system does not support file or 
+                directory names that contain spaces.
+                If you attempt to run the 
+                <filename>oe-init-build-env-memres</filename> script
+                from a Source Directory that contains spaces in either the 
+                filenames or directory names, the script returns an error 
+                indicating no such file or directory.
+                Be sure to use a Source Directory free of names containing 
+                spaces.
+            </note>
+        </para>
+    </section>
+
     <section id='structure-basic-top-level'>
         <title><filename>LICENSE, README, and README.hardware</filename></title>
 
@@ -243,47 +334,123 @@
         <title><filename>build/conf/local.conf</filename></title>
 
         <para>
-            This file contains all the local user configuration for your build environment.
-            If there is no <filename>local.conf</filename> present, it is created from
-            <filename>local.conf.sample</filename>.
-            The <filename>local.conf</filename> file contains documentation on the various configuration options.
-            Any variable set here overrides any variable set elsewhere within the environment unless
-            that variable is hard-coded within a file (e.g. by using '=' instead of '?=').
-            Some variables are hard-coded for various reasons but these variables are
-            relatively rare.
+            This configuration file contains all the local user configurations 
+            for your build environment.
+            The <filename>local.conf</filename> file contains documentation on 
+            the various configuration options.
+            Any variable set here overrides any variable set elsewhere within 
+            the environment unless that variable is hard-coded within a file 
+            (e.g. by using '=' instead of '?=').
+            Some variables are hard-coded for various reasons but these 
+            variables are relatively rare.
         </para>
 
         <para>
-            Edit this file to set the <filename><link linkend='var-MACHINE'>MACHINE</link></filename>
+            Edit this file to set the 
+            <filename><link linkend='var-MACHINE'>MACHINE</link></filename>
             for which you want to build, which package types you wish to use
             (<link linkend='var-PACKAGE_CLASSES'><filename>PACKAGE_CLASSES</filename></link>),
             the location from which you want to downloaded files
             (<filename><link linkend='var-DL_DIR'>DL_DIR</link></filename>),
             and how you want your host machine to use resources
-            (<link linkend='var-BB_NUMBER_THREADS'><filename>BB_NUMBER_THREADS</filename></link> and
+            (<link linkend='var-BB_NUMBER_THREADS'><filename>BB_NUMBER_THREADS</filename></link> 
+            and
             <link linkend='var-PARALLEL_MAKE'><filename>PARALLEL_MAKE</filename></link>).
         </para>
+
+        <para>
+            If <filename>local.conf</filename> is not present when you 
+            start the build, the OpenEmbedded build system creates it from 
+            <filename>local.conf.sample</filename> when
+            you <filename>source</filename> the top-level build environment 
+            setup script (i.e.
+            <link linkend='structure-core-script'><filename>&OE_INIT_FILE;</filename></link>
+            or 
+            <link linkend='structure-memres-core-script'><filename>oe-init-build-env-memres</filename></link>).
+        </para>
+
+        <para>
+            The source <filename>local.conf.sample</filename> file used 
+            depends on the <filename>$TEMPLATECONF</filename> script variable,
+            which defaults to <filename>/meta-yocto/conf</filename>
+            when you are building from the Yocto Project development 
+            environment and defaults to <filename>/meta/conf</filename> when 
+            you are building from the OpenEmbedded Core environment.
+            Because the script variable points to the source of the 
+            <filename>local.conf.sample</filename> file, this implies that 
+            you can configure your build environment from any layer by setting     
+            the variable in the top-level build environment setup script as 
+            follows:
+            <literallayout class='monospaced'>
+     TEMPLATECONF=&lt;your_layer&gt;/conf
+            </literallayout>
+            Once the build process gets the sample file, it uses 
+            <filename>sed</filename> to substitute final 
+            <filename>${</filename><link linkend='var-OEROOT'><filename>OEROOT</filename></link><filename>}</filename> 
+            values for all <filename>##OEROOT##</filename> values.
+            <note>
+                You can see how the <filename>TEMPLATECONF</filename> variable
+                is used by looking at the       
+                <filename>/scripts/oe-setup-builddir</filename> script in the 
+                <ulink url='&YOCTO_DOCS_DEV_URL;#source-directory'>Source Directory</ulink>.
+                You can find the Yocto Project version of the 
+                <filename>local.conf.sample</filename> file in the 
+                <filename>/meta-yocto/conf</filename> directory.                
+            </note>
+        </para>
     </section>
 
     <section id='structure-build-conf-bblayers.conf'>
         <title><filename>build/conf/bblayers.conf</filename></title>
 
         <para>
-            This file defines
+            This configuration file defines
             <ulink url='&YOCTO_DOCS_DEV_URL;#understanding-and-creating-layers'>layers</ulink>,
             which are directory trees, traversed (or walked) by BitBake.
-            If <filename>bblayers.conf</filename>
-            is not present, it is created from <filename>bblayers.conf.sample</filename> when
-            you <filename>source</filename> the environment setup script.
+            The <filename>bblayers.conf</filename> file uses the
+            <link linkend='var-BBLAYERS'><filename>BBLAYERS</filename></link> 
+            variable to list the layers BitBake tries to find, and uses the
+            <link linkend='var-BBLAYERS_NON_REMOVABLE'><filename>BBLAYERS_NON_REMOVABLE</filename></link>
+            variable to list layers that must not be removed.
         </para>
 
         <para>
-            The <filename>bblayers.conf</filename> file uses the
-            <link linkend='var-BBLAYERS'><filename>BBLAYERS</filename></link> variable to
-            list the layers BitBake tries to find.
-            The file uses the
-            <link linkend='var-BBLAYERS_NON_REMOVABLE'><filename>BBLAYERS_NON_REMOVABLE</filename></link>
-            variable to list layers that must not be removed.
+            If <filename>bblayers.conf</filename> is not present when you 
+            start the build, the OpenEmbedded build system creates it from 
+            <filename>bblayers.conf.sample</filename> when
+            you <filename>source</filename> the top-level build environment 
+            setup script (i.e.
+            <link linkend='structure-core-script'><filename>&OE_INIT_FILE;</filename></link>
+            or 
+            <link linkend='structure-memres-core-script'><filename>oe-init-build-env-memres</filename></link>).
+        </para>
+
+        <para>
+            The source <filename>bblayers.conf.sample</filename> file used 
+            depends on the <filename>$TEMPLATECONF</filename> script variable,
+            which defaults to <filename>/meta-yocto/conf</filename>
+            when you are building from the Yocto Project development 
+            environment and defaults to <filename>/meta/conf</filename> when 
+            you are building from the OpenEmbedded Core environment.
+            Because the script variable points to the source of the 
+            <filename>bblayers.conf.sample</filename> file, this implies that 
+            you can base your build from any layer by setting the variable in
+            the top-level build environment setup script as follows:
+            <literallayout class='monospaced'>
+     TEMPLATECONF=&lt;your_layer&gt;/conf
+            </literallayout>
+            Once the build process gets the sample file, it uses 
+            <filename>sed</filename> to substitute final 
+            <filename>${</filename><link linkend='var-OEROOT'><filename>OEROOT</filename></link><filename>}</filename> 
+            values for all <filename>##OEROOT##</filename> values.
+            <note>
+                You can see how the <filename>TEMPLATECONF</filename> variable
+                <filename>/scripts/oe-setup-builddir</filename> script in the 
+                <ulink url='&YOCTO_DOCS_DEV_URL;#source-directory'>Source Directory</ulink>.
+                You can find the Yocto Project version of the 
+                <filename>bblayers.conf.sample</filename> file in the 
+                <filename>/meta-yocto/conf</filename> directory.                
+            </note>
         </para>
     </section>
 

documentation/ref-manual/usingpoky.xml

@@ -27,8 +27,13 @@
         <title>Build Overview</title>
 
         <para>
-            The first thing you need to do is set up the OpenEmbedded build environment by sourcing
-            the <link linkend='structure-core-script'>environment setup script</link> as follows:
+            The first thing you need to do is set up the OpenEmbedded build 
+            environment by sourcing an environment setup script
+            (i.e.
+            <link linkend='structure-core-script'><filename>&OE_INIT_FILE;</filename></link>
+            or
+            <link linkend='structure-memres-core-script'><filename>oe-init-build-env-memres</filename></link>).
+            Here is an example:
             <literallayout class='monospaced'>
      $ source &OE_INIT_FILE; [&lt;build_dir&gt;]
             </literallayout>

documentation/yocto-project-qs/yocto-project-qs.xml

@@ -11,6 +11,21 @@
             <holder>Linux Foundation</holder>
         </copyright>
 
+        <legalnotice>
+            <para>
+                Permission is granted to copy, distribute and/or modify this document under
+                the terms of the <ulink type="http" url="http://creativecommons.org/licenses/by-sa/2.0/uk/">Creative Commons Attribution-Share Alike 2.0 UK: England &amp; Wales</ulink> as published by Creative Commons.
+            </para>
+            <note>
+                Due to production processes, there could be differences between the Yocto Project
+                documentation bundled in the release tarball and the
+                <ulink url='&YOCTO_DOCS_QS_URL;'>Yocto Project Quick Start</ulink> on
+                the <ulink url='&YOCTO_HOME_URL;'>Yocto Project</ulink> website.
+                For the latest version of this manual, see the manual on the website.
+            </note>
+        </legalnotice>
+
+
         <abstract>
             <imagedata fileref="figures/yocto-project-transp.png"
                         width="6in" depth="1in"
@@ -449,7 +464,12 @@
                  to the Build Directory.
                  Later, when the build completes, the Build Directory contains all the files
                  created during the build.
-                 </para></listitem>
+                 <note>
+                     For information on running a memory-resident BitBake, see
+                     the
+                     <ulink url='&YOCTO_DOCS_REF_URL;#structure-memres-core-script'><filename>oe-init-build-env-memres</filename></ulink>
+                     setup script.
+                 </note></para></listitem>
          </itemizedlist>
          <para>
              Take some time to examine your <filename>local.conf</filename> file
@@ -576,11 +596,12 @@
             </para>
 
             <literallayout class='monospaced'>
-     poky-eglibc-&lt;<emphasis>host_system</emphasis>&gt;-&lt;<emphasis>image_type</emphasis>&gt;-&lt;<emphasis>arch</emphasis>&gt;.sh
+     poky-eglibc-&lt;<emphasis>host_system</emphasis>&gt;-&lt;<emphasis>image_type</emphasis>&gt;-&lt;<emphasis>arch</emphasis>&gt;-&lt;<emphasis>release_version</emphasis>&gt;.sh
 
      Where:
          &lt;<emphasis>host_system</emphasis>&gt; is a string representing your development system:
-                i686 or x86_64.
+
+                    i686 or x86_64.
 
          &lt;<emphasis>image_type</emphasis>&gt; is a string representing the image you wish to
                 develop a Software Development Toolkit (SDK) for use against.
@@ -591,7 +612,12 @@
 
          &lt;<emphasis>arch</emphasis>&gt; is a string representing the tuned target architecture:
 
-                i586, x86_64, powerpc, mips, armv7a or armv5te
+                    i586, x86_64, powerpc, mips, armv7a or armv5te
+
+         &lt;<emphasis>release_version</emphasis>&gt; is a string representing the release number of the
+                Yocto Project:
+
+                    &DISTRO;, &DISTRO;+snapshot
             </literallayout>
 
             <para>
@@ -599,7 +625,7 @@
                 development host system and a i586-tuned target architecture
                 based off the SDK for <filename>core-image-sato</filename>:
                 <literallayout class='monospaced'>
-     poky-eglibc-x86_64-core-image-sato-i586.sh
+     poky-eglibc-x86_64-core-image-sato-i586-&DISTRO;.sh
                 </literallayout>
             </para>
 
@@ -628,7 +654,7 @@
 
             <para>
                 <literallayout class='monospaced'>
-     $ ~/Downloads/poky-eglibc-x86_64-core-image-sato-i586.sh
+     $ ~/Downloads/poky-eglibc-x86_64-core-image-sato-i586-&DISTRO;.sh
                 </literallayout>
             </para>
 
@@ -884,6 +910,17 @@
             </literallayout>
         </para>
 
+        <para>
+            In the previous example, the
+            <ulink url='&YOCTO_DOCS_REF_URL;#var-BB_GENERATE_MIRROR_TARBALLS'><filename>BB_GENERATE_MIRROR_TARBALLS</filename></ulink>
+            variable causes the OpenEmbedded build system to generate tarballs
+            of the Git repositories and store them in the
+            <ulink url='&YOCTO_DOCS_REF_URL;#var-DL_DIR'><filename>DL_DIR</filename></ulink>
+            directory.
+            Due to performance reasons, generating and storing these tarballs
+            is not the build system's default behavior.
+        </para>
+
         <para>
             You can also use the
             <ulink url='&YOCTO_DOCS_REF_URL;#var-PREMIRRORS'><filename>PREMIRRORS</filename></ulink>

meta-yocto-bsp/conf/machine/beagleboard.conf

@@ -13,7 +13,7 @@ XSERVER ?= "xserver-xorg \
 MACHINE_EXTRA_RRECOMMENDS = " kernel-modules"
 
 # Allow for MMC booting (required by the NAND-less Beagleboard XM)
-EXTRA_IMAGEDEPENDS += "u-boot x-load"
+EXTRA_IMAGEDEPENDS += "u-boot"
 
 # Uncomment the following line to enable the hard floating point abi. Note that
 # this breaks some binary libraries and 3D (neither of which ship with
@@ -29,12 +29,15 @@ EXTRA_IMAGECMD_jffs2 = "-lnp "
 SERIAL_CONSOLE = "115200 ttyO2"
 
 PREFERRED_PROVIDER_virtual/kernel ?= "linux-yocto"
-PREFERRED_VERSION_linux-yocto ?= "3.4%"
+PREFERRED_VERSION_linux-yocto ?= "3.10%"
 
-KERNEL_IMAGETYPE = "uImage"
+KERNEL_IMAGETYPE = "zImage"
+KERNEL_DEVICETREE = "${S}/arch/arm/boot/dts/omap3-beagle.dts ${S}/arch/arm/boot/dts/omap3-beagle-xm.dts"
 
+SPL_BINARY = "MLO"
+UBOOT_SUFFIX = "img"
 UBOOT_MACHINE = "omap3_beagle_config"
 UBOOT_ENTRYPOINT = "0x80008000"
 UBOOT_LOADADDRESS = "0x80008000"
 
-MACHINE_FEATURES = "apm usbgadget usbhost vfat alsa"
+MACHINE_FEATURES = "usbgadget usbhost vfat alsa"

meta-yocto-bsp/conf/machine/genericx86-64.conf

@@ -4,31 +4,4 @@
 #@DESCRIPTION: Machine configuration for generic X86_64 (64-bit) PCs and servers. Supports a moderately wide range of drivers that should boot and be usable on "typical" hardware.
 
 include conf/machine/include/tune-x86_64.inc
-
-MACHINE_FEATURES = "screen keyboard pci usbhost ext2 ext3 x86 wifi acpi alsa efi pcbios"
-
-KERNEL_IMAGETYPE = "bzImage"
-
-PREFERRED_PROVIDER_virtual/kernel ?= "linux-yocto"
-PREFERRED_VERSION_linux-yocto ?= "3.10%"
-PREFERRED_PROVIDER_virtual/xserver ?= "xserver-xorg"
-XSERVER ?= "xserver-xorg \
-           xserver-xorg-extension-glx \
-           xf86-input-mouse \
-           xf86-input-keyboard \
-           xf86-input-evdev \
-           xf86-input-synaptics \
-           xf86-video-fbdev \
-           xf86-video-modesetting \
-           xf86-video-vesa \
-           xf86-video-intel \
-           mesa-driver-i915 \
-           mesa-driver-i965"
-
-MACHINE_EXTRA_RRECOMMENDS = "kernel-modules eee-acpi-scripts linux-firmware v86d"
-
-IMAGE_FSTYPES ?= "ext3 cpio.gz live"
-
-GLIBC_ADDONS = "nptl"
-
-EXTRA_OECONF_append_pn-matchbox-panel-2 = " --with-battery=acpi"
+include conf/machine/include/genericx86-common.inc

meta-yocto-bsp/conf/machine/genericx86.conf

@@ -4,33 +4,4 @@
 #@DESCRIPTION: Machine configuration for generic X86 (32-bit) PCs. Supports a moderately wide range of drivers that should boot and be usable on "typical" hardware.
 
 include conf/machine/include/tune-atom.inc
-
-MACHINE_FEATURES = "screen keyboard pci usbhost ext2 ext3 x86 wifi acpi alsa"
-
-KERNEL_IMAGETYPE = "bzImage"
-
-PREFERRED_PROVIDER_virtual/kernel ?= "linux-yocto"
-PREFERRED_VERSION_linux-yocto ?= "3.10%"
-PREFERRED_PROVIDER_virtual/xserver ?= "xserver-xorg"
-XSERVER ?= "xserver-xorg \
-           xserver-xorg-extension-glx \
-           xf86-input-mouse \
-           xf86-input-keyboard \
-           xf86-input-evdev \
-           xf86-input-synaptics \
-           xf86-video-fbdev \
-           xf86-video-modesetting \
-           xf86-video-vesa \
-           xf86-video-intel \
-           mesa-driver-i915 \
-           mesa-driver-i965"
-
-#MACHINE_EXTRA_RDEPENDS = "rt2860"
-
-MACHINE_EXTRA_RRECOMMENDS = "kernel-modules eee-acpi-scripts linux-firmware v86d"
-
-IMAGE_FSTYPES ?= "ext3 cpio.gz live"
-
-GLIBC_ADDONS = "nptl"
-
-EXTRA_OECONF_append_pn-matchbox-panel-2 = " --with-battery=acpi"
+include conf/machine/include/genericx86-common.inc

meta-yocto-bsp/conf/machine/include/genericx86-common.inc

@@ -0,0 +1,20 @@
+include conf/machine/include/ia32-base.inc
+MACHINE_FEATURES += "wifi efi pcbios"
+
+PREFERRED_PROVIDER_virtual/kernel ?= "linux-yocto"
+PREFERRED_VERSION_linux-yocto ?= "3.10%"
+PREFERRED_PROVIDER_virtual/xserver ?= "xserver-xorg"
+XSERVER ?= "${XSERVER_IA32_BASE} \
+            ${XSERVER_IA32_EXT} \
+            ${XSERVER_IA32_I915} \
+            ${XSERVER_IA32_I965} \
+            ${XSERVER_IA32_FBDEV} \
+            ${XSERVER_IA32_VESA} \
+            ${XSERVER_IA32_MODESETTING} \
+           "
+
+MACHINE_EXTRA_RRECOMMENDS += "linux-firmware v86d"
+
+GLIBC_ADDONS = "nptl"
+
+EXTRA_OECONF_append_pn-matchbox-panel-2 = " --with-battery=acpi"

meta-yocto-bsp/conf/machine/mpc8315e-rdb.conf

@@ -14,7 +14,7 @@ SERIAL_CONSOLE = "115200 ttyS0"
 
 MACHINE_FEATURES = "keyboard pci ext2 ext3 serial"
 
-PREFERRED_VERSION_linux-yocto ?= "3.4%"
+PREFERRED_VERSION_linux-yocto ?= "3.10%"
 PREFERRED_PROVIDER_virtual/kernel = "linux-yocto"
 
 PREFERRED_PROVIDER_virtual/xserver ?= "xserver-xorg"
@@ -22,7 +22,7 @@ XSERVER ?= "xserver-xorg \
             xf86-input-evdev \
             xf86-video-fbdev"
 
-PREFERRED_VERSION_u-boot ?= "v2011.06%"
+PREFERRED_VERSION_u-boot ?= "v2013.07%"
 UBOOT_ENTRYPOINT = "0x00000000"
 
 KERNEL_DEVICETREE = "${S}/arch/powerpc/boot/dts/mpc8315erdb.dts"

meta-yocto-bsp/recipes-kernel/linux/linux-yocto_3.10.bbappend

@@ -7,11 +7,11 @@ KBRANCH_beagleboard = "standard/beagleboard"
 KMACHINE_genericx86 ?= "common-pc"
 KMACHINE_genericx86-64 ?= "common-pc-64"
 
-SRCREV_machine_genericx86 ?= "6c1528b2b78d1ec7e75bb7a9880074ec35aa1aa0"
-SRCREV_machine_genericx86-64 ?= "6c1528b2b78d1ec7e75bb7a9880074ec35aa1aa0"
-SRCREV_machine_routerstationpro ?= "3991d03bd450e9363d5b6e97ede0628ba073db79"
-SRCREV_machine_mpc8315e-rdb ?= "8d8339af518104db03f9c36b82f4372569e55b0a"
-SRCREV_machine_beagleboard ?= "6c1528b2b78d1ec7e75bb7a9880074ec35aa1aa0"
+SRCREV_machine_genericx86 ?= "702040ac7c7ec66a29b4d147665ccdd0ff015577"
+SRCREV_machine_genericx86-64 ?= "702040ac7c7ec66a29b4d147665ccdd0ff015577"
+SRCREV_machine_routerstationpro ?= "d4e6adefaf92a1e7b6539d371ba49b78bd194a84"
+SRCREV_machine_mpc8315e-rdb ?= "12dc46ba4efb80e135fec4dce913eeb87ee671b3"
+SRCREV_machine_beagleboard ?= "702040ac7c7ec66a29b4d147665ccdd0ff015577"
 
 COMPATIBLE_MACHINE_genericx86 = "genericx86"
 COMPATIBLE_MACHINE_genericx86-64 = "genericx86-64"

meta-yocto/conf/bblayers.conf.sample

@@ -6,11 +6,11 @@ BBPATH = "${TOPDIR}"
 BBFILES ?= ""
 
 BBLAYERS ?= " \
-  ##COREBASE##/meta \
-  ##COREBASE##/meta-yocto \
-  ##COREBASE##/meta-yocto-bsp \
+  ##OEROOT##/meta \
+  ##OEROOT##/meta-yocto \
+  ##OEROOT##/meta-yocto-bsp \
   "
 BBLAYERS_NON_REMOVABLE ?= " \
-  ##COREBASE##/meta \
-  ##COREBASE##/meta-yocto \
+  ##OEROOT##/meta \
+  ##OEROOT##/meta-yocto \
   "

meta-yocto/conf/distro/include/distro_alias.inc

@@ -13,7 +13,6 @@ DISTRO_PN_ALIAS_pn-aaina = "Intel"
 DISTRO_PN_ALIAS_pn-abiword-embedded = "Fedora=abiword Ubuntu=abiword"
 DISTRO_PN_ALIAS_pn-adt-installer = "Intel"
 DISTRO_PN_ALIAS_pn-alsa-state = "OE-Core"
-DISTRO_PN_ALIAS_pn-anjuta-remote-run = "OpenedHand"
 DISTRO_PN_ALIAS_pn-augeas = "Ubuntu=libaugeas0 Debian=libaugeas0"
 DISTRO_PN_ALIAS_pn-avahi-ui = "Ubuntu=avahi-discover Debian=avahi-discover"
 DISTRO_PN_ALIAS_pn-babeltrace = "OSPDT"

meta-yocto/conf/distro/include/maintainers.inc

@@ -35,7 +35,6 @@ RECIPE_MAINTAINER_pn-alsa-state = "Cristian Iorga <cristian.iorga@intel.com>"
 RECIPE_MAINTAINER_pn-alsa-tools = "Kai Kang <kai.kang@windriver.com>"
 RECIPE_MAINTAINER_pn-alsa-utils = "Cristian Iorga <cristian.iorga@intel.com>"
 RECIPE_MAINTAINER_pn-alsa-utils-alsaconf = "Cristian Iorga <cristian.iorga@intel.com>"
-RECIPE_MAINTAINER_pn-anjuta-remote-run = "Paul Eggleton <paul.eggleton@linux.intel.com>"
 RECIPE_MAINTAINER_pn-apmd = "Cristian Iorga <cristian.iorga@intel.com>"
 RECIPE_MAINTAINER_pn-apr = "Cristiana Voicu <cristiana.voicu@intel.com>"
 RECIPE_MAINTAINER_pn-apr-util = "Cristiana Voicu <cristiana.voicu@intel.com>"

meta-yocto/conf/distro/poky.conf

@@ -1,6 +1,6 @@
 DISTRO = "poky"
-DISTRO_NAME = "Poky Next (Yocto Project Reference Distro)"
-DISTRO_VERSION = "1.4+snapshot-${DATE}"
+DISTRO_NAME = "Poky (Yocto Project Reference Distro)"
+DISTRO_VERSION = "1.5"
 DISTRO_CODENAME = "next"
 SDK_VENDOR = "-pokysdk"
 SDK_VERSION := "${@'${DISTRO_VERSION}'.replace('snapshot-${DATE}','snapshot')}"
@@ -74,6 +74,7 @@ CONNECTIVITY_CHECK_URIS ?= " \
 SANITY_TESTED_DISTROS ?= " \
             Poky-1.3 \n \
             Poky-1.4 \n \
+            Poky-1.5 \n \
             Ubuntu-12.04 \n \
             Ubuntu-12.10 \n \
             Ubuntu-13.04 \n \
@@ -107,6 +108,6 @@ ERROR_QA = "dev-so debug-deps dev-deps debug-files arch pkgconfig la perms \
             useless-rpaths rpaths staticdev ldflags pkgvarcheck already-stripped \
             compile-host-path dep-cmp installed-vs-shipped install-host-path \
             packages-list perm-config perm-line perm-link pkgv-undefined \
-            pn-overrides split-strip var-undefined"
+            pn-overrides split-strip var-undefined version-going-backwards"
 
 include conf/distro/include/seperatebuilddir.inc

meta-yocto/conf/layer.conf

@@ -11,6 +11,6 @@ BBFILE_PRIORITY_yocto = "5"
 
 # This should only be incremented on significant changes that will
 # cause compatibility issues with other layers
-LAYERVERSION_yocto = "1"
+LAYERVERSION_yocto = "2"
 
 LAYERDEPENDS_yocto = "core"

meta-yocto/conf/local.conf.sample

@@ -173,23 +173,9 @@ USER_CLASSES ?= "buildstats image-mklibs image-prelink"
 #
 # The build system can test booting virtual machine images under qemu (an emulator)
 # after any root filesystems are created and run tests against those images. To
-# enable this uncomment this line
-#IMAGETEST = "qemu"
-#
-# This variable controls which tests are run against virtual images if enabled
-# above. The following would enable bat, boot the test case under the sanity suite
-# and perform toolchain tests
-#TEST_SCEN = "sanity bat sanity:boot toolchain"
-#
-# Because of the QEMU booting slowness issue (see bug #646 and #618), the
-# autobuilder may suffer a timeout issue when running sanity tests. We introduce
-# the variable TEST_SERIALIZE here to reduce the time taken by the sanity tests.
-# It is set to 1 by default, which will boot the image and run cases in the same
-# image without rebooting or killing the machine instance. If it is set to 0, the
-# image will be copied and tested for each case, which will take longer but be
-# more precise.
-#TEST_SERIALIZE = "1"
-
+# enable this uncomment this line. See classes/testimage(-auto).bbclass for
+# further details.
+#TEST_IMAGE = "1"
 #
 # Interactive shell configuration
 #

meta-yocto/conf/local.conf.sample.extended

@@ -233,3 +233,33 @@
 #groupmod -g 1020 developers; \
 #usermod -s /bin/sh tester; \
 #"
+
+# Enabling FORTRAN
+# Note this is not officially supported and is just illustrated here to
+# show an example of how it can be done
+FORTRAN_forcevariable = ",fortran"
+RUNTIMETARGET_append_pn-gcc-runtime = " libquadmath libgfortran"
+export BUILD_FC = "${CCACHE}${BUILD_PREFIX}gfortran ${BUILD_CC_ARCH}"
+export FC = "${CCACHE}${HOST_PREFIX}gfortran ${HOST_CC_ARCH}"
+
+#
+# Kernel image features
+#
+# The INITRAMFS_IMAGE image variable will cause an additional recipe to
+# be built as a dependency to the what ever rootfs recipe you might be
+# using such as core-image-sato.  The initramfs might be needed for
+# the initial boot of of the target system such as to load kernel
+# modules prior to mounting the root file system.
+#
+# INITRAMFS_IMAGE_BUNDLE variable controls if the image recipe
+# specified by the INITRAMFS_IMAGE will be run through an extra pass
+# through the kernel compilation in order to build a single binary
+# which contains both the kernel image and the initramfs.  The
+# combined binary will be deposited into the tmp/deploy directory.
+# NOTE: You can set INITRAMFS_IMAGE in an image recipe, but
+#       INITRAMFS_IMAGE_BUNDLE can only be set in a conf file.
+#
+#INITRAMFS_IMAGE = "core-image-minimal-initramfs"
+#INITRAMFS_IMAGE_BUNDLE = "1"
+
+

meta-yocto/recipes-core/busybox/busybox_1.21.1.bbappend

@@ -1,2 +1,2 @@
-FILESEXTRAPATHS_prepend := "${THISDIR}/${P}:"
+FILESEXTRAPATHS_prepend_poky := "${THISDIR}/${P}:"
 

meta-yocto/recipes-core/psplash/psplash_git.bbappend

@@ -1,2 +1,2 @@
-FILESEXTRAPATHS_prepend := "${THISDIR}/files:"
+FILESEXTRAPATHS_prepend_poky := "${THISDIR}/files:"
 

meta/classes/allarch.bbclass

@@ -2,6 +2,10 @@
 # This class is used for architecture independent recipes/data files (usally scripts)
 #
 
+# Expand STAGING_DIR_HOST since for cross-canadian/native/nativesdk, this will
+# point elsewhere after these changes.
+STAGING_DIR_HOST := "${STAGING_DIR_HOST}"
+
 PACKAGE_ARCH = "all"
 
 python () {

meta/classes/archiver.bbclass

@@ -99,7 +99,7 @@ def get_bb_inc(d):
     licenses = get_licenses(d)
     script_logs = os.path.join(work_dir, 'script-logs/'+ target_sys + '/' + licenses + '/' + pf + '/script-logs')
     bb_inc = os.path.join(script_logs, 'bb_inc')
-    bb.mkdirhier(bb_inc)
+    bb.utils.mkdirhier(bb_inc)
 
     def find_file(dir, file):
         for root, dirs, files in os.walk(dir):
@@ -139,7 +139,7 @@ def get_logs(d):
     script_logs = os.path.join(work_dir, 'script-logs/'+ target_sys + '/' + licenses + '/' + pf + '/script-logs')
 
     try:
-        bb.mkdirhier(os.path.join(script_logs, 'temp'))
+        bb.utils.mkdirhier(os.path.join(script_logs, 'temp'))
         oe.path.copytree(os.path.join(work_dir, 'temp'), os.path.join(script_logs, 'temp'))
     except (IOError, AttributeError):
         pass
@@ -158,7 +158,7 @@ def get_series(d):
     s = d.getVar('S', True)
     dest = os.path.join(work_dir, pf + '-series')
     shutil.rmtree(dest, ignore_errors=True)
-    bb.mkdirhier(dest)
+    bb.utils.mkdirhier(dest)
 
     src_uri = d.getVar('SRC_URI', True).split()
     fetch = bb.fetch2.Fetch(src_uri, d)
@@ -175,7 +175,7 @@ def get_series(d):
             shutil.copy(patch, dest)
         except IOError:
             if os.path.isdir(patch):
-                bb.mkdirhier(os.path.join(dest, patch))
+                bb.utils.mkdirhier(os.path.join(dest, patch))
                 oe.path.copytree(patch, os.path.join(dest, patch))
     return dest
 
@@ -190,11 +190,11 @@ def get_applying_patches(d):
     work_dir = d.getVar('WORKDIR', True)
     dest = os.path.join(work_dir, pf + '-patches')
     shutil.rmtree(dest, ignore_errors=True)
-    bb.mkdirhier(dest)
+    bb.utils.mkdirhier(dest)
 
     patches = src_patches(d)
     for patch in patches:
-        _, _, local, _, _, parm = bb.decodeurl(patch)
+        _, _, local, _, _, parm = bb.fetch.decodeurl(patch)
         if local:
              shutil.copy(local, dest)
     return dest
@@ -357,7 +357,7 @@ def move_tarball_deploy(d, tarball_list):
     work_dir = d.getVar('WORKDIR', True)
     tar_sources = d.getVar('DEPLOY_DIR', True) + '/sources/' + target_sys + '/' + licenses + '/' + pf
     if not os.path.exists(tar_sources):
-        bb.mkdirhier(tar_sources)
+        bb.utils.mkdirhier(tar_sources)
     for source in tarball_list:
         if source:
             if os.path.exists(os.path.join(tar_sources, source)):
@@ -459,7 +459,7 @@ def dumpdata(d):
     licenses = get_licenses(d)
     dumpdir = os.path.join(workdir, 'diffgz-envdata/'+ target_sys + '/' + licenses + '/' + pf )
     if not os.path.exists(dumpdir):
-        bb.mkdirhier(dumpdir)
+        bb.utils.mkdirhier(dumpdir)
 
     dumpfile = os.path.join(dumpdir, bb.data.expand("${P}-${PR}.showdata.dump", d))
 
@@ -499,7 +499,7 @@ def create_diff_gz(d):
     distro = d.getVar('DISTRO',True) or ""
     dest = s + '/' + distro + '/files'
     if not os.path.exists(dest):
-        bb.mkdirhier(dest)
+        bb.utils.mkdirhier(dest)
     for i in os.listdir(os.getcwd()):
         if os.path.isfile(i):
             try:

meta/classes/autotools.bbclass

@@ -108,10 +108,16 @@ CONFIGURESTAMPFILE = "${WORKDIR}/configure.sstate"
 
 autotools_preconfigure() {
 	if [ -n "${CONFIGURESTAMPFILE}" -a -e "${CONFIGURESTAMPFILE}" ]; then
-		if [ "`cat ${CONFIGURESTAMPFILE}`" != "${BB_TASKHASH}" -a "${S}" != "${B}" ]; then
-			echo "Previously configured separate build directory detected, cleaning ${B}"
-			rm -rf ${B}
-			mkdir ${B}
+		if [ "`cat ${CONFIGURESTAMPFILE}`" != "${BB_TASKHASH}" ]; then
+			if [ "${S}" != "${B}" ]; then
+				echo "Previously configured separate build directory detected, cleaning ${B}"
+				rm -rf ${B}
+				mkdir ${B}
+			else
+				# At least remove the .la files since automake won't automatically
+				# regenerate them even if CFLAGS/LDFLAGS are different
+				cd ${S}; find ${S} -name \*.la -delete
+			fi
 		fi
 	fi
 }