bitbake: Hob: check if warnings are related to package build

If warnings come from recipes parsing
and not from package build, 'parent' object
will be None; so don't update the color for it.

[YOCTO #5621]

(Bitbake rev: f9d24f55a5ffa9257e7ba3257be1210687513733)

Signed-off-by: Valentin Popa <valentin.popa@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

bitbake/LICENSE

@@ -0,0 +1,10 @@
+BitBake is licensed under the GNU General Public License version 2.0. See COPYING for further details.
+
+The following external components are distributed with this software:
+
+* The Toaster Simple UI application is based upon the Django project template, the files of which are covered by the BSD license and are copyright (c) Django Software
+Foundation and individual contributors.
+
+* Twitter Bootstrap (including Glyphicons), redistributed under the Apache License 2.0.
+
+* jQuery is redistributed under the MIT license.

bitbake/bin/bitbake

@@ -41,7 +41,7 @@ from bb import ui
 from bb import server
 from bb import cookerdata
 
-__version__ = "1.20.0"
+__version__ = "1.21.0"
 logger = logging.getLogger("BitBake")
 
 # Python multiprocessing requires /dev/shm
@@ -190,6 +190,9 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
         parser.add_option("", "--observe-only", help = "Connect to a server as an observing-only client.",
                    action = "store_true", dest = "observe_only", default = False)
 
+        parser.add_option("", "--status-only", help = "Check the status of the remote bitbake server.",
+                   action = "store_true", dest = "status_only", default = False)
+
         options, targets = parser.parse_args(sys.argv)
 
         # some environmental variables set also configuration options
@@ -205,8 +208,10 @@ def start_server(servermodule, configParams, configuration):
     if configParams.bind:
         (host, port) = configParams.bind.split(':')
         server.initServer((host, int(port)))
+        configuration.interface = [ server.serverImpl.host, server.serverImpl.port ]
     else:
         server.initServer()
+        configuration.interface = []
 
     try:
         configuration.setServerRegIdleCallback(server.getServerIdleCB())
@@ -284,7 +289,9 @@ def main():
 
     # Ensure logging messages get sent to the UI as events
     handler = bb.event.LogHandler()
-    logger.addHandler(handler)
+    if not configParams.status_only:
+        # In status only mode there are no logs and no UI
+        logger.addHandler(handler)
 
     # Clear away any spurious environment variables while we stoke up the cooker
     cleanedvars = bb.utils.clean_environment()
@@ -297,13 +304,31 @@ def main():
         # we start a stub server that is actually a XMLRPClient that connects to a real server
         server = servermodule.BitBakeXMLRPCClient(configParams.observe_only)
         server.saveConnectionDetails(configParams.remote_server)
+        server.saveConnectionConfigParams(configParams)
 
     if not configParams.server_only:
         # Collect the feature set for the UI
         featureset = getattr(ui_module, "featureSet", [])
 
+        if configParams.status_only:
+            try:
+                server_connection = server.establishConnection(featureset)
+            except:
+                sys.exit(1)
+            if not server_connection:
+                sys.exit(1)
+            server_connection.terminate()
+            sys.exit(0)
+
         # Setup a connection to the server (cooker)
         server_connection = server.establishConnection(featureset)
+        if not server_connection:
+            if configParams.kill_server:
+                bb.fatal("Server already killed")
+            configParams.bind = configParams.remote_server
+            start_server(servermodule, configParams, configuration)
+            bb.event.ui_queue = []
+            server_connection = server.establishConnection(featureset)
 
         # Restore the environment in case the UI needs it
         for k in cleanedvars:

bitbake/bin/bitbake-layers

@@ -55,10 +55,16 @@ def main(args):
 
 class Commands(cmd.Cmd):
     def __init__(self):
-        cmd.Cmd.__init__(self)
-        self.bbhandler = bb.tinfoil.Tinfoil()
+        self.bbhandler = None
         self.returncode = 0
-        self.bblayers = (self.bbhandler.config_data.getVar('BBLAYERS', True) or "").split()
+        self.bblayers = []
+        cmd.Cmd.__init__(self)
+
+    def init_bbhandler(self, config_only = False):
+        if not self.bbhandler:
+            self.bbhandler = bb.tinfoil.Tinfoil()
+            self.bblayers = (self.bbhandler.config_data.getVar('BBLAYERS', True) or "").split()
+            self.bbhandler.prepare(config_only)
 
     def default(self, line):
         """Handle unrecognised commands"""
@@ -83,7 +89,7 @@ class Commands(cmd.Cmd):
 
     def do_show_layers(self, args):
         """show current configured layers"""
-        self.bbhandler.prepare(config_only = True)
+        self.init_bbhandler(config_only = True)
         logger.plain("%s  %s  %s" % ("layer".ljust(20), "path".ljust(40), "priority"))
         logger.plain('=' * 74)
         for layerdir in self.bblayers:
@@ -120,7 +126,7 @@ Options:
        recipes with the ones they overlay indented underneath
   -s   only list overlayed recipes where the version is the same
 """
-        self.bbhandler.prepare()
+        self.init_bbhandler()
 
         show_filenames = False
         show_same_ver_only = False
@@ -203,7 +209,7 @@ Options:
   -m   only list where multiple recipes (in the same layer or different
        layers) exist for the same recipe name
 """
-        self.bbhandler.prepare()
+        self.init_bbhandler()
 
         show_filenames = False
         show_multi_provider_only = False
@@ -341,7 +347,7 @@ build results (as the layer priority order has effectively changed).
             logger.error('Directory %s exists and is non-empty, please clear it out first' % outputdir)
             return
 
-        self.bbhandler.prepare()
+        self.init_bbhandler()
         layers = self.bblayers
         if len(arglist) > 2:
             layernames = arglist[:-1]
@@ -497,7 +503,7 @@ usage: show-appends
 
 Recipes are listed with the bbappends that apply to them as subitems.
 """
-        self.bbhandler.prepare()
+        self.init_bbhandler()
         if not self.bbhandler.cooker.collection.appendlist:
             logger.plain('No append files found')
             return
@@ -570,7 +576,7 @@ Options:
 NOTE:
 The .bbappend file can impact the dependency.
 """
-        self.bbhandler.prepare()
+        self.init_bbhandler()
 
         show_filenames = False
         for arg in args.split():

bitbake/bin/bitbake-worker

@@ -81,7 +81,7 @@ def workerlog_write(msg):
         lf.write(msg)
         lf.flush()
 
-def fork_off_task(cfg, data, workerdata, fn, task, taskname, appends, quieterrors=False):
+def fork_off_task(cfg, data, workerdata, fn, task, taskname, appends, taskdepdata, quieterrors=False):
     # We need to setup the environment BEFORE the fork, since
     # a fork() or exec*() activates PSEUDO...
 
@@ -97,7 +97,8 @@ def fork_off_task(cfg, data, workerdata, fn, task, taskname, appends, quieterror
         except TypeError:
              umask = taskdep['umask'][taskname]
 
-    if 'fakeroot' in taskdep and taskname in taskdep['fakeroot']:
+    # We can't use the fakeroot environment in a dry run as it possibly hasn't been built
+    if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not cfg.dry_run:
         envvars = (workerdata["fakerootenv"][fn] or "").split()
         for key, value in (var.split('=') for var in envvars):
             envbackup[key] = os.environ.get(key)
@@ -147,6 +148,7 @@ def fork_off_task(cfg, data, workerdata, fn, task, taskname, appends, quieterror
                 os.umask(umask)
 
             data.setVar("BB_WORKERCONTEXT", "1")
+            data.setVar("BB_TASKDEPDATA", taskdepdata)
             data.setVar("BUILDNAME", workerdata["buildname"])
             data.setVar("DATE", workerdata["date"])
             data.setVar("TIME", workerdata["time"])
@@ -299,10 +301,10 @@ class BitbakeWorker(object):
         sys.exit(0)
 
     def handle_runtask(self, data):
-        fn, task, taskname, quieterrors, appends = pickle.loads(data)
+        fn, task, taskname, quieterrors, appends, taskdepdata = pickle.loads(data)
         workerlog_write("Handling runtask %s %s %s\n" % (task, fn, taskname))
 
-        pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.workerdata, fn, task, taskname, appends, quieterrors)
+        pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.workerdata, fn, task, taskname, appends, taskdepdata, quieterrors)
 
         self.build_pids[pid] = task
         self.build_pipes[pid] = runQueueWorkerPipe(pipein, pipeout)

bitbake/bin/toaster

@@ -0,0 +1,164 @@
+#!/bin/bash
+# (c) 2013 Intel Corp.
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+
+# This script enables toaster event logging and
+# starts bitbake resident server
+# use as:  source toaster [start|stop]
+
+# Helper function to kill a background toaster development server
+
+function webserverKillAll()
+{
+	local pidfile
+	for pidfile in ${BUILDDIR}/.toastermain.pid; do
+		if [ -f ${pidfile} ]; then
+		while kill -0 $(< ${pidfile}) 2>/dev/null; do
+			kill -SIGTERM -$(< ${pidfile}) 2>/dev/null
+			sleep 1;
+		done;
+		rm  ${pidfile}
+		fi
+	done
+}
+
+function webserverStartAll()
+{
+        retval=0
+        python $BBBASEDIR/lib/toaster/manage.py syncdb || retval=1
+        if [ $retval -eq 1 ]; then
+                echo "Failed db sync, stopping system start" 1>&2
+        else
+            python $BBBASEDIR/lib/toaster/manage.py runserver 0.0.0.0:8000 </dev/null >${BUILDDIR}/toaster_web.log 2>&1 & echo $! >${BUILDDIR}/.toastermain.pid
+        fi
+        return $retval
+}
+
+# Helper functions to add a special configuration file
+
+function addtoConfiguration()
+{
+        echo "#Created by toaster start script" > ${BUILDDIR}/conf/$2
+        echo $1 >> ${BUILDDIR}/conf/$2
+}
+
+# define the stop command
+function stop_system()
+{
+    if [ -f ${BUILDDIR}/.toasterui.pid ]; then
+        kill $(< ${BUILDDIR}/.toasterui.pid )
+        rm ${BUILDDIR}/.toasterui.pid
+    fi
+    BBSERVER=localhost:8200 bitbake -m
+    unset BBSERVER
+    webserverKillAll
+    # force stop any misbehaving bitbake server
+    lsof bitbake.lock | awk '{print $2}' | grep "[0-9]\+" | xargs -n1 -r kill
+}
+
+# We make sure we're running in the current shell and in a good environment
+
+if [ -z "$ZSH_NAME" ] && [ `basename \"$0\"` = `basename \"$BASH_SOURCE\"` ]; then
+    echo "Error: This script needs to be sourced. Please run as 'source toaster [start|stop]'" 1>&2;
+    exit 1
+fi
+
+if [ -z "$BUILDDIR" ] || [ -z `which bitbake` ]; then
+    echo "Error: Build environment is not setup or bitbake is not in path." 1>&2;
+    return 2
+fi
+
+BBBASEDIR=`dirname ${BASH_SOURCE}`/..
+
+
+# Verify prerequisites
+
+if ! echo "import django; print (1,4,5) == django.VERSION[0:3]" | python 2>/dev/null | grep True >/dev/null; then
+    echo -e "This program needs Django 1.4.5. Please install with\n\nsudo pip install django==1.4.5"
+    return 2
+fi
+
+
+
+# Determine the action. If specified by arguments, fine, if not, toggle it
+if [ "x$1" == "xstart" ] || [ "x$1" == "xstop" ]; then
+    CMD="$1"
+else
+    if [ -z "$BBSERVER" ]; then
+        CMD="start"
+    else
+        CMD="stop"
+    fi;
+fi
+
+NOTOASTERUI=0
+if [ "x$2" == "xnoui" ]; then
+    NOTOASTERUI=1
+fi
+
+echo "The system will $CMD."
+
+# Make sure it's safe to run by checking bitbake lock
+
+lock=1
+if [ -e $BUILDDIR/bitbake.lock ]; then
+    (flock -n 200 ) 200<$BUILDDIR/bitbake.lock || lock=0
+fi
+
+if [ ${CMD} == "start" ] && ( [ $lock -eq 0 ] || [ -e $BUILDDIR/.toastermain.pid ] ); then
+    echo "Error: bitbake lock state error. System is already on." 2>&1
+    return 3
+elif [ ${CMD} == "stop" ] && ( [ $lock -eq 1 ] || ! [ -e $BUILDDIR/.toastermain.pid ] ) ; then
+    echo "Error: bitbake lock state error. Trying to stop a stopped system ?
+If you think the system is hanged up, you can try to manually stop system with the commands
+
+# BBSERVER=localhost:8200 bitbake -m
+
+and
+
+# webserverKillAll
+" 2>&1
+    return 3
+fi
+
+
+
+
+
+# Execute the commands
+
+case $CMD in
+    start )
+        addtoConfiguration "INHERIT+=\"toaster buildhistory\"" toaster.conf
+        webserverStartAll || return 4
+        unset BBSERVER
+        bitbake --postread conf/toaster.conf --server-only -t xmlrpc -B localhost:8200
+        export BBSERVER=localhost:8200
+        if [ $NOTOASTERUI == 0 ]; then        # we start the TOASTERUI only if not inhibited
+            bitbake --observe-only -u toasterui >${BUILDDIR}/toaster_ui.log 2>&1 & echo $! >${BUILDDIR}/.toasterui.pid
+        fi
+        # stop system on terminal exit
+        trap stop_system SIGHUP
+    ;;
+    stop )
+        stop_system
+        trap '' SIGHUP
+    ;;
+esac
+
+echo "Successful ${CMD}."
+

bitbake/contrib/vim/plugin/newbb.vim

@@ -17,7 +17,7 @@ endif
 fun! <SID>GetUserName()
     let l:user_name = system("git config --get user.name")
     if v:shell_error
-        return "Unknow User"
+        return "Unknown User"
     else
         return substitute(l:user_name, "\n", "", "")
 endfun

bitbake/lib/bb/__init__.py

@@ -21,7 +21,7 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-__version__ = "1.20.0"
+__version__ = "1.21.0"
 
 import sys
 if sys.version_info < (2, 7, 3):
@@ -141,6 +141,3 @@ def deprecate_import(current, modulename, fromlist, renames = None):
 
         setattr(sys.modules[current], newname, newobj)
 
-deprecate_import(__name__, "bb.fetch", ("MalformedUrl", "encodeurl", "decodeurl"))
-deprecate_import(__name__, "bb.utils", ("mkdirhier", "movefile", "copyfile", "which"))
-deprecate_import(__name__, "bb.utils", ["vercmp_string"], ["vercmp"])

bitbake/lib/bb/build.py

@@ -91,6 +91,9 @@ class TaskBase(event.Event):
 
 class TaskStarted(TaskBase):
     """Task execution started"""
+    def __init__(self, t, logfile, taskflags, d):
+        super(TaskStarted, self).__init__(t, logfile, d)
+        self.taskflags = taskflags
 
 class TaskSucceeded(TaskBase):
     """Task execution completed"""
@@ -285,7 +288,7 @@ set -e
         if bb.msg.loggerVerboseLogs:
             script.write("set -x\n")
         if cwd:
-            script.write("cd %s\n" % cwd)
+            script.write("cd '%s'\n" % cwd)
         script.write("%s\n" % func)
         script.write('''
 # cleanup
@@ -422,7 +425,9 @@ def _exec_task(fn, task, d, quieterr):
     localdata.setVar('BB_LOGFILE', logfn)
     localdata.setVar('BB_RUNTASK', task)
 
-    event.fire(TaskStarted(task, logfn, localdata), localdata)
+    flags = localdata.getVarFlags(task)
+
+    event.fire(TaskStarted(task, logfn, flags, localdata), localdata)
     try:
         for func in (prefuncs or '').split():
             exec_func(func, localdata)

bitbake/lib/bb/codeparser.py

@@ -35,7 +35,7 @@ def check_indent(codestr):
 
 class CodeParserCache(MultiProcessCache):
     cache_file_name = "bb_codeparser.dat"
-    CACHE_VERSION = 3
+    CACHE_VERSION = 4
 
     def __init__(self):
         MultiProcessCache.__init__(self)
@@ -64,6 +64,8 @@ class CodeParserCache(MultiProcessCache):
         for h in data[0]:
             data[0][h]["refs"] = self.internSet(data[0][h]["refs"])
             data[0][h]["execs"] = self.internSet(data[0][h]["execs"])
+            for k in data[0][h]["contains"]:
+                data[0][h]["contains"][k] = self.internSet(data[0][h]["contains"][k])
         for h in data[1]:
             data[1][h]["execs"] = self.internSet(data[1][h]["execs"])
         return
@@ -122,7 +124,13 @@ class PythonParser():
         name = self.called_node_name(node.func)
         if name in self.getvars or name in self.containsfuncs:
             if isinstance(node.args[0], ast.Str):
-                self.var_references.add(node.args[0].s)
+                varname = node.args[0].s
+                if name in self.containsfuncs and isinstance(node.args[1], ast.Str):
+                    if varname not in self.contains:
+                        self.contains[varname] = set()
+                    self.contains[varname].add(node.args[1].s)
+                else:                      
+                    self.references.add(node.args[0].s)
             else:
                 self.warn(node.func, node.args[0])
         elif name in self.execfuncs:
@@ -147,11 +155,11 @@ class PythonParser():
                 break
 
     def __init__(self, name, log):
-        self.var_references = set()
         self.var_execs = set()
+        self.contains = {}
         self.execs = set()
         self.references = set()
-        self.log = BufferedLogger('BitBake.Data.%s' % name, logging.DEBUG, log)
+        self.log = BufferedLogger('BitBake.Data.PythonParser', logging.DEBUG, log)
 
         self.unhandled_message = "in call of %s, argument '%s' is not a string literal"
         self.unhandled_message = "while parsing %s, %s" % (name, self.unhandled_message)
@@ -162,14 +170,15 @@ class PythonParser():
         if h in codeparsercache.pythoncache:
             self.references = codeparsercache.pythoncache[h]["refs"]
             self.execs = codeparsercache.pythoncache[h]["execs"]
+            self.contains = codeparsercache.pythoncache[h]["contains"]
             return
 
         if h in codeparsercache.pythoncacheextras:
             self.references = codeparsercache.pythoncacheextras[h]["refs"]
             self.execs = codeparsercache.pythoncacheextras[h]["execs"]
+            self.contains = codeparsercache.pythoncacheextras[h]["contains"]
             return
 
-
         code = compile(check_indent(str(node)), "<string>", "exec",
                        ast.PyCF_ONLY_AST)
 
@@ -177,12 +186,12 @@ class PythonParser():
             if n.__class__.__name__ == "Call":
                 self.visit_Call(n)
 
-        self.references.update(self.var_references)
         self.references.update(self.var_execs)
 
         codeparsercache.pythoncacheextras[h] = {}
         codeparsercache.pythoncacheextras[h]["refs"] = self.references
         codeparsercache.pythoncacheextras[h]["execs"] = self.execs
+        codeparsercache.pythoncacheextras[h]["contains"] = self.contains
 
 class ShellParser():
     def __init__(self, name, log):

bitbake/lib/bb/command.py

@@ -86,6 +86,8 @@ class Command:
 
     def runAsyncCommand(self):
         try:
+            if self.cooker.state == bb.cooker.state.error:
+                return False
             if self.currentAsyncCommand is not None:
                 (command, options) = self.currentAsyncCommand
                 commandmethod = getattr(CommandsAsync, command)
@@ -194,18 +196,11 @@ class CommandsSync:
         """
         command.cooker.disableDataTracking()
 
-    def initCooker(self, command, params):
-        """
-        Init the cooker to initial state with nothing parsed
-        """
-        command.cooker.initialize()
-
-    def resetCooker(self, command, params):
-        """
-        Reset the cooker to its initial state, thus forcing a reparse for
-        any async command that has the needcache property set to True
-        """
-        command.cooker.reset()
+    def setPrePostConfFiles(self, command, params):
+        prefiles = params[0].split()
+        postfiles = params[1].split()
+        command.cooker.configuration.prefile = prefiles
+        command.cooker.configuration.postfile = postfiles
 
     def getCpuCount(self, command, params):
         """
@@ -418,18 +413,6 @@ class CommandsAsync:
             command.finishAsyncCommand()
     compareRevisions.needcache = True
 
-    def parseConfigurationFiles(self, command, params):
-        """
-        Parse the configuration files
-        """
-        prefiles = params[0].split()
-        postfiles = params[1].split()
-        command.cooker.configuration.prefile = prefiles
-        command.cooker.configuration.postfile = postfiles
-        command.cooker.loadConfigurationData()
-        command.finishAsyncCommand()
-    parseConfigurationFiles.needcache = False
-
     def triggerEvent(self, command, params):
         """
         Trigger a certain event
@@ -439,3 +422,12 @@ class CommandsAsync:
         command.currentAsyncCommand = None
     triggerEvent.needcache = False
 
+    def resetCooker(self, command, params):
+        """
+        Reset the cooker to its initial state, thus forcing a reparse for
+        any async command that has the needcache property set to True
+        """
+        command.cooker.reset()
+        command.finishAsyncCommand()
+    resetCooker.needcache = False
+

bitbake/lib/bb/cooker.py

@@ -61,7 +61,7 @@ class CollectionError(bb.BBHandledException):
     """
 
 class state:
-    initial, parsing, running, shutdown, forceshutdown, stopped = range(6)
+    initial, parsing, running, shutdown, forceshutdown, stopped, error = range(7)
 
 
 class SkippedPackage:
@@ -81,7 +81,7 @@ class SkippedPackage:
 
 
 class CookerFeatures(object):
-    _feature_list = [HOB_EXTRA_CACHES, SEND_DEPENDS_TREE] = range(2)
+    _feature_list = [HOB_EXTRA_CACHES, SEND_DEPENDS_TREE, BASEDATASTORE_TRACKING] = range(3)
 
     def __init__(self):
         self._features=set()
@@ -117,7 +117,7 @@ class BBCooker:
 
         self.configuration = configuration
 
-        self.loadConfigurationData()
+        self.initConfigurationData()
 
         # Take a lock so only one copy of bitbake can run against a given build
         # directory at a time
@@ -125,6 +125,14 @@ class BBCooker:
         self.lock = bb.utils.lockfile(lockfile, False, False)
         if not self.lock:
             bb.fatal("Only one copy of bitbake should be run against a build directory")
+        try:
+            self.lock.seek(0)
+            self.lock.truncate()
+            if len(configuration.interface) >= 2:
+                self.lock.write("%s:%s\n" % (configuration.interface[0], configuration.interface[1]));
+            self.lock.flush()
+        except:
+            pass
 
         # TOSTOP must not be set or our children will hang when they output
         fd = sys.stdout.fileno()
@@ -144,9 +152,11 @@ class BBCooker:
     def initConfigurationData(self):
 
         self.state = state.initial
-
         self.caches_array = []
 
+        if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset:
+            self.enableDataTracking()
+
         all_extra_cache_names = []
         # We hardcode all known cache types in a single place, here.
         if CookerFeatures.HOB_EXTRA_CACHES in self.featureset:
@@ -168,16 +178,6 @@ class BBCooker:
         self.databuilder = bb.cookerdata.CookerDataBuilder(self.configuration, False)
         self.data = self.databuilder.data
 
-    def enableDataTracking(self):
-        self.configuration.tracking = True
-        self.data.enableTracking()
-
-    def disableDataTracking(self):
-        self.configuration.tracking = False
-        self.data.disableTracking()
-
-    def loadConfigurationData(self):
-        self.initConfigurationData()
         self.databuilder.parseBaseConfiguration()
         self.data = self.databuilder.data
         self.data_hash = self.databuilder.data_hash
@@ -189,6 +189,17 @@ class BBCooker:
         bb.data.update_data(self.event_data)
         bb.parse.init_parser(self.event_data)
 
+        if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset:
+            self.disableDataTracking()
+
+    def enableDataTracking(self):
+        self.configuration.tracking = True
+        self.data.enableTracking()
+
+    def disableDataTracking(self):
+        self.configuration.tracking = False
+        self.data.disableTracking()
+
     def modifyConfigurationVar(self, var, val, default_file, op):
         if op == "append":
             self.appendConfigurationVar(var, val, default_file)
@@ -202,20 +213,11 @@ class BBCooker:
         #add append var operation to the end of default_file
         default_file = bb.cookerdata.findConfigFile(default_file, self.data)
 
-        with open(default_file, 'r') as f:
-            contents = f.readlines()
-        f.close()
-
-        total = ""
-        for c in contents:
-            total += c
-
-        total += "#added by hob"
+        total = "#added by hob"
         total += "\n%s += \"%s\"\n" % (var, val)
 
-        with open(default_file, 'w') as f:
+        with open(default_file, 'a') as f:
             f.write(total)
-        f.close()
 
         #add to history
         loginfo = {"op":append, "file":default_file, "line":total.count("\n")}
@@ -244,7 +246,6 @@ class BBCooker:
             if topdir in conf_file:
                 with open(conf_file, 'r') as f:
                     contents = f.readlines()
-                f.close()
 
                 lines = self.data.varhistory.get_variable_lines(var, conf_file)
                 for line in lines:
@@ -270,12 +271,8 @@ class BBCooker:
                         for ii in range(begin_line, end_line):
                             contents[ii] = "#" + contents[ii]
 
-                total = ""
-                for c in contents:
-                    total += c
                 with open(conf_file, 'w') as f:
-                    f.write(total)
-                f.close()
+                    f.writelines(contents)
 
         if replaced == False:
             #remove var from history
@@ -284,21 +281,12 @@ class BBCooker:
             #add var to the end of default_file
             default_file = bb.cookerdata.findConfigFile(default_file, self.data)
 
-            with open(default_file, 'r') as f:
-                contents = f.readlines()
-            f.close()
-
-            total = ""
-            for c in contents:
-                total += c
-
             #add the variable on a single line, to be easy to replace the second time
-            total += "\n#added by hob"
+            total = "\n#added by hob"
             total += "\n%s %s \"%s\"\n" % (var, op, val)
 
-            with open(default_file, 'w') as f:
+            with open(default_file, 'a') as f:
                 f.write(total)
-            f.close()
 
             #add to history
             loginfo = {"op":set, "file":default_file, "line":total.count("\n")}
@@ -312,7 +300,6 @@ class BBCooker:
             if topdir in conf_file:
                 with open(conf_file, 'r') as f:
                     contents = f.readlines()
-                f.close()
 
                 lines = self.data.varhistory.get_variable_lines(var, conf_file)
                 for line in lines:
@@ -335,12 +322,8 @@ class BBCooker:
                     #remove var from history
                     self.data.varhistory.del_var_history(var, conf_file, line)
 
-                total = ""
-                for c in contents:
-                    total += c
                 with open(conf_file, 'w') as f:
-                    f.write(total)
-                f.close()
+                    f.writelines(contents)
 
     def createConfigFile(self, name):
         path = os.getcwd()
@@ -348,7 +331,6 @@ class BBCooker:
         open(confpath, 'w').close()
 
     def parseConfiguration(self):
-
         # Set log file verbosity
         verboselogs = bb.utils.to_boolean(self.data.getVar("BB_VERBOSE_LOGS", "0"))
         if verboselogs:
@@ -511,6 +493,7 @@ class BBCooker:
         depend_tree["packages"] = {}
         depend_tree["rdepends-pkg"] = {}
         depend_tree["rrecs-pkg"] = {}
+        depend_tree["layer-priorities"] = self.recipecache.bbfile_config_priorities
 
         for task in xrange(len(rq.rqdata.runq_fnid)):
             taskname = rq.rqdata.runq_task[task]
@@ -522,6 +505,7 @@ class BBCooker:
                 depend_tree["pn"][pn] = {}
                 depend_tree["pn"][pn]["filename"] = fn
                 depend_tree["pn"][pn]["version"] = version
+                depend_tree["pn"][pn]["inherits"] = self.recipecache.inherits.get(fn, None)
 
                 # if we have extra caches, list all attributes they bring in
                 extra_info = []
@@ -726,14 +710,9 @@ class BBCooker:
         logger.info("Task dependencies saved to 'task-depends.dot'")
 
     def show_appends_with_no_recipes( self ):
-        recipes = set(os.path.basename(f)
-                      for f in self.recipecache.pkg_fn.iterkeys())
-        recipes |= set(os.path.basename(f)
-                      for f in self.skiplist.iterkeys())
-        appended_recipes = self.collection.appendlist.iterkeys()
         appends_without_recipes = [self.collection.appendlist[recipe]
-                                   for recipe in appended_recipes
-                                   if recipe not in recipes]
+                                   for recipe in self.collection.appendlist
+                                   if recipe not in self.collection.appliedappendlist]
         if appends_without_recipes:
             appendlines = ('  %s' % append
                            for appends in appends_without_recipes
@@ -1083,7 +1062,6 @@ class BBCooker:
 
         self.buildSetVars()
 
-        self.recipecache = bb.cache.CacheData(self.caches_array)
         infos = bb.cache.Cache.parse(fn, self.collection.get_file_appends(fn), \
                                      self.data,
                                      self.caches_array)
@@ -1130,10 +1108,13 @@ class BBCooker:
 
         def buildFileIdle(server, rq, abort):
 
+            msg = None
             if abort or self.state == state.forceshutdown:
                 rq.finish_runqueue(True)
+                msg = "Forced shutdown"
             elif self.state == state.shutdown:
                 rq.finish_runqueue(False)
+                msg = "Stopped build"
             failures = 0
             try:
                 retval = rq.execute_runqueue()
@@ -1146,7 +1127,7 @@ class BBCooker:
 
             if not retval:
                 bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runq_fnid), buildname, item, failures), self.event_data)
-                self.command.finishAsyncCommand()
+                self.command.finishAsyncCommand(msg)
                 return False
             if retval is True:
                 return True
@@ -1160,10 +1141,13 @@ class BBCooker:
         """
 
         def buildTargetsIdle(server, rq, abort):
+            msg = None
             if abort or self.state == state.forceshutdown:
                 rq.finish_runqueue(True)
+                msg = "Forced shutdown"
             elif self.state == state.shutdown:
                 rq.finish_runqueue(False)
+                msg = "Stopped build"
             failures = 0
             try:
                 retval = rq.execute_runqueue()
@@ -1176,7 +1160,7 @@ class BBCooker:
 
             if not retval:
                 bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runq_fnid), buildname, targets, failures), self.data)
-                self.command.finishAsyncCommand()
+                self.command.finishAsyncCommand(msg)
                 return False
             if retval is True:
                 return True
@@ -1202,7 +1186,10 @@ class BBCooker:
             try:
                 v = self.data.getVar(k, True)
                 if not k.startswith("__") and not isinstance(v, bb.data_smart.DataSmart):
-                    dump[k] = { 'v' : v }
+                    dump[k] = {
+    'v' : v ,
+    'history' : self.data.varhistory.variable(k),
+                    }
                     for d in flaglist:
                         dump[k][d] = self.data.getVarFlag(k, d)
             except Exception as e:
@@ -1295,6 +1282,11 @@ class BBCooker:
         if len(pkgs_to_build) == 0:
             raise NothingToBuild
 
+        ignore = (self.data.getVar("ASSUME_PROVIDED", True) or "").split()
+        for pkg in pkgs_to_build:
+            if pkg in ignore:
+                parselog.warn("Explicit target \"%s\" is in ASSUME_PROVIDED, ignoring" % pkg)
+
         if 'world' in pkgs_to_build:
             self.buildWorldTargetList()
             pkgs_to_build.remove('world')
@@ -1321,6 +1313,7 @@ class BBCooker:
             self.prhost = prserv.serv.auto_start(self.data)
         except prserv.serv.PRServiceConfigError:
             bb.event.fire(CookerExit(), self.event_data)
+            self.state = state.error
         return
 
     def post_serve(self):
@@ -1336,11 +1329,8 @@ class BBCooker:
     def finishcommand(self):
         self.state = state.initial
 
-    def initialize(self):
-        self.initConfigurationData()
-
     def reset(self):
-        self.loadConfigurationData()
+        self.initConfigurationData()
 
 def server_main(cooker, func, *args):
     cooker.pre_serve()
@@ -1377,6 +1367,7 @@ class CookerExit(bb.event.Event):
 class CookerCollectFiles(object):
     def __init__(self, priorities):
         self.appendlist = {}
+        self.appliedappendlist = []
         self.bbfile_config_priorities = priorities
 
     def calc_bbfile_priority( self, filename, matched = None ):
@@ -1493,10 +1484,14 @@ class CookerCollectFiles(object):
         """
         Returns a list of .bbappend files to apply to fn
         """
+        filelist = []
         f = os.path.basename(fn)
-        if f in self.appendlist:
-            return self.appendlist[f]
-        return []
+        for bbappend in self.appendlist:
+            if (bbappend == f) or ('%' in bbappend and bbappend.startswith(f[:bbappend.index('%')])):
+                self.appliedappendlist.append(bbappend)
+                for filename in self.appendlist[bbappend]:
+                    filelist.append(filename)
+        return filelist
 
     def collection_priorities(self, pkgfns):
 

bitbake/lib/bb/cookerdata.py

@@ -127,6 +127,7 @@ class CookerConfiguration(object):
         self.dump_signatures = False
         self.dry_run = False
         self.tracking = False
+        self.interface = []
 
         self.env = {}
 

bitbake/lib/bb/data.py

@@ -214,7 +214,7 @@ def emit_var(var, o=sys.__stdout__, d = init(), all=False):
         o.write('unset %s\n' % varExpanded)
         return 0
 
-    if not val:
+    if val is None:
         return 0
 
     val = str(val)
@@ -229,7 +229,7 @@ def emit_var(var, o=sys.__stdout__, d = init(), all=False):
 
     # if we're going to output this within doublequotes,
     # to a shell, we need to escape the quotes in the var
-    alter = re.sub('"', '\\"', val.strip())
+    alter = re.sub('"', '\\"', val)
     alter = re.sub('\n', ' \\\n', alter)
     o.write('%s="%s"\n' % (varExpanded, alter))
     return 0
@@ -299,6 +299,21 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
         vardeps = varflags.get("vardeps")
         value = d.getVar(key, False)
 
+        def handle_contains(value, contains, d):
+            newvalue = ""
+            for k in contains:
+                l = (d.getVar(k, True) or "").split()
+                for word in contains[k]:
+                    if word in l:
+                        newvalue += "\n%s{%s} = Set" %  (k, word)
+                    else:
+                        newvalue += "\n%s{%s} = Unset" %  (k, word)
+            if not newvalue:
+                return value
+            if not value:
+                return newvalue
+            return value + newvalue
+
         if "vardepvalue" in varflags:
            value = varflags.get("vardepvalue")
         elif varflags.get("func"):
@@ -309,6 +324,7 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
                     logger.warn("Variable %s contains tabs, please remove these (%s)" % (key, d.getVar("FILE", True)))
                 parser.parse_python(parsedvar.value)
                 deps = deps | parser.references
+                value = handle_contains(value, parser.contains, d)
             else:
                 parsedvar = d.expandWithRefs(value, key)
                 parser = bb.codeparser.ShellParser(key, logger)
@@ -318,10 +334,12 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
                 parser.log.flush()
             deps = deps | parsedvar.references
             deps = deps | (keys & parser.execs) | (keys & parsedvar.execs)
+            value = handle_contains(value, parsedvar.contains, d)
         else:
             parser = d.expandWithRefs(value, key)
             deps |= parser.references
             deps = deps | (keys & parser.execs)
+            value = handle_contains(value, parser.contains, d)
 
         # Add varflags, assuming an exclusion list is set
         if varflagsexcl:

bitbake/lib/bb/data_smart.py

@@ -88,6 +88,7 @@ class VariableParse:
 
         self.references = set()
         self.execs = set()
+        self.contains = {}
 
     def var_sub(self, match):
             key = match.group()[2:-1]
@@ -98,7 +99,7 @@ class VariableParse:
                 varparse = self.d.expand_cache[key]
                 var = varparse.value
             else:
-                var = self.d.getVar(key, True)
+                var = self.d.getVarFlag(key, "_content", True)
             self.references.add(key)
             if var is not None:
                 return var
@@ -120,6 +121,11 @@ class VariableParse:
             self.references |= parser.references
             self.execs |= parser.execs
 
+            for k in parser.contains:
+                if k not in self.contains:
+                    self.contains[k] = parser.contains[k]
+                else:
+                    self.contains[k].update(parser.contains[k])
             value = utils.better_eval(codeobj, DataContext(self.d))
             return str(value)
 

bitbake/lib/bb/fetch2/__init__.py

@@ -507,7 +507,7 @@ def fetcher_compare_revisions(d):
 def mirror_from_string(data):
     return [ i.split() for i in (data or "").replace('\\n','\n').split('\n') if i ]
 
-def verify_checksum(u, ud, d):
+def verify_checksum(ud, d):
     """
     verify the MD5 and SHA256 checksum for downloaded src
 
@@ -530,7 +530,7 @@ def verify_checksum(u, ud, d):
             raise NoChecksumError('No checksum specified for %s, please add at least one to the recipe:\n'
                              'SRC_URI[%s] = "%s"\nSRC_URI[%s] = "%s"' %
                              (ud.localpath, ud.md5_name, md5data,
-                              ud.sha256_name, sha256data), u)
+                              ud.sha256_name, sha256data), ud.url)
 
         # Log missing sums so user can more easily add them
         if ud.md5_expected == None:
@@ -568,10 +568,10 @@ def verify_checksum(u, ud, d):
         msg = msg + '\nIf this change is expected (e.g. you have upgraded to a new version without updating the checksums) then you can use these lines within the recipe:\nSRC_URI[%s] = "%s"\nSRC_URI[%s] = "%s"\nOtherwise you should retry the download and/or check with upstream to determine if the file has become corrupted or otherwise unexpectedly modified.\n' % (ud.md5_name, md5data, ud.sha256_name, sha256data)
 
     if len(msg):
-        raise ChecksumError('Checksum mismatch!%s' % msg, u, md5data)
+        raise ChecksumError('Checksum mismatch!%s' % msg, ud.url, md5data)
 
 
-def update_stamp(u, ud, d):
+def update_stamp(ud, d):
     """
         donestamp is file stamp indicating the whole fetching is done
         this function update the stamp after verifying the checksum
@@ -584,7 +584,7 @@ def update_stamp(u, ud, d):
             # Errors aren't fatal here
             pass
     else:
-        verify_checksum(u, ud, d)
+        verify_checksum(ud, d)
         open(ud.donestamp, 'w').close()
 
 def subprocess_setup():
@@ -619,7 +619,7 @@ def get_srcrev(d):
         raise FetchError("SRCREV was used yet no valid SCM was found in SRC_URI")
 
     if len(scms) == 1 and len(urldata[scms[0]].names) == 1:
-        autoinc, rev = urldata[scms[0]].method.sortable_revision(scms[0], urldata[scms[0]], d, urldata[scms[0]].names[0])
+        autoinc, rev = urldata[scms[0]].method.sortable_revision(urldata[scms[0]], d, urldata[scms[0]].names[0])
         if len(rev) > 10:
             rev = rev[:10]
         if autoinc:
@@ -637,7 +637,7 @@ def get_srcrev(d):
     for scm in scms:
         ud = urldata[scm]
         for name in ud.names:
-            autoinc, rev = ud.method.sortable_revision(scm, ud, d, name)
+            autoinc, rev = ud.method.sortable_revision(ud, d, name)
             seenautoinc = seenautoinc or autoinc
             if len(rev) > 10:
                 rev = rev[:10]
@@ -730,7 +730,7 @@ def build_mirroruris(origud, mirrors, ld):
     replacements["BASENAME"] = origud.path.split("/")[-1]
     replacements["MIRRORNAME"] = origud.host.replace(':','.') + origud.path.replace('/', '.').replace('*', '.')
 
-    def adduri(uri, ud, uris, uds):
+    def adduri(ud, uris, uds):
         for line in mirrors:
             try:
                 (find, replace) = line
@@ -753,9 +753,9 @@ def build_mirroruris(origud, mirrors, ld):
             uris.append(newuri)
             uds.append(newud)
 
-            adduri(newuri, newud, uris, uds)
+            adduri(newud, uris, uds)
 
-    adduri(None, origud, uris, uds)
+    adduri(origud, uris, uds)
 
     return uris, uds
 
@@ -772,22 +772,22 @@ def rename_bad_checksum(ud, suffix):
     bb.utils.movefile(ud.localpath, new_localpath)
 
 
-def try_mirror_url(newuri, origud, ud, ld, check = False):
+def try_mirror_url(origud, ud, ld, check = False):
     # Return of None or a value means we're finished
     # False means try another url
     try:
         if check:
-            found = ud.method.checkstatus(newuri, ud, ld)
+            found = ud.method.checkstatus(ud, ld)
             if found:
                 return found
             return False
 
         os.chdir(ld.getVar("DL_DIR", True))
 
-        if not os.path.exists(ud.donestamp) or ud.method.need_update(newuri, ud, ld):
-            ud.method.download(newuri, ud, ld)
+        if not os.path.exists(ud.donestamp) or ud.method.need_update(ud, ld):
+            ud.method.download(ud, ld)
             if hasattr(ud.method,"build_mirror_data"):
-                ud.method.build_mirror_data(newuri, ud, ld)
+                ud.method.build_mirror_data(ud, ld)
 
         if not ud.localpath or not os.path.exists(ud.localpath):
             return False
@@ -805,6 +805,10 @@ def try_mirror_url(newuri, origud, ud, ld, check = False):
             dest = os.path.join(dldir, os.path.basename(ud.localpath))
             if not os.path.exists(dest):
                 os.symlink(ud.localpath, dest)
+            if not os.path.exists(origud.donestamp) or origud.method.need_update(origud, ld):
+                origud.method.download(origud, ld)
+                if hasattr(ud.method,"build_mirror_data"):
+                    origud.method.build_mirror_data(origud, ld)
             return None
         # Otherwise the result is a local file:// and we symlink to it
         if not os.path.exists(origud.localpath):
@@ -813,7 +817,7 @@ def try_mirror_url(newuri, origud, ud, ld, check = False):
                 os.unlink(origud.localpath)
 
             os.symlink(ud.localpath, origud.localpath)
-        update_stamp(newuri, origud, ld)
+        update_stamp(origud, ld)
         return ud.localpath
 
     except bb.fetch2.NetworkAccess:
@@ -821,13 +825,13 @@ def try_mirror_url(newuri, origud, ud, ld, check = False):
 
     except bb.fetch2.BBFetchException as e:
         if isinstance(e, ChecksumError):
-            logger.warn("Mirror checksum failure for url %s (original url: %s)\nCleaning and trying again." % (newuri, origud.url))
+            logger.warn("Mirror checksum failure for url %s (original url: %s)\nCleaning and trying again." % (ud.url, origud.url))
             logger.warn(str(e))
             rename_bad_checksum(ud, e.checksum)
         elif isinstance(e, NoChecksumError):
             raise
         else:
-            logger.debug(1, "Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url))
+            logger.debug(1, "Mirror fetch failure for url %s (original url: %s)" % (ud.url, origud.url))
             logger.debug(1, str(e))
         try:
             ud.method.clean(ud, ld)
@@ -849,7 +853,7 @@ def try_mirrors(d, origud, mirrors, check = False):
     uris, uds = build_mirroruris(origud, mirrors, ld)
 
     for index, uri in enumerate(uris):
-        ret = try_mirror_url(uri, origud, uds[index], ld, check)
+        ret = try_mirror_url(origud, uds[index], ld, check)
         if ret != False:
             return ret
     return None
@@ -884,7 +888,7 @@ def srcrev_internal_helper(ud, d, name):
             var = "SRCREV_%s_pn-%s" % (name, pn)
         raise FetchError("Please set %s to a valid value" % var, ud.url)
     if rev == "AUTOINC":
-        rev = ud.method.latest_revision(ud.url, ud, d, name)
+        rev = ud.method.latest_revision(ud, d, name)
 
     return rev
 
@@ -1005,7 +1009,7 @@ class FetchData(object):
 
         self.method = None
         for m in methods:
-            if m.supports(url, self, d):
+            if m.supports(self, d):
                 self.method = m
                 break                
 
@@ -1027,7 +1031,7 @@ class FetchData(object):
             self.localpath = self.parm["localpath"]
             self.basename = os.path.basename(self.localpath)
         elif self.localfile:
-            self.localpath = self.method.localpath(self.url, self, d)
+            self.localpath = self.method.localpath(self, d)
 
         dldir = d.getVar("DL_DIR", True)
         # Note: .done and .lock files should always be in DL_DIR whereas localpath may not be.
@@ -1051,7 +1055,7 @@ class FetchData(object):
 
     def setup_localpath(self, d):
         if not self.localpath:
-            self.localpath = self.method.localpath(self.url, self, d)
+            self.localpath = self.method.localpath(self, d)
 
     def getSRCDate(self, d):
         """
@@ -1075,13 +1079,13 @@ class FetchMethod(object):
     def __init__(self, urls = []):
         self.urls = []
 
-    def supports(self, url, urldata, d):
+    def supports(self, urldata, d):
         """
         Check to see if this fetch class supports a given url.
         """
         return 0
 
-    def localpath(self, url, urldata, d):
+    def localpath(self, urldata, d):
         """
         Return the local filename of a given url assuming a successful fetch.
         Can also setup variables in urldata for use in go (saving code duplication
@@ -1125,7 +1129,7 @@ class FetchMethod(object):
 
     urls = property(getUrls, setUrls, None, "Urls property")
 
-    def need_update(self, url, ud, d):
+    def need_update(self, ud, d):
         """
         Force a fetch, even if localpath exists?
         """
@@ -1139,7 +1143,7 @@ class FetchMethod(object):
         """
         return False
 
-    def download(self, url, urldata, d):
+    def download(self, urldata, d):
         """
         Fetch urls
         Assumes localpath was called first
@@ -1263,13 +1267,13 @@ class FetchMethod(object):
        """
        bb.utils.remove(urldata.localpath)
 
-    def try_premirror(self, url, urldata, d):
+    def try_premirror(self, urldata, d):
         """
         Should premirrors be used?
         """
         return True
 
-    def checkstatus(self, url, urldata, d):
+    def checkstatus(self, urldata, d):
         """
         Check the status of a URL
         Assumes localpath was called first
@@ -1277,7 +1281,7 @@ class FetchMethod(object):
         logger.info("URL %s could not be checked for status since no method exists.", url)
         return True
 
-    def latest_revision(self, url, ud, d, name):
+    def latest_revision(self, ud, d, name):
         """
         Look in the cache for the latest revision, if not present ask the SCM.
         """
@@ -1285,19 +1289,19 @@ class FetchMethod(object):
             raise ParameterError("The fetcher for this URL does not support _latest_revision", url)
 
         revs = bb.persist_data.persist('BB_URI_HEADREVS', d)
-        key = self.generate_revision_key(url, ud, d, name)
+        key = self.generate_revision_key(ud, d, name)
         try:
             return revs[key]
         except KeyError:
-            revs[key] = rev = self._latest_revision(url, ud, d, name)
+            revs[key] = rev = self._latest_revision(ud, d, name)
             return rev
 
-    def sortable_revision(self, url, ud, d, name):
-        latest_rev = self._build_revision(url, ud, d, name)
+    def sortable_revision(self, ud, d, name):
+        latest_rev = self._build_revision(ud, d, name)
         return True, str(latest_rev)
 
-    def generate_revision_key(self, url, ud, d, name):
-        key = self._revision_key(url, ud, d, name)
+    def generate_revision_key(self, ud, d, name):
+        key = self._revision_key(ud, d, name)
         return "%s-%s" % (key, d.getVar("PN", True) or "")
 
 class Fetch(object):
@@ -1368,9 +1372,9 @@ class Fetch(object):
             try:
                 self.d.setVar("BB_NO_NETWORK", network)
  
-                if os.path.exists(ud.donestamp) and not m.need_update(u, ud, self.d):
+                if os.path.exists(ud.donestamp) and not m.need_update(ud, self.d):
                     localpath = ud.localpath
-                elif m.try_premirror(u, ud, self.d):
+                elif m.try_premirror(ud, self.d):
                     logger.debug(1, "Trying PREMIRRORS")
                     mirrors = mirror_from_string(self.d.getVar('PREMIRRORS', True))
                     localpath = try_mirrors(self.d, ud, mirrors, False)
@@ -1381,16 +1385,16 @@ class Fetch(object):
                 os.chdir(self.d.getVar("DL_DIR", True))
 
                 firsterr = None
-                if not localpath and ((not os.path.exists(ud.donestamp)) or m.need_update(u, ud, self.d)):
+                if not localpath and ((not os.path.exists(ud.donestamp)) or m.need_update(ud, self.d)):
                     try:
                         logger.debug(1, "Trying Upstream")
-                        m.download(u, ud, self.d)
+                        m.download(ud, self.d)
                         if hasattr(m, "build_mirror_data"):
-                            m.build_mirror_data(u, ud, self.d)
+                            m.build_mirror_data(ud, self.d)
                         localpath = ud.localpath
                         # early checksum verify, so that if checksum mismatched,
                         # fetcher still have chance to fetch from mirror
-                        update_stamp(u, ud, self.d)
+                        update_stamp(ud, self.d)
 
                     except bb.fetch2.NetworkAccess:
                         raise
@@ -1417,7 +1421,7 @@ class Fetch(object):
                         logger.error(str(firsterr))
                     raise FetchError("Unable to fetch URL from any source.", u)
 
-                update_stamp(u, ud, self.d)
+                update_stamp(ud, self.d)
 
             except BBFetchException as e:
                 if isinstance(e, NoChecksumError):
@@ -1448,7 +1452,7 @@ class Fetch(object):
             if not ret:
                 # Next try checking from the original uri, u
                 try:
-                    ret = m.checkstatus(u, ud, self.d)
+                    ret = m.checkstatus(ud, self.d)
                 except:
                     # Finally, try checking uri, u, from MIRRORS
                     mirrors = mirror_from_string(self.d.getVar('MIRRORS', True))

bitbake/lib/bb/fetch2/bzr.py

@@ -34,7 +34,7 @@ from bb.fetch2 import runfetchcmd
 from bb.fetch2 import logger
 
 class Bzr(FetchMethod):
-    def supports(self, url, ud, d):
+    def supports(self, ud, d):
         return ud.type in ['bzr']
 
     def urldata_init(self, ud, d):
@@ -48,7 +48,7 @@ class Bzr(FetchMethod):
         ud.setup_revisons(d)
 
         if not ud.revision:
-            ud.revision = self.latest_revision(ud.url, ud, d)
+            ud.revision = self.latest_revision(ud, d)
 
         ud.localfile = data.expand('bzr_%s_%s_%s.tar.gz' % (ud.host, ud.path.replace('/', '.'), ud.revision), d)
 
@@ -81,12 +81,12 @@ class Bzr(FetchMethod):
 
         return bzrcmd
 
-    def download(self, loc, ud, d):
+    def download(self, ud, d):
         """Fetch url"""
 
         if os.access(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir), '.bzr'), os.R_OK):
             bzrcmd = self._buildbzrcommand(ud, d, "update")
-            logger.debug(1, "BZR Update %s", loc)
+            logger.debug(1, "BZR Update %s", ud.url)
             bb.fetch2.check_network_access(d, bzrcmd, ud.url)
             os.chdir(os.path.join (ud.pkgdir, os.path.basename(ud.path)))
             runfetchcmd(bzrcmd, d)
@@ -94,7 +94,7 @@ class Bzr(FetchMethod):
             bb.utils.remove(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir)), True)
             bzrcmd = self._buildbzrcommand(ud, d, "fetch")
             bb.fetch2.check_network_access(d, bzrcmd, ud.url)
-            logger.debug(1, "BZR Checkout %s", loc)
+            logger.debug(1, "BZR Checkout %s", ud.url)
             bb.utils.mkdirhier(ud.pkgdir)
             os.chdir(ud.pkgdir)
             logger.debug(1, "Running %s", bzrcmd)
@@ -114,17 +114,17 @@ class Bzr(FetchMethod):
     def supports_srcrev(self):
         return True
 
-    def _revision_key(self, url, ud, d, name):
+    def _revision_key(self, ud, d, name):
         """
         Return a unique key for the url
         """
         return "bzr:" + ud.pkgdir
 
-    def _latest_revision(self, url, ud, d, name):
+    def _latest_revision(self, ud, d, name):
         """
         Return the latest upstream revision number
         """
-        logger.debug(2, "BZR fetcher hitting network for %s", url)
+        logger.debug(2, "BZR fetcher hitting network for %s", ud.url)
 
         bb.fetch2.check_network_access(d, self._buildbzrcommand(ud, d, "revno"), ud.url)
 
@@ -132,12 +132,12 @@ class Bzr(FetchMethod):
 
         return output.strip()
 
-    def sortable_revision(self, url, ud, d, name):
+    def sortable_revision(self, ud, d, name):
         """
         Return a sortable revision number which in our case is the revision number
         """
 
-        return False, self._build_revision(url, ud, d)
+        return False, self._build_revision(ud, d)
 
-    def _build_revision(self, url, ud, d):
+    def _build_revision(self, ud, d):
         return ud.revision

bitbake/lib/bb/fetch2/cvs.py

@@ -36,7 +36,7 @@ class Cvs(FetchMethod):
     """
     Class to fetch a module or modules from cvs repositories
     """
-    def supports(self, url, ud, d):
+    def supports(self, ud, d):
         """
         Check to see if a given url can be fetched with cvs.
         """
@@ -65,14 +65,14 @@ class Cvs(FetchMethod):
 
         ud.localfile = bb.data.expand('%s_%s_%s_%s%s%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.tag, ud.date, norecurse, fullpath), d)
 
-    def need_update(self, url, ud, d):
+    def need_update(self, ud, d):
         if (ud.date == "now"):
             return True
         if not os.path.exists(ud.localpath):
             return True
         return False
 
-    def download(self, loc, ud, d):
+    def download(self, ud, d):
 
         method = ud.parm.get('method', 'pserver')
         localdir = ud.parm.get('localdir', ud.module)
@@ -124,13 +124,13 @@ class Cvs(FetchMethod):
         pkgdir = os.path.join(d.getVar('CVSDIR', True), pkg)
         moddir = os.path.join(pkgdir, localdir)
         if os.access(os.path.join(moddir, 'CVS'), os.R_OK):
-            logger.info("Update " + loc)
+            logger.info("Update " + ud.url)
             bb.fetch2.check_network_access(d, cvsupdatecmd, ud.url)
             # update sources there
             os.chdir(moddir)
             cmd = cvsupdatecmd
         else:
-            logger.info("Fetch " + loc)
+            logger.info("Fetch " + ud.url)
             # check out sources there
             bb.utils.mkdirhier(pkgdir)
             os.chdir(pkgdir)

bitbake/lib/bb/fetch2/git.py

@@ -73,7 +73,7 @@ class Git(FetchMethod):
     def init(self, d):
         pass
 
-    def supports(self, url, ud, d):
+    def supports(self, ud, d):
         """
         Check to see if a given url can be fetched with git.
         """
@@ -125,7 +125,7 @@ class Git(FetchMethod):
             if not ud.revisions[name] or len(ud.revisions[name]) != 40  or (False in [c in "abcdef0123456789" for c in ud.revisions[name]]):
                 if ud.revisions[name]:
                     ud.branches[name] = ud.revisions[name]
-                ud.revisions[name] = self.latest_revision(ud.url, ud, d, name)
+                ud.revisions[name] = self.latest_revision(ud, d, name)
 
         gitsrcname = '%s%s' % (ud.host.replace(':','.'), ud.path.replace('/', '.').replace('*', '.'))
         # for rebaseable git repo, it is necessary to keep mirror tar ball
@@ -142,21 +142,21 @@ class Git(FetchMethod):
 
         ud.localfile = ud.clonedir
 
-    def localpath(self, url, ud, d):
+    def localpath(self, ud, d):
         return ud.clonedir
 
-    def need_update(self, u, ud, d):
+    def need_update(self, ud, d):
         if not os.path.exists(ud.clonedir):
             return True
         os.chdir(ud.clonedir)
         for name in ud.names:
-            if not self._contains_ref(ud.revisions[name], d):
+            if not self._contains_ref(ud.revisions[name], ud.branches[name], d):
                 return True
         if ud.write_tarballs and not os.path.exists(ud.fullmirror):
             return True
         return False
 
-    def try_premirror(self, u, ud, d):
+    def try_premirror(self, ud, d):
         # If we don't do this, updating an existing checkout with only premirrors
         # is not possible
         if d.getVar("BB_FETCH_PREMIRRORONLY", True) is not None:
@@ -165,7 +165,7 @@ class Git(FetchMethod):
             return False
         return True
 
-    def download(self, loc, ud, d):
+    def download(self, ud, d):
         """Fetch url"""
 
         if ud.user:
@@ -197,7 +197,7 @@ class Git(FetchMethod):
         # Update the checkout if needed
         needupdate = False
         for name in ud.names:
-            if not self._contains_ref(ud.revisions[name], d):
+            if not self._contains_ref(ud.revisions[name], ud.branches[name], d):
                 needupdate = True
         if needupdate:
             try: 
@@ -214,7 +214,7 @@ class Git(FetchMethod):
             runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d)
             ud.repochanged = True
 
-    def build_mirror_data(self, url, ud, d):
+    def build_mirror_data(self, ud, d):
         # Generate a mirror tarball if needed
         if ud.write_tarballs and (ud.repochanged or not os.path.exists(ud.fullmirror)):
             # it's possible that this symlink points to read-only filesystem with PREMIRROR
@@ -281,21 +281,24 @@ class Git(FetchMethod):
     def supports_srcrev(self):
         return True
 
-    def _contains_ref(self, tag, d):
+    def _contains_ref(self, tag, branch, d):
         basecmd = data.getVar("FETCHCMD_git", d, True) or "git"
-        cmd = "%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % (basecmd, tag)
-        output = runfetchcmd(cmd, d, quiet=True)
+        cmd =  "%s branch --contains %s --list %s 2> /dev/null | wc -l" % (basecmd, tag, branch)
+        try:
+            output = runfetchcmd(cmd, d, quiet=True)
+        except bb.fetch2.FetchError:
+            return False
         if len(output.split()) > 1:
             raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output))
         return output.split()[0] != "0"
 
-    def _revision_key(self, url, ud, d, name):
+    def _revision_key(self, ud, d, name):
         """
         Return a unique key for the url
         """
         return "git:" + ud.host + ud.path.replace('/', '.') + ud.branches[name]
 
-    def _latest_revision(self, url, ud, d, name):
+    def _latest_revision(self, ud, d, name):
         """
         Compute the HEAD revision for the url
         """
@@ -311,14 +314,14 @@ class Git(FetchMethod):
             bb.fetch2.check_network_access(d, cmd)
         output = runfetchcmd(cmd, d, True)
         if not output:
-            raise bb.fetch2.FetchError("The command %s gave empty output unexpectedly" % cmd, url)
+            raise bb.fetch2.FetchError("The command %s gave empty output unexpectedly" % cmd, ud.url)
         return output.split()[0]
 
-    def _build_revision(self, url, ud, d, name):
+    def _build_revision(self, ud, d, name):
         return ud.revisions[name]
 
-    def checkstatus(self, uri, ud, d):
-        fetchcmd = "%s ls-remote %s" % (ud.basecmd, uri)
+    def checkstatus(self, ud, d):
+        fetchcmd = "%s ls-remote %s" % (ud.basecmd, ud.url)
         try:
             runfetchcmd(fetchcmd, d, quiet=True)
             return True

bitbake/lib/bb/fetch2/gitsm.py

@@ -27,7 +27,7 @@ from   bb.fetch2 import runfetchcmd
 from   bb.fetch2 import logger
 
 class GitSM(Git):
-    def supports(self, url, ud, d):
+    def supports(self, ud, d):
         """
         Check to see if a given url can be fetched with git.
         """
@@ -42,7 +42,7 @@ class GitSM(Git):
                 pass
         return False
 
-    def update_submodules(self, u, ud, d):
+    def update_submodules(self, ud, d):
         # We have to convert bare -> full repo, do the submodule bit, then convert back
         tmpclonedir = ud.clonedir + ".tmp"
         gitdir = tmpclonedir + os.sep + ".git"
@@ -58,13 +58,13 @@ class GitSM(Git):
         os.rename(gitdir, ud.clonedir,)
         bb.utils.remove(tmpclonedir, True)
 
-    def download(self, loc, ud, d):
-        Git.download(self, loc, ud, d)
+    def download(self, ud, d):
+        Git.download(self, ud, d)
 
         os.chdir(ud.clonedir)
         submodules = self.uses_submodules(ud, d)
         if submodules:
-            self.update_submodules(loc, ud, d)
+            self.update_submodules(ud, d)
 
     def unpack(self, ud, destdir, d):
         Git.unpack(self, ud, destdir, d)

bitbake/lib/bb/fetch2/hg.py

@@ -37,7 +37,7 @@ from bb.fetch2 import logger
 
 class Hg(FetchMethod):
     """Class to fetch from mercurial repositories"""
-    def supports(self, url, ud, d):
+    def supports(self, ud, d):
         """
         Check to see if a given url can be fetched with mercurial.
         """
@@ -62,11 +62,11 @@ class Hg(FetchMethod):
         if 'rev' in ud.parm:
             ud.revision = ud.parm['rev']
         elif not ud.revision:
-            ud.revision = self.latest_revision(ud.url, ud, d)
+            ud.revision = self.latest_revision(ud, d)
 
         ud.localfile = data.expand('%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision), d)
 
-    def need_update(self, url, ud, d):
+    def need_update(self, ud, d):
         revTag = ud.parm.get('rev', 'tip')
         if revTag == "tip":
             return True
@@ -92,7 +92,10 @@ class Hg(FetchMethod):
         if not ud.user:
             hgroot = host + ud.path
         else:
-            hgroot = ud.user + "@" + host + ud.path
+            if ud.pswd:
+                hgroot = ud.user + ":" + ud.pswd + "@" + host + ud.path
+            else:
+                hgroot = ud.user + "@" + host + ud.path
 
         if command == "info":
             return "%s identify -i %s://%s/%s" % (basecmd, proto, hgroot, ud.module)
@@ -112,7 +115,10 @@ class Hg(FetchMethod):
             # do not pass options list; limiting pull to rev causes the local
             # repo not to contain it and immediately following "update" command
             # will crash
-            cmd = "%s pull" % (basecmd)
+            if ud.user and ud.pswd:
+                cmd = "%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" pull" % (basecmd, ud.user, ud.pswd, proto)
+            else:
+                cmd = "%s pull" % (basecmd)
         elif command == "update":
             cmd = "%s update -C %s" % (basecmd, " ".join(options))
         else:
@@ -120,14 +126,14 @@ class Hg(FetchMethod):
 
         return cmd
 
-    def download(self, loc, ud, d):
+    def download(self, ud, d):
         """Fetch url"""
 
         logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'")
 
         if os.access(os.path.join(ud.moddir, '.hg'), os.R_OK):
             updatecmd = self._buildhgcommand(ud, d, "pull")
-            logger.info("Update " + loc)
+            logger.info("Update " + ud.url)
             # update sources there
             os.chdir(ud.moddir)
             logger.debug(1, "Running %s", updatecmd)
@@ -136,7 +142,7 @@ class Hg(FetchMethod):
 
         else:
             fetchcmd = self._buildhgcommand(ud, d, "fetch")
-            logger.info("Fetch " + loc)
+            logger.info("Fetch " + ud.url)
             # check out sources there
             bb.utils.mkdirhier(ud.pkgdir)
             os.chdir(ud.pkgdir)
@@ -163,7 +169,7 @@ class Hg(FetchMethod):
     def supports_srcrev(self):
         return True
 
-    def _latest_revision(self, url, ud, d, name):
+    def _latest_revision(self, ud, d, name):
         """
         Compute tip revision for the url
         """
@@ -171,10 +177,10 @@ class Hg(FetchMethod):
         output = runfetchcmd(self._buildhgcommand(ud, d, "info"), d)
         return output.strip()
 
-    def _build_revision(self, url, ud, d, name):
+    def _build_revision(self, ud, d, name):
         return ud.revision
 
-    def _revision_key(self, url, ud, d, name):
+    def _revision_key(self, ud, d, name):
         """
         Return a unique key for the url
         """

bitbake/lib/bb/fetch2/local.py

@@ -34,7 +34,7 @@ from   bb.fetch2 import FetchMethod, FetchError
 from   bb.fetch2 import logger
 
 class Local(FetchMethod):
-    def supports(self, url, urldata, d):
+    def supports(self, urldata, d):
         """
         Check to see if a given url represents a local fetch.
         """
@@ -47,7 +47,7 @@ class Local(FetchMethod):
         ud.basepath = ud.decodedurl
         return
 
-    def localpath(self, url, urldata, d):
+    def localpath(self, urldata, d):
         """
         Return the local filename of a given url assuming a successful fetch.
         """
@@ -75,14 +75,14 @@ class Local(FetchMethod):
                 return dldirfile
         return newpath
 
-    def need_update(self, url, ud, d):
-        if url.find("*") != -1:
+    def need_update(self, ud, d):
+        if ud.url.find("*") != -1:
             return False
         if os.path.exists(ud.localpath):
             return False
         return True
 
-    def download(self, url, urldata, d):
+    def download(self, urldata, d):
         """Fetch urls (no-op for Local method)"""
         # no need to fetch local files, we'll deal with them in place.
         if self.supports_checksum(urldata) and not os.path.exists(urldata.localpath):
@@ -95,17 +95,17 @@ class Local(FetchMethod):
                 locations.append(filesdir)
             locations.append(d.getVar("DL_DIR", True))
 
-            msg = "Unable to find file " + url + " anywhere. The paths that were searched were:\n    " + "\n    ".join(locations)
+            msg = "Unable to find file " + urldata.url + " anywhere. The paths that were searched were:\n    " + "\n    ".join(locations)
             raise FetchError(msg)
 
         return True
 
-    def checkstatus(self, url, urldata, d):
+    def checkstatus(self, urldata, d):
         """
         Check the status of the url
         """
         if urldata.localpath.find("*") != -1:
-            logger.info("URL %s looks like a glob and was therefore not checked.", url)
+            logger.info("URL %s looks like a glob and was therefore not checked.", urldata.url)
             return True
         if os.path.exists(urldata.localpath):
             return True

bitbake/lib/bb/fetch2/osc.py

@@ -20,7 +20,7 @@ class Osc(FetchMethod):
     """Class to fetch a module or modules from Opensuse build server
        repositories."""
 
-    def supports(self, url, ud, d):
+    def supports(self, ud, d):
         """
         Check to see if a given url can be fetched with osc.
         """
@@ -77,7 +77,7 @@ class Osc(FetchMethod):
 
         return osccmd
 
-    def download(self, loc, ud, d):
+    def download(self, ud, d):
         """
         Fetch url
         """
@@ -86,7 +86,7 @@ class Osc(FetchMethod):
 
         if os.access(os.path.join(data.expand('${OSCDIR}', d), ud.path, ud.module), os.R_OK):
             oscupdatecmd = self._buildosccommand(ud, d, "update")
-            logger.info("Update "+ loc)
+            logger.info("Update "+ ud.url)
             # update sources there
             os.chdir(ud.moddir)
             logger.debug(1, "Running %s", oscupdatecmd)
@@ -94,7 +94,7 @@ class Osc(FetchMethod):
             runfetchcmd(oscupdatecmd, d)
         else:
             oscfetchcmd = self._buildosccommand(ud, d, "fetch")
-            logger.info("Fetch " + loc)
+            logger.info("Fetch " + ud.url)
             # check out sources there
             bb.utils.mkdirhier(ud.pkgdir)
             os.chdir(ud.pkgdir)

bitbake/lib/bb/fetch2/perforce.py

@@ -37,7 +37,7 @@ from   bb.fetch2 import logger
 from   bb.fetch2 import runfetchcmd
 
 class Perforce(FetchMethod):
-    def supports(self, url, ud, d):
+    def supports(self, ud, d):
         return ud.type in ['p4']
 
     def doparse(url, d):
@@ -112,7 +112,7 @@ class Perforce(FetchMethod):
         base = path
         which = path.find('/...')
         if which != -1:
-            base = path[:which]
+            base = path[:which-1]
 
         base = self._strip_leading_slashes(base)
 
@@ -120,12 +120,12 @@ class Perforce(FetchMethod):
 
         ud.localfile = data.expand('%s+%s+%s.tar.gz' % (host, base.replace('/', '.'), cset), d)
 
-    def download(self, loc, ud, d):
+    def download(self, ud, d):
         """
         Fetch urls
         """
 
-        (host, depot, user, pswd, parm) = Perforce.doparse(loc, d)
+        (host, depot, user, pswd, parm) = Perforce.doparse(ud.url, d)
 
         if depot.find('/...') != -1:
             path = depot[:depot.find('/...')]
@@ -158,7 +158,7 @@ class Perforce(FetchMethod):
         tmpfile, errors = bb.process.run(data.getVar('MKTEMPDIRCMD', localdata, True) or "false")
         tmpfile = tmpfile.strip()
         if not tmpfile:
-            raise FetchError("Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.", loc)
+            raise FetchError("Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.", ud.url)
 
         if "label" in parm:
             depot = "%s@%s" % (depot, parm["label"])
@@ -167,13 +167,13 @@ class Perforce(FetchMethod):
             depot = "%s@%s" % (depot, cset)
 
         os.chdir(tmpfile)
-        logger.info("Fetch " + loc)
+        logger.info("Fetch " + ud.url)
         logger.info("%s%s files %s", p4cmd, p4opt, depot)
         p4file, errors = bb.process.run("%s%s files %s" % (p4cmd, p4opt, depot))
         p4file = [f.rstrip() for f in p4file.splitlines()]
 
         if not p4file:
-            raise FetchError("Fetch: unable to get the P4 files from %s" % depot, loc)
+            raise FetchError("Fetch: unable to get the P4 files from %s" % depot, ud.url)
 
         count = 0
 
@@ -191,7 +191,7 @@ class Perforce(FetchMethod):
 
         if count == 0:
             logger.error()
-            raise FetchError("Fetch: No files gathered from the P4 fetch", loc)
+            raise FetchError("Fetch: No files gathered from the P4 fetch", ud.url)
 
         runfetchcmd("tar -czf %s %s" % (ud.localpath, module), d, cleanup = [ud.localpath])
         # cleanup

bitbake/lib/bb/fetch2/repo.py

@@ -31,7 +31,7 @@ from   bb.fetch2 import runfetchcmd
 
 class Repo(FetchMethod):
     """Class to fetch a module or modules from repo (git) repositories"""
-    def supports(self, url, ud, d):
+    def supports(self, ud, d):
         """
         Check to see if a given url can be fetched with repo.
         """
@@ -53,7 +53,7 @@ class Repo(FetchMethod):
 
         ud.localfile = data.expand("repo_%s%s_%s_%s.tar.gz" % (ud.host, ud.path.replace("/", "."), ud.manifest, ud.branch), d)
 
-    def download(self, loc, ud, d):
+    def download(self, ud, d):
         """Fetch url"""
 
         if os.access(os.path.join(data.getVar("DL_DIR", d, True), ud.localfile), os.R_OK):
@@ -91,8 +91,8 @@ class Repo(FetchMethod):
     def supports_srcrev(self):
         return False
 
-    def _build_revision(self, url, ud, d):
+    def _build_revision(self, ud, d):
         return ud.manifest
 
-    def _want_sortable_revision(self, url, ud, d):
+    def _want_sortable_revision(self, ud, d):
         return False

bitbake/lib/bb/fetch2/sftp.py

@@ -72,7 +72,7 @@ from bb.fetch2 import runfetchcmd
 class SFTP(FetchMethod):
     """Class to fetch urls via 'sftp'"""
 
-    def supports(self, url, ud, d):
+    def supports(self, ud, d):
         """
         Check to see if a given url can be fetched with sftp.
         """
@@ -95,10 +95,10 @@ class SFTP(FetchMethod):
 
         ud.localfile = data.expand(urllib.unquote(ud.basename), d)
 
-    def download(self, uri, ud, d):
+    def download(self, ud, d):
         """Fetch urls"""
 
-        urlo = URI(uri)
+        urlo = URI(ud.url)
         basecmd = 'sftp -oPasswordAuthentication=no'
         port = ''
         if urlo.port:
@@ -124,6 +124,6 @@ class SFTP(FetchMethod):
         cmd = '%s %s %s %s' % (basecmd, port, commands.mkarg(remote),
                                commands.mkarg(lpath))
 
-        bb.fetch2.check_network_access(d, cmd, uri)
+        bb.fetch2.check_network_access(d, cmd, ud.url)
         runfetchcmd(cmd, d)
         return True

bitbake/lib/bb/fetch2/ssh.py

@@ -72,8 +72,8 @@ __pattern__ = re.compile(r'''
 class SSH(FetchMethod):
     '''Class to fetch a module or modules via Secure Shell'''
 
-    def supports(self, url, urldata, d):
-        return __pattern__.match(url) != None
+    def supports(self, urldata, d):
+        return __pattern__.match(urldata.url) != None
 
     def supports_checksum(self, urldata):
         return False
@@ -89,10 +89,10 @@ class SSH(FetchMethod):
         host = m.group('host')
         urldata.localpath = os.path.join(d.getVar('DL_DIR', True), os.path.basename(path))
 
-    def download(self, url, urldata, d):
+    def download(self, urldata, d):
         dldir = d.getVar('DL_DIR', True)
 
-        m = __pattern__.match(url)
+        m = __pattern__.match(urldata.url)
         path = m.group('path')
         host = m.group('host')
         port = m.group('port')

bitbake/lib/bb/fetch2/svk.py

@@ -37,7 +37,7 @@ from   bb.fetch2 import runfetchcmd
 
 class Svk(FetchMethod):
     """Class to fetch a module or modules from svk repositories"""
-    def supports(self, url, ud, d):
+    def supports(self, ud, d):
         """
         Check to see if a given url can be fetched with svk.
         """
@@ -54,14 +54,14 @@ class Svk(FetchMethod):
 
         ud.localfile = data.expand('%s_%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision, ud.date), d)
 
-    def need_update(self, url, ud, d):
+    def need_update(self, ud, d):
         if ud.date == "now":
             return True
         if not os.path.exists(ud.localpath):
             return True
         return False
 
-    def download(self, loc, ud, d):
+    def download(self, ud, d):
         """Fetch urls"""
 
         svkroot = ud.host + ud.path
@@ -81,11 +81,11 @@ class Svk(FetchMethod):
         tmpfile = tmpfile.strip()
         if not tmpfile:
             logger.error()
-            raise FetchError("Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.", loc)
+            raise FetchError("Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.", ud.url)
 
         # check out sources there
         os.chdir(tmpfile)
-        logger.info("Fetch " + loc)
+        logger.info("Fetch " + ud.url)
         logger.debug(1, "Running %s", svkcmd)
         runfetchcmd(svkcmd, d, cleanup = [tmpfile])
 

bitbake/lib/bb/fetch2/svn.py

@@ -27,6 +27,7 @@ import os
 import sys
 import logging
 import bb
+import re
 from   bb import data
 from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import FetchError
@@ -36,7 +37,7 @@ from   bb.fetch2 import logger
 
 class Svn(FetchMethod):
     """Class to fetch a module or modules from svn repositories"""
-    def supports(self, url, ud, d):
+    def supports(self, ud, d):
         """
         Check to see if a given url can be fetched with svn.
         """
@@ -91,6 +92,8 @@ class Svn(FetchMethod):
 
         if command == "info":
             svncmd = "%s info %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module)
+        elif command == "log1":
+            svncmd = "%s log --limit 1 %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module)
         else:
             suffix = ""
             if ud.revision:
@@ -109,14 +112,14 @@ class Svn(FetchMethod):
 
         return svncmd
 
-    def download(self, loc, ud, d):
+    def download(self, ud, d):
         """Fetch url"""
 
         logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'")
 
         if os.access(os.path.join(ud.moddir, '.svn'), os.R_OK):
             svnupdatecmd = self._buildsvncommand(ud, d, "update")
-            logger.info("Update " + loc)
+            logger.info("Update " + ud.url)
             # update sources there
             os.chdir(ud.moddir)
             # We need to attempt to run svn upgrade first in case its an older working format
@@ -129,7 +132,7 @@ class Svn(FetchMethod):
             runfetchcmd(svnupdatecmd, d)
         else:
             svnfetchcmd = self._buildsvncommand(ud, d, "fetch")
-            logger.info("Fetch " + loc)
+            logger.info("Fetch " + ud.url)
             # check out sources there
             bb.utils.mkdirhier(ud.pkgdir)
             os.chdir(ud.pkgdir)
@@ -157,33 +160,32 @@ class Svn(FetchMethod):
     def supports_srcrev(self):
         return True
 
-    def _revision_key(self, url, ud, d, name):
+    def _revision_key(self, ud, d, name):
         """
         Return a unique key for the url
         """
         return "svn:" + ud.moddir
 
-    def _latest_revision(self, url, ud, d, name):
+    def _latest_revision(self, ud, d, name):
         """
         Return the latest upstream revision number
         """
-        bb.fetch2.check_network_access(d, self._buildsvncommand(ud, d, "info"))
+        bb.fetch2.check_network_access(d, self._buildsvncommand(ud, d, "log1"))
 
-        output = runfetchcmd("LANG=C LC_ALL=C " + self._buildsvncommand(ud, d, "info"), d, True)
+        output = runfetchcmd("LANG=C LC_ALL=C " + self._buildsvncommand(ud, d, "log1"), d, True)
 
-        revision = None
-        for line in output.splitlines():
-            if "Last Changed Rev" in line:
-                revision = line.split(":")[1].strip()
+        # skip the first line, as per output of svn log
+        # then we expect the revision on the 2nd line
+        revision = re.search('^r([0-9]*)', output.splitlines()[1]).group(1)
 
         return revision
 
-    def sortable_revision(self, url, ud, d, name):
+    def sortable_revision(self, ud, d, name):
         """
         Return a sortable revision number which in our case is the revision number
         """
 
-        return False, self._build_revision(url, ud, d)
+        return False, self._build_revision(ud, d)
 
-    def _build_revision(self, url, ud, d):
+    def _build_revision(self, ud, d):
         return ud.revision

bitbake/lib/bb/fetch2/wget.py

@@ -37,7 +37,7 @@ from   bb.fetch2 import runfetchcmd
 
 class Wget(FetchMethod):
     """Class to fetch urls via 'wget'"""
-    def supports(self, url, ud, d):
+    def supports(self, ud, d):
         """
         Check to see if a given url can be fetched with wget.
         """
@@ -58,7 +58,7 @@ class Wget(FetchMethod):
 
         ud.localfile = data.expand(urllib.unquote(ud.basename), d)
 
-    def download(self, uri, ud, d, checkonly = False):
+    def download(self, ud, d, checkonly = False):
         """Fetch urls"""
 
         basecmd = d.getVar("FETCHCMD_wget", True) or "/usr/bin/env wget -t 2 -T 30 -nv --passive-ftp --no-check-certificate"
@@ -76,7 +76,7 @@ class Wget(FetchMethod):
         else:
             fetchcmd = d.getVar("FETCHCOMMAND_wget", True) or d.expand(basecmd + " -P ${DL_DIR} '${URI}'")
 
-        uri = uri.split(";")[0]
+        uri = ud.url.split(";")[0]
 
         fetchcmd = fetchcmd.replace("${URI}", uri.split(";")[0])
         fetchcmd = fetchcmd.replace("${FILE}", ud.basename)
@@ -93,5 +93,5 @@ class Wget(FetchMethod):
 
         return True
 
-    def checkstatus(self, uri, ud, d):
-        return self.download(uri, ud, d, True)
+    def checkstatus(self, ud, d):
+        return self.download(ud, d, True)

bitbake/lib/bb/monitordisk.py

@@ -243,7 +243,7 @@ class diskMonitor:
                     # zero, this is a feature of the fs, we disable the inode
                     # checking for such a fs.
                     if st.f_files == 0:
-                        logger.warn("Inode check for %s is unavaliable, will remove it from disk monitor" % path)
+                        logger.info("Inode check for %s is unavaliable, will remove it from disk monitor" % path)
                         self.devDict[k][2] = None
                         continue
                     # Always show warning, the self.checked would always be False if the action is WARN

bitbake/lib/bb/parse/__init__.py

@@ -73,9 +73,17 @@ def update_mtime(f):
 def mark_dependency(d, f):
     if f.startswith('./'):
         f = "%s/%s" % (os.getcwd(), f[2:])
-    deps = (d.getVar('__depends') or []) + [(f, cached_mtime(f))]
-    d.setVar('__depends', deps)
+    deps = (d.getVar('__depends') or [])
+    s = (f, cached_mtime_noerror(f))
+    if s not in deps:
+        deps.append(s)
+        d.setVar('__depends', deps)
 
+def check_dependency(d, f):
+    s = (f, cached_mtime_noerror(f))
+    deps = (d.getVar('__depends') or [])
+    return s in deps
+   
 def supports(fn, data):
     """Returns true if we have a handler for this file, false otherwise"""
     for h in handlers:
@@ -102,11 +110,14 @@ def init_parser(d):
 def resolve_file(fn, d):
     if not os.path.isabs(fn):
         bbpath = d.getVar("BBPATH", True)
-        newfn = bb.utils.which(bbpath, fn)
+        newfn, attempts = bb.utils.which(bbpath, fn, history=True)
+        for af in attempts:
+            mark_dependency(d, af)
         if not newfn:
             raise IOError("file %s not found in %s" % (fn, bbpath))
         fn = newfn
 
+    mark_dependency(d, fn)
     if not os.path.isfile(fn):
         raise IOError("file %s not found" % fn)
 

bitbake/lib/bb/parse/parse_py/BBHandler.py

@@ -77,7 +77,10 @@ def inherit(files, fn, lineno, d):
         if not os.path.isabs(file):
             dname = os.path.dirname(fn)
             bbpath = "%s:%s" % (dname, d.getVar("BBPATH", True))
-            abs_fn = bb.utils.which(bbpath, file)
+            abs_fn, attempts = bb.utils.which(bbpath, file, history=True)
+            for af in attempts:
+                if af != abs_fn:
+                    bb.parse.mark_dependency(d, af)
             if abs_fn:
                 file = abs_fn
 

bitbake/lib/bb/parse/parse_py/ConfHandler.py

@@ -82,9 +82,15 @@ def include(oldfn, fn, lineno, data, error_out):
     if not os.path.isabs(fn):
         dname = os.path.dirname(oldfn)
         bbpath = "%s:%s" % (dname, data.getVar("BBPATH", True))
-        abs_fn = bb.utils.which(bbpath, fn)
+        abs_fn, attempts = bb.utils.which(bbpath, fn, history=True)
+        if abs_fn and bb.parse.check_dependency(data, abs_fn):
+            bb.warn("Duplicate inclusion for %s in %s" % (abs_fn, data.getVar('FILE', True)))
+        for af in attempts:
+            bb.parse.mark_dependency(data, af)
         if abs_fn:
             fn = abs_fn
+    elif bb.parse.check_dependency(data, fn):
+        bb.warn("Duplicate inclusion for %s in %s" % (fn, data.getVar('FILE', True)))
 
     from bb.parse import handle
     try:
@@ -93,6 +99,7 @@ def include(oldfn, fn, lineno, data, error_out):
         if error_out:
             raise ParseError("Could not %(error_out)s file %(fn)s" % vars(), oldfn, lineno)
         logger.debug(2, "CONF file '%s' not found", fn)
+        bb.parse.mark_dependency(data, fn)
 
 # We have an issue where a UI might want to enforce particular settings such as
 # an empty DISTRO variable. If configuration files do something like assigning

bitbake/lib/bb/runqueue.py

@@ -98,26 +98,49 @@ class RunQueueScheduler(object):
         """
         self.rq = runqueue
         self.rqdata = rqdata
-        numTasks = len(self.rqdata.runq_fnid)
+        self.numTasks = len(self.rqdata.runq_fnid)
 
         self.prio_map = []
-        self.prio_map.extend(range(numTasks))
+        self.prio_map.extend(range(self.numTasks))
+
+        self.buildable = []
+        self.stamps = {}
+        for taskid in xrange(self.numTasks):
+            fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[taskid]]
+            taskname = self.rqdata.runq_task[taskid]
+            self.stamps[taskid] = bb.build.stampfile(taskname, self.rqdata.dataCache, fn)
+            if self.rq.runq_buildable[taskid] == 1:
+                self.buildable.append(taskid)
+
+        self.rev_prio_map = None
 
     def next_buildable_task(self):
         """
         Return the id of the first task we find that is buildable
         """
-        for tasknum in xrange(len(self.rqdata.runq_fnid)):
-            taskid = self.prio_map[tasknum]
-            if self.rq.runq_running[taskid] == 1:
-                continue
-            if self.rq.runq_buildable[taskid] == 1:
-                fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[taskid]]
-                taskname = self.rqdata.runq_task[taskid]
-                stamp = bb.build.stampfile(taskname, self.rqdata.dataCache, fn)
-                if stamp in self.rq.build_stamps.values():
+        self.buildable = [x for x in self.buildable if not self.rq.runq_running[x] == 1]
+        if not self.buildable:
+            return None
+        if len(self.buildable) == 1:
+            return self.buildable[0]
+
+        if not self.rev_prio_map:
+            self.rev_prio_map = range(self.numTasks)
+            for taskid in xrange(self.numTasks):
+                self.rev_prio_map[self.prio_map[taskid]] = taskid
+
+        best = None
+        bestprio = None
+        for taskid in self.buildable:
+            prio = self.rev_prio_map[taskid]
+            if not bestprio or bestprio > prio:
+                stamp = self.stamps[taskid]
+                if stamp in self.rq.build_stamps.itervalues():
                     continue
-                return taskid
+                bestprio = prio
+                best = taskid
+
+        return best
 
     def next(self):
         """
@@ -126,6 +149,9 @@ class RunQueueScheduler(object):
         if self.rq.stats.active < self.rq.number_tasks:
             return self.next_buildable_task()
 
+    def newbuilable(self, task):
+        self.buildable.append(task)
+
 class RunQueueSchedulerSpeed(RunQueueScheduler):
     """
     A scheduler optimised for speed. The priority map is sorted by task weight,
@@ -137,9 +163,7 @@ class RunQueueSchedulerSpeed(RunQueueScheduler):
         """
         The priority map is sorted by task weight.
         """
-
-        self.rq = runqueue
-        self.rqdata = rqdata
+        RunQueueScheduler.__init__(self, runqueue, rqdata)
 
         sortweight = sorted(copy.deepcopy(self.rqdata.runq_weight))
         copyweight = copy.deepcopy(self.rqdata.runq_weight)
@@ -1116,6 +1140,7 @@ class RunQueueExecute:
         self.runq_complete = []
 
         self.build_stamps = {}
+        self.build_stamps2 = []
         self.failed_fnids = []
 
         self.stampcache = {}
@@ -1128,6 +1153,7 @@ class RunQueueExecute:
 
         # self.build_stamps[pid] may not exist when use shared work directory.
         if task in self.build_stamps:
+            self.build_stamps2.remove(self.build_stamps[task])
             del self.build_stamps[task]
 
         if status != 0:
@@ -1204,6 +1230,8 @@ class RunQueueExecuteTasks(RunQueueExecute):
 
         self.stampcache = {}
 
+        initial_covered = self.rq.scenequeue_covered.copy()
+
         # Mark initial buildable tasks
         for task in xrange(self.stats.total):
             self.runq_running.append(0)
@@ -1257,13 +1285,28 @@ class RunQueueExecuteTasks(RunQueueExecute):
             except TypeError:
                 covered_remove = bb.utils.better_eval(call2, locs)
 
-        for task in covered_remove:
+        def removecoveredtask(task):
             fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]]
             taskname = self.rqdata.runq_task[task] + '_setscene'
             bb.build.del_stamp(taskname, self.rqdata.dataCache, fn)
-            logger.debug(1, 'Not skipping task %s due to setsceneverify', task)
             self.rq.scenequeue_covered.remove(task)
 
+        toremove = covered_remove
+        for task in toremove:
+            logger.debug(1, 'Not skipping task %s due to setsceneverify', task)
+        while toremove:
+            covered_remove = []
+            for task in toremove:
+                removecoveredtask(task)
+                for deptask in self.rqdata.runq_depends[task]:
+                    if deptask not in self.rq.scenequeue_covered:
+                        continue
+                    if deptask in toremove or deptask in covered_remove or deptask in initial_covered:
+                        continue
+                    logger.debug(1, 'Task %s depends on task %s so not skipping' % (task, deptask))
+                    covered_remove.append(deptask)
+            toremove = covered_remove
+
         logger.debug(1, 'Full skip list %s', self.rq.scenequeue_covered)
 
         event.fire(bb.event.StampUpdate(self.rqdata.target_pairs, self.rqdata.dataCache.stamp), self.cfgData)
@@ -1300,6 +1343,10 @@ class RunQueueExecuteTasks(RunQueueExecute):
                     schedulers.add(getattr(module, name))
         return schedulers
 
+    def setbuildable(self, task):
+        self.runq_buildable[task] = 1
+        self.sched.newbuilable(task)
+
     def task_completeoutright(self, task):
         """
         Mark a task as completed
@@ -1317,7 +1364,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
                 if self.runq_complete[dep] != 1:
                     alldeps = 0
             if alldeps == 1:
-                self.runq_buildable[revdep] = 1
+                self.setbuildable(revdep)
                 fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[revdep]]
                 taskname = self.rqdata.runq_task[revdep]
                 logger.debug(1, "Marking task %s (%s, %s) as buildable", revdep, fn, taskname)
@@ -1341,7 +1388,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
 
     def task_skip(self, task, reason):
         self.runq_running[task] = 1
-        self.runq_buildable[task] = 1
+        self.setbuildable(task)
         bb.event.fire(runQueueTaskSkipped(task, self.stats, self.rq, reason), self.cfgData)
         self.task_completeoutright(task)
         self.stats.taskCompleted()
@@ -1390,17 +1437,20 @@ class RunQueueExecuteTasks(RunQueueExecute):
                 startevent = runQueueTaskStarted(task, self.stats, self.rq)
                 bb.event.fire(startevent, self.cfgData)
 
+            taskdepdata = self.build_taskdepdata(task)
+
             taskdep = self.rqdata.dataCache.task_deps[fn]
-            if 'fakeroot' in taskdep and taskname in taskdep['fakeroot']:
+            if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run:
                 if not self.rq.fakeworker:
                     self.rq.start_fakeworker(self)
-                self.rq.fakeworker.stdin.write("<runtask>" + pickle.dumps((fn, task, taskname, False, self.cooker.collection.get_file_appends(fn))) + "</runtask>")
+                self.rq.fakeworker.stdin.write("<runtask>" + pickle.dumps((fn, task, taskname, False, self.cooker.collection.get_file_appends(fn), taskdepdata)) + "</runtask>")
                 self.rq.fakeworker.stdin.flush()
             else:
-                self.rq.worker.stdin.write("<runtask>" + pickle.dumps((fn, task, taskname, False, self.cooker.collection.get_file_appends(fn))) + "</runtask>")
+                self.rq.worker.stdin.write("<runtask>" + pickle.dumps((fn, task, taskname, False, self.cooker.collection.get_file_appends(fn), taskdepdata)) + "</runtask>")
                 self.rq.worker.stdin.flush()
 
             self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCache, fn)
+            self.build_stamps2.append(self.build_stamps[task]) 
             self.runq_running[task] = 1
             self.stats.taskActive()
             if self.stats.active < self.number_tasks:
@@ -1426,6 +1476,26 @@ class RunQueueExecuteTasks(RunQueueExecute):
 
         return True
 
+    def build_taskdepdata(self, task):
+        taskdepdata = {}
+        next = self.rqdata.runq_depends[task]
+        next.add(task)
+        while next:
+            additional = []
+            for revdep in next:
+                fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[revdep]]
+                pn = self.rqdata.dataCache.pkg_fn[fn]
+                taskname = self.rqdata.runq_task[revdep]
+                deps = self.rqdata.runq_depends[revdep]
+                taskdepdata[revdep] = [pn, taskname, fn, deps]
+                for revdep2 in deps:
+                    if revdep2 not in taskdepdata:
+                        additional.append(revdep2)
+            next = additional
+
+        #bb.note("Task %s: " % task + str(taskdepdata).replace("], ", "],\n"))
+        return taskdepdata
+
 class RunQueueExecuteScenequeue(RunQueueExecute):
     def __init__(self, rq):
         RunQueueExecute.__init__(self, rq)
@@ -1736,10 +1806,10 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
             if 'fakeroot' in taskdep and taskname in taskdep['fakeroot']:
                 if not self.rq.fakeworker:
                     self.rq.start_fakeworker(self)
-                self.rq.fakeworker.stdin.write("<runtask>" + pickle.dumps((fn, realtask, taskname, True, self.cooker.collection.get_file_appends(fn))) + "</runtask>")
+                self.rq.fakeworker.stdin.write("<runtask>" + pickle.dumps((fn, realtask, taskname, True, self.cooker.collection.get_file_appends(fn), None)) + "</runtask>")
                 self.rq.fakeworker.stdin.flush()
             else:
-                self.rq.worker.stdin.write("<runtask>" + pickle.dumps((fn, realtask, taskname, True, self.cooker.collection.get_file_appends(fn))) + "</runtask>")
+                self.rq.worker.stdin.write("<runtask>" + pickle.dumps((fn, realtask, taskname, True, self.cooker.collection.get_file_appends(fn), None)) + "</runtask>")
                 self.rq.worker.stdin.flush()
 
             self.runq_running[task] = 1

bitbake/lib/bb/server/xmlrpc.py

@@ -198,6 +198,11 @@ class XMLRPCServer(SimpleXMLRPCServer, BaseImplServer):
         Constructor
         """
         BaseImplServer.__init__(self)
+        if (interface[1] == 0):     # anonymous port, not getting reused
+            self.single_use = True
+        # Use auto port configuration
+        if (interface[1] == -1):
+            interface = (interface[0], 0)
         SimpleXMLRPCServer.__init__(self, interface,
                                     requestHandler=BitBakeXMLRPCRequestHandler,
                                     logRequests=False, allow_none=True)
@@ -208,8 +213,6 @@ class XMLRPCServer(SimpleXMLRPCServer, BaseImplServer):
         self.autoregister_all_functions(self.commands, "")
         self.interface = interface
         self.single_use = False
-        if (interface[1] == 0):     # anonymous port, not getting reused
-            self.single_use = True
 
     def addcooker(self, cooker):
         BaseImplServer.addcooker(self, cooker)
@@ -338,13 +341,38 @@ class BitBakeXMLRPCClient(BitBakeBaseServer):
     def saveConnectionDetails(self, remote):
         self.remote = remote
 
+    def saveConnectionConfigParams(self, configParams):
+        self.configParams = configParams
+
     def establishConnection(self, featureset):
         # The format of "remote" must be "server:port"
         try:
             [host, port] = self.remote.split(":")
             port = int(port)
-        except:
-            return None
+        except Exception as e:
+            bb.fatal("Failed to read remote definition (%s)" % str(e))
+
+        # use automatic port if port set to -1, meaning read it from
+        # the bitbake.lock file
+        if port == -1:
+            lock_location = "%s/bitbake.lock" % self.configParams.environment.get('BUILDDIR')
+            lock = bb.utils.lockfile(lock_location, False, False)
+            if lock:
+                # This means there is no server running which we can
+                # connect to on the local system.
+                bb.utils.unlockfile(lock)
+                return None
+
+            try:
+                lf = open(lock_location, 'r')
+                remotedef = lf.readline()
+                [host, port] = remotedef.split(":")
+                port = int(port)
+                lf.close()
+                self.remote = remotedef
+            except Exception as e:
+                bb.fatal("Failed to read bitbake.lock (%s)" % str(e))
+
         # We need our IP for the server connection. We get the IP
         # by trying to connect with the server
         try:
@@ -352,8 +380,8 @@ class BitBakeXMLRPCClient(BitBakeBaseServer):
             s.connect((host, port))
             ip = s.getsockname()[0]
             s.close()
-        except:
-            return None
+        except Exception as e:
+            bb.fatal("Could not create socket for %s:%s (%s)" % (host, port, str(e)))
         try:
             self.serverImpl = XMLRPCProxyServer(host, port)
             self.connection = BitBakeXMLRPCServerConnection(self.serverImpl, (ip, 0), self.observer_only, featureset)

bitbake/lib/bb/siggen.py

@@ -224,7 +224,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
 
         if runtime and k in self.taskhash:
             data['runtaskdeps'] = self.runtaskdeps[k]
-            data['file_checksum_values'] = self.file_checksum_values[k]
+            data['file_checksum_values'] = [(os.path.basename(f), cs) for f,cs in self.file_checksum_values[k].items()]
             data['runtaskhashes'] = {}
             for dep in data['runtaskdeps']:
                 data['runtaskhashes'][dep] = self.taskhash[dep]
@@ -322,6 +322,39 @@ def compare_sigfiles(a, b, recursecb = None):
         removed = sb - sa
         return changed, added, removed
 
+    def file_checksums_diff(a, b):
+        from collections import Counter
+        # Handle old siginfo format
+        if isinstance(a, dict):
+            a = [(os.path.basename(f), cs) for f, cs in a.items()]
+        if isinstance(b, dict):
+            b = [(os.path.basename(f), cs) for f, cs in b.items()]
+        # Compare lists, ensuring we can handle duplicate filenames if they exist
+        removedcount = Counter(a)
+        removedcount.subtract(b)
+        addedcount = Counter(b)
+        addedcount.subtract(a)
+        added = []
+        for x in b:
+            if addedcount[x] > 0:
+                addedcount[x] -= 1
+                added.append(x)
+        removed = []
+        changed = []
+        for x in a:
+            if removedcount[x] > 0:
+                removedcount[x] -= 1
+                for y in added:
+                    if y[0] == x[0]:
+                        changed.append((x[0], x[1], y[1]))
+                        added.remove(y)
+                        break
+                else:
+                    removed.append(x)
+        added = [x[0] for x in added]
+        removed = [x[0] for x in removed]
+        return changed, added, removed
+
     if 'basewhitelist' in a_data and a_data['basewhitelist'] != b_data['basewhitelist']:
         output.append("basewhitelist changed from '%s' to '%s'" % (a_data['basewhitelist'], b_data['basewhitelist']))
         if a_data['basewhitelist'] and b_data['basewhitelist']:
@@ -357,10 +390,10 @@ def compare_sigfiles(a, b, recursecb = None):
         for dep in changed:
             output.append("Variable %s value changed from '%s' to '%s'" % (dep, a_data['varvals'][dep], b_data['varvals'][dep]))
 
-    changed, added, removed = dict_diff(a_data['file_checksum_values'], b_data['file_checksum_values'])
+    changed, added, removed = file_checksums_diff(a_data['file_checksum_values'], b_data['file_checksum_values'])
     if changed:
-        for f in changed:
-            output.append("Checksum for file %s changed from %s to %s" % (f, a_data['file_checksum_values'][f], b_data['file_checksum_values'][f]))
+        for f, old, new in changed:
+            output.append("Checksum for file %s changed from %s to %s" % (f, old, new))
     if added:
         for f in added:
             output.append("Dependency on checksum of file %s was added" % (f))

bitbake/lib/bb/ui/buildinfohelper.py

@@ -0,0 +1,738 @@
+#
+# BitBake ToasterUI Implementation
+#
+# Copyright (C) 2013        Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import datetime
+import sys
+import bb
+import re
+import subprocess
+
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "toaster.toastermain.settings")
+
+import toaster.toastermain.settings as toaster_django_settings
+from toaster.orm.models import Build, Task, Recipe, Layer_Version, Layer, Target, LogMessage
+from toaster.orm.models import Variable, VariableHistory
+from toaster.orm.models import Target_Package, Build_Package, Build_File
+from toaster.orm.models import Task_Dependency, Build_Package_Dependency
+from toaster.orm.models import Target_Package_Dependency, Recipe_Dependency
+from bb.msg import BBLogFormatter as format
+
+class ORMWrapper(object):
+    """ This class creates the dictionaries needed to store information in the database
+        following the format defined by the Django models. It is also used to save this
+        information in the database.
+    """
+
+    def __init__(self):
+        pass
+
+
+    def create_build_object(self, build_info):
+
+        build = Build.objects.create(
+                                    machine=build_info['machine'],
+                                    image_fstypes=build_info['image_fstypes'],
+                                    distro=build_info['distro'],
+                                    distro_version=build_info['distro_version'],
+                                    started_on=build_info['started_on'],
+                                    completed_on=build_info['completed_on'],
+                                    cooker_log_path=build_info['cooker_log_path'],
+                                    build_name=build_info['build_name'],
+                                    bitbake_version=build_info['bitbake_version'])
+
+        return build
+
+    def create_target_objects(self, target_info):
+        targets = []
+        for tgt_name in target_info['targets']:
+            tgt_object = Target.objects.create( build = target_info['build'],
+                                    target = tgt_name,
+                                    is_image = False,
+                                    file_name = "",
+                                    file_size = 0);
+            targets.append(tgt_object)
+        return targets
+
+    def update_build_object(self, build, errors, warnings, taskfailures):
+
+        outcome = Build.SUCCEEDED
+        if errors or taskfailures:
+            outcome = Build.FAILED
+
+        build.completed_on = datetime.datetime.now()
+        build.errors_no = errors
+        build.warnings_no = warnings
+        build.outcome = outcome
+        build.save()
+
+
+    def get_update_task_object(self, task_information):
+        task_object, created = Task.objects.get_or_create(
+                                build=task_information['build'],
+                                recipe=task_information['recipe'],
+                                task_name=task_information['task_name'],
+                                )
+
+        for v in vars(task_object):
+            if v in task_information.keys():
+                vars(task_object)[v] = task_information[v]
+        # if we got covered by a setscene task, we're SSTATE
+        if task_object.outcome == Task.OUTCOME_COVERED and 1 == Task.objects.filter(task_executed=True, build = task_object.build, recipe = task_object.recipe, task_name=task_object.task_name+"_setscene").count():
+            task_object.outcome = Task.OUTCOME_SSTATE
+            outcome_task_setscene = Task.objects.get(task_executed=True, build = task_object.build,
+                                    recipe = task_object.recipe, task_name=task_object.task_name+"_setscene").outcome
+            if outcome_task_setscene == Task.OUTCOME_SUCCESS:
+                task_object.sstate_result = Task.SSTATE_RESTORED
+            elif outcome_task_setscene == Task.OUTCOME_FAILED:
+                task_object.sstate_result = Task.SSTATE_FAILED
+
+        # mark down duration if we have a start time
+        if 'start_time' in task_information.keys():
+            duration = datetime.datetime.now() - task_information['start_time']
+            task_object.elapsed_time = duration.total_seconds()
+
+        task_object.save()
+        return task_object
+
+
+    def get_update_recipe_object(self, recipe_information):
+
+        recipe_object, created = Recipe.objects.get_or_create(
+                                         layer_version=recipe_information['layer_version'],
+                                         file_path=recipe_information['file_path'])
+
+        for v in vars(recipe_object):
+            if v in recipe_information.keys():
+                vars(recipe_object)[v] = recipe_information[v]
+
+        recipe_object.save()
+
+        return recipe_object
+
+    def get_layer_version_object(self, layer_version_information):
+
+        layer_version_object = Layer_Version.objects.get_or_create(
+                                    layer = layer_version_information['layer'],
+                                    branch = layer_version_information['branch'],
+                                    commit = layer_version_information['commit'],
+                                    priority = layer_version_information['priority']
+                                    )
+
+        layer_version_object[0].save()
+
+        return layer_version_object[0]
+
+    def get_update_layer_object(self, layer_information):
+
+        layer_object = Layer.objects.get_or_create(
+                                name=layer_information['name'],
+                                local_path=layer_information['local_path'],
+                                layer_index_url=layer_information['layer_index_url'])
+        layer_object[0].save()
+
+        return layer_object[0]
+
+
+    def save_target_package_information(self, target_obj, packagedict, bldpkgs, recipes):
+        for p in packagedict:
+            packagedict[p]['object'] = Target_Package.objects.create( target = target_obj,
+                                        name = p,
+                                        size = packagedict[p]['size'])
+            if p in bldpkgs:
+                packagedict[p]['object'].version = bldpkgs[p]['version']
+                packagedict[p]['object'].recipe =  recipes[bldpkgs[p]['pn']]
+                packagedict[p]['object'].save()
+
+        for p in packagedict:
+            for (px,deptype) in packagedict[p]['depends']:
+                Target_Package_Dependency.objects.create( package = packagedict[p]['object'],
+                                        depends_on = packagedict[px]['object'],
+                                        dep_type = deptype);
+
+
+    def create_logmessage(self, log_information):
+        log_object = LogMessage.objects.create(
+                        build = log_information['build'],
+                        level = log_information['level'],
+                        message = log_information['message'])
+
+        for v in vars(log_object):
+            if v in log_information.keys():
+                vars(log_object)[v] = log_information[v]
+
+        return log_object.save()
+
+
+    def save_build_package_information(self, build_obj, package_info, recipes):
+        # create and save the object
+        bp_object = Build_Package.objects.create( build = build_obj,
+                                       recipe = recipes[package_info['PN']],
+                                       name = package_info['PKG'],
+                                       version = package_info['PKGV'],
+                                       revision = package_info['PKGR'],
+                                       summary = package_info['SUMMARY'],
+                                       description = package_info['DESCRIPTION'],
+                                       size = int(package_info['PKGSIZE']),
+                                       section = package_info['SECTION'],
+                                       license = package_info['LICENSE'],
+                                       )
+        # save any attached file information
+        for path in package_info['FILES_INFO']:
+                fo = Build_File.objects.create( bpackage = bp_object,
+                                        path = path,
+                                        size = package_info['FILES_INFO'][path] )
+
+        # save soft dependency information
+        if 'RDEPENDS' in package_info and package_info['RDEPENDS']:
+            for p in bb.utils.explode_deps(package_info['RDEPENDS']):
+                Build_Package_Dependency.objects.get_or_create( package = bp_object,
+                    depends_on = p, dep_type = Build_Package_Dependency.TYPE_RDEPENDS)
+        if 'RPROVIDES' in package_info and package_info['RPROVIDES']:
+            for p in bb.utils.explode_deps(package_info['RPROVIDES']):
+                Build_Package_Dependency.objects.get_or_create( package = bp_object,
+                    depends_on = p, dep_type = Build_Package_Dependency.TYPE_RPROVIDES)
+        if 'RRECOMMENDS' in package_info and package_info['RRECOMMENDS']:
+            for p in bb.utils.explode_deps(package_info['RRECOMMENDS']):
+                Build_Package_Dependency.objects.get_or_create( package = bp_object,
+                    depends_on = p, dep_type = Build_Package_Dependency.TYPE_RRECOMMENDS)
+        if 'RSUGGESTS' in package_info and package_info['RSUGGESTS']:
+            for p in bb.utils.explode_deps(package_info['RSUGGESTS']):
+                Build_Package_Dependency.objects.get_or_create( package = bp_object,
+                    depends_on = p, dep_type = Build_Package_Dependency.TYPE_RSUGGESTS)
+        if 'RREPLACES' in package_info and package_info['RREPLACES']:
+            for p in bb.utils.explode_deps(package_info['RREPLACES']):
+                Build_Package_Dependency.objects.get_or_create( package = bp_object,
+                    depends_on = p, dep_type = Build_Package_Dependency.TYPE_RREPLACES)
+        if 'RCONFLICTS' in package_info and package_info['RCONFLICTS']:
+            for p in bb.utils.explode_deps(package_info['RCONFLICTS']):
+                Build_Package_Dependency.objects.get_or_create( package = bp_object,
+                    depends_on = p, dep_type = Build_Package_Dependency.TYPE_RCONFLICTS)
+
+        return bp_object
+
+    def save_build_variables(self, build_obj, vardump):
+        for k in vardump:
+            if not bool(vardump[k]['func']):
+                value = vardump[k]['v'];
+                if value is None:
+                    value = ''
+                desc = vardump[k]['doc'];
+                if desc is None:
+                    var_words = [word for word in k.split('_')]
+                    root_var = "_".join([word for word in var_words if word.isupper()])
+                    if root_var and root_var != k and root_var in vardump:
+                        desc = vardump[root_var]['doc']
+                if desc is None:
+                    desc = ''
+                variable_obj = Variable.objects.create( build = build_obj,
+                    variable_name = k,
+                    variable_value = value,
+                    description = desc)
+                for vh in vardump[k]['history']:
+                    VariableHistory.objects.create( variable = variable_obj,
+                            file_name = vh['file'],
+                            line_number = vh['line'],
+                            operation = vh['op'])
+
+class BuildInfoHelper(object):
+    """ This class gathers the build information from the server and sends it
+        towards the ORM wrapper for storing in the database
+        It is instantiated once per build
+        Keeps in memory all data that needs matching before writing it to the database
+    """
+
+    def __init__(self, server, has_build_history = False):
+        self._configure_django()
+        self.internal_state = {}
+        self.task_order = 0
+        self.server = server
+        self.orm_wrapper = ORMWrapper()
+        self.has_build_history = has_build_history
+        self.tmp_dir = self.server.runCommand(["getVariable", "TMPDIR"])[0]
+
+    def _configure_django(self):
+        # Add toaster to sys path for importing modules
+        sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'toaster'))
+
+    ###################
+    ## methods to convert event/external info into objects that the ORM layer uses
+
+    def _get_layer_dict(self, layer_path):
+
+        layer_info = {}
+        layer_name = layer_path.split('/')[-1]
+        layer_url = 'http://layers.openembedded.org/layerindex/layer/{layer}/'
+        layer_url_name = self._get_url_map_name(layer_name)
+
+        layer_info['name'] = layer_name
+        layer_info['local_path'] = layer_path
+        layer_info['layer_index_url'] = layer_url.format(layer=layer_url_name)
+
+        return layer_info
+
+    def _get_url_map_name(self, layer_name):
+        """ Some layers have a different name on openembedded.org site,
+            this method returns the correct name to use in the URL
+        """
+
+        url_name = layer_name
+        url_mapping = {'meta': 'openembedded-core'}
+
+        for key in url_mapping.keys():
+            if key == layer_name:
+                url_name = url_mapping[key]
+
+        return url_name
+
+    def _get_layer_information(self):
+
+        layer_info = {}
+
+        return layer_info
+
+    def _get_layer_version_information(self, layer_object):
+
+        layer_version_info = {}
+        layer_version_info['build'] = self.internal_state['build']
+        layer_version_info['layer'] = layer_object
+        layer_version_info['branch'] = self._get_git_branch(layer_object.local_path)
+        layer_version_info['commit'] = self._get_git_revision(layer_object.local_path)
+        layer_version_info['priority'] = 0
+
+        return layer_version_info
+
+
+    def _get_git_branch(self, layer_path):
+        branch = subprocess.Popen("git symbolic-ref HEAD 2>/dev/null ", cwd=layer_path, shell=True, stdout=subprocess.PIPE).communicate()[0]
+        branch = branch.replace('refs/heads/', '').rstrip()
+        return branch
+
+    def _get_git_revision(self, layer_path):
+        revision = subprocess.Popen("git rev-parse HEAD 2>/dev/null ", cwd=layer_path, shell=True, stdout=subprocess.PIPE).communicate()[0].rstrip()
+        return revision
+
+
+    def _get_build_information(self):
+        build_info = {}
+        # Generate an identifier for each new build
+
+        build_info['machine'] = self.server.runCommand(["getVariable", "MACHINE"])[0]
+        build_info['distro'] = self.server.runCommand(["getVariable", "DISTRO"])[0]
+        build_info['distro_version'] = self.server.runCommand(["getVariable", "DISTRO_VERSION"])[0]
+        build_info['started_on'] = datetime.datetime.now()
+        build_info['completed_on'] = datetime.datetime.now()
+        build_info['image_fstypes'] = self._remove_redundant(self.server.runCommand(["getVariable", "IMAGE_FSTYPES"])[0] or "")
+        build_info['cooker_log_path'] = self.server.runCommand(["getVariable", "BB_CONSOLELOG"])[0]
+        build_info['build_name'] = self.server.runCommand(["getVariable", "BUILDNAME"])[0]
+        build_info['bitbake_version'] = self.server.runCommand(["getVariable", "BB_VERSION"])[0]
+
+        return build_info
+
+    def _get_task_information(self, event, recipe):
+
+
+        task_information = {}
+        task_information['build'] = self.internal_state['build']
+        task_information['outcome'] = Task.OUTCOME_NA
+        task_information['recipe'] = recipe
+        task_information['task_name'] = event.taskname
+        try:
+            # some tasks don't come with a hash. and that's ok
+            task_information['sstate_checksum'] = event.taskhash
+        except AttributeError:
+            pass
+        return task_information
+
+    def _get_layer_version_for_path(self, path):
+        def _slkey(layer_version):
+            return len(layer_version.layer.local_path)
+
+        # Heuristics: we always match recipe to the deepest layer path that
+        # we can match to the recipe file path
+        for bl in sorted(self.internal_state['layer_versions'], reverse=True, key=_slkey):
+            if (path.startswith(bl.layer.local_path)):
+                return bl
+
+        #TODO: if we get here, we didn't read layers correctly
+        assert False
+        return None
+
+    def _get_recipe_information_from_build_event(self, event):
+
+        layer_version_obj = self._get_layer_version_for_path(re.split(':', event.taskfile)[-1])
+
+        recipe_info = {}
+        recipe_info['layer_version'] = layer_version_obj
+        recipe_info['file_path'] = re.split(':', event.taskfile)[-1]
+
+        return recipe_info
+
+    def _get_task_build_stats(self, task_object):
+        bs_path = self._get_path_information(task_object)
+        for bp in bs_path:  # TODO: split for each target
+            task_build_stats = self._get_build_stats_from_file(bp, task_object.task_name)
+
+        return task_build_stats
+
+    def _get_path_information(self, task_object):
+        build_stats_format = "{tmpdir}/buildstats/{target}-{machine}/{buildname}/{package}/"
+        build_stats_path = []
+
+        for t in self.internal_state['targets']:
+            target = t.target
+            machine = self.internal_state['build'].machine
+            buildname = self.internal_state['build'].build_name
+            pe, pv = task_object.recipe.version.split(":",1)
+            if len(pe) > 0:
+                package = task_object.recipe.name + "-" + pe + "_" + pv
+            else:
+                package = task_object.recipe.name + "-" + pv
+
+            build_stats_path.append(build_stats_format.format(tmpdir=self.tmp_dir, target=target,
+                                                     machine=machine, buildname=buildname,
+                                                     package=package))
+
+        return build_stats_path
+
+    def _get_build_stats_from_file(self, bs_path, task_name):
+
+        task_bs_filename = str(bs_path) + str(task_name)
+        task_bs = open(task_bs_filename, 'r')
+
+        cpu_usage = 0
+        disk_io = 0
+        startio = ''
+        endio = ''
+
+        for line in task_bs.readlines():
+            if line.startswith('CPU usage: '):
+                cpu_usage = line[11:]
+            elif line.startswith('EndTimeIO: '):
+                endio = line[11:]
+            elif line.startswith('StartTimeIO: '):
+                startio = line[13:]
+
+        task_bs.close()
+
+        if startio and endio:
+            disk_io = int(endio.strip('\n ')) - int(startio.strip('\n '))
+
+        if cpu_usage:
+            cpu_usage = float(cpu_usage.strip('% \n'))
+
+        task_build_stats = {'cpu_usage': cpu_usage, 'disk_io': disk_io}
+
+        return task_build_stats
+
+    def _remove_redundant(self, string):
+        ret = []
+        for i in string.split():
+            if i not in ret:
+                ret.append(i)
+        return " ".join(ret)
+
+
+    ################################
+    ## external available methods to store information
+
+    def store_layer_info(self):
+        layers = self.server.runCommand(["getVariable", "BBLAYERS"])[0].strip().split(" ")
+        self.internal_state['layers'] = []
+        for layer_path in { l for l in layers if len(l) }:
+            layer_information = self._get_layer_dict(layer_path)
+            self.internal_state['layers'].append(self.orm_wrapper.get_update_layer_object(layer_information))
+
+    def store_started_build(self, event):
+
+        build_information = self._get_build_information()
+
+        build_obj = self.orm_wrapper.create_build_object(build_information)
+        self.internal_state['build'] = build_obj
+
+        # create target information
+        target_information = {}
+        target_information['targets'] = event.getPkgs()
+        target_information['build'] = build_obj
+
+        self.internal_state['targets'] = self.orm_wrapper.create_target_objects(target_information)
+
+        # Load layer information for the build
+        self.internal_state['layer_versions'] = []
+        for layer_object in self.internal_state['layers']:
+            layer_version_information = self._get_layer_version_information(layer_object)
+            self.internal_state['layer_versions'].append(self.orm_wrapper.get_layer_version_object(layer_version_information))
+
+        del self.internal_state['layers']
+        # Save build configuration
+        self.orm_wrapper.save_build_variables(build_obj, self.server.runCommand(["getAllKeysWithFlags", ["doc", "func"]])[0])
+
+
+    def update_build_information(self, event, errors, warnings, taskfailures):
+        if 'build' in self.internal_state:
+            self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures)
+
+    def store_started_task(self, event):
+        identifier = event.taskfile + event.taskname
+
+        recipe_information = self._get_recipe_information_from_build_event(event)
+        recipe = self.orm_wrapper.get_update_recipe_object(recipe_information)
+
+        task_information = self._get_task_information(event, recipe)
+        task_information['outcome'] = Task.OUTCOME_NA
+
+        if isinstance(event, bb.runqueue.runQueueTaskSkipped):
+            task_information['task_executed'] = False
+            if event.reason == "covered":
+                task_information['outcome'] = Task.OUTCOME_COVERED
+            if event.reason == "existing":
+                task_information['outcome'] = Task.OUTCOME_EXISTING
+        else:
+            task_information['task_executed'] = True
+            if 'noexec' in vars(event) and event.noexec == True:
+                task_information['script_type'] = Task.CODING_NOEXEC
+
+        self.task_order += 1
+        task_information['order'] = self.task_order
+        task_obj = self.orm_wrapper.get_update_task_object(task_information)
+
+        self.internal_state[identifier] = {'start_time': datetime.datetime.now()}
+
+    def update_and_store_task(self, event):
+        identifier = event.taskfile + event.taskname
+        recipe_information = self._get_recipe_information_from_build_event(event)
+        recipe = self.orm_wrapper.get_update_recipe_object(recipe_information)
+        task_information = self._get_task_information(event,recipe)
+        try:
+            task_information['start_time'] = self.internal_state[identifier]['start_time']
+        except:
+            pass
+
+        if 'logfile' in vars(event):
+            task_information['logfile'] = event.logfile
+
+        if '_message' in vars(event):
+            task_information['message'] = event._message
+
+        if 'taskflags' in vars(event):
+            # with TaskStarted, we get even more information
+            if 'python' in event.taskflags.keys() and event.taskflags['python'] == '1':
+                task_information['script_type'] = Task.CODING_PYTHON
+            else:
+                task_information['script_type'] = Task.CODING_SHELL
+
+        if isinstance(event, (bb.runqueue.runQueueTaskCompleted, bb.runqueue.sceneQueueTaskCompleted)):
+            task_information['outcome'] = Task.OUTCOME_SUCCESS
+            task_build_stats = self._get_task_build_stats(self.orm_wrapper.get_update_task_object(task_information))
+            task_information['cpu_usage'] = task_build_stats['cpu_usage']
+            task_information['disk_io'] = task_build_stats['disk_io']
+            del self.internal_state[identifier]
+
+        if isinstance(event, (bb.runqueue.runQueueTaskFailed, bb.runqueue.sceneQueueTaskFailed)):
+            task_information['outcome'] = Task.OUTCOME_FAILED
+            del self.internal_state[identifier]
+
+        self.orm_wrapper.get_update_task_object(task_information)
+
+
+    def read_target_package_dep_data(self, event):
+        # for all targets
+        for target in self.internal_state['targets']:
+            # verify that we have something to read
+            if not target.is_image or not self.has_build_history:
+                print "not collecting package info ", target.is_image, self.has_build_history
+                break
+
+            # TODO this is a temporary replication of the code in buildhistory.bbclass
+            # This MUST be changed to query the actual BUILD_DIR_IMAGE in the target context when
+            # the capability will be implemented in Bitbake
+
+            MACHINE_ARCH, error = self.server.runCommand(['getVariable', 'MACHINE_ARCH'])
+            TCLIBC, error = self.server.runCommand(['getVariable', 'TCLIBC'])
+            BUILDHISTORY_DIR, error = self.server.runCommand(['getVariable', 'BUILDHISTORY_DIR'])
+            BUILDHISTORY_DIR_IMAGE = "%s/images/%s/%s/%s" % (BUILDHISTORY_DIR, MACHINE_ARCH, TCLIBC, target.target)
+
+            self.internal_state['packages'] = {}
+
+            with open("%s/installed-package-sizes.txt" % BUILDHISTORY_DIR_IMAGE, "r") as fin:
+                for line in fin:
+                    line = line.rstrip(";")
+                    psize, px = line.split("\t")
+                    punit, pname = px.split(" ")
+                    self.internal_state['packages'][pname.strip()] = {'size':int(psize)*1024, 'depends' : []}
+
+            with open("%s/depends.dot" % BUILDHISTORY_DIR_IMAGE, "r") as fin:
+                p = re.compile(r' -> ')
+                dot = re.compile(r'.*style=dotted')
+                for line in fin:
+                    line = line.rstrip(';')
+                    linesplit = p.split(line)
+                    if len(linesplit) == 2:
+                        pname = linesplit[0].rstrip('"').strip('"')
+                        dependsname = linesplit[1].split(" ")[0].strip().strip(";").strip('"').rstrip('"')
+                        deptype = Target_Package_Dependency.TYPE_DEPENDS
+                        if dot.match(line):
+                            deptype = Target_Package_Dependency.TYPE_RECOMMENDS
+                        if not pname in self.internal_state['packages']:
+                            self.internal_state['packages'][pname] = {'size': 0, 'depends' : []}
+                        if not dependsname in self.internal_state['packages']:
+                            self.internal_state['packages'][dependsname] = {'size': 0, 'depends' : []}
+                        self.internal_state['packages'][pname]['depends'].append((dependsname, deptype))
+
+            self.orm_wrapper.save_target_package_information(target,
+                        self.internal_state['packages'],
+                        self.internal_state['bldpkgs'], self.internal_state['recipes'])
+
+
+    def store_dependency_information(self, event):
+        # save layer version priorities
+        if 'layer-priorities' in event._depgraph.keys():
+            for lv in event._depgraph['layer-priorities']:
+                (name, path, regexp, priority) = lv
+                layer_version_obj = self._get_layer_version_for_path(path[1:]) # paths start with a ^
+                assert layer_version_obj is not None
+                layer_version_obj.priority = priority
+                layer_version_obj.save()
+
+        # save build time package information
+        self.internal_state['bldpkgs'] = {}
+        for pkg  in event._depgraph['packages']:
+            self.internal_state['bldpkgs'][pkg] = event._depgraph['packages'][pkg]
+
+        # save recipe information
+        self.internal_state['recipes'] = {}
+        for pn in event._depgraph['pn']:
+
+            file_name = re.split(':', event._depgraph['pn'][pn]['filename'])[-1]
+            layer_version_obj = self._get_layer_version_for_path(re.split(':', file_name)[-1])
+
+            assert layer_version_obj is not None
+
+            recipe_info = {}
+            recipe_info['name'] = pn
+            recipe_info['version'] = event._depgraph['pn'][pn]['version']
+            recipe_info['layer_version'] = layer_version_obj
+            recipe_info['summary'] = event._depgraph['pn'][pn]['summary']
+            recipe_info['license'] = event._depgraph['pn'][pn]['license']
+            recipe_info['description'] = event._depgraph['pn'][pn]['description']
+            recipe_info['section'] = event._depgraph['pn'][pn]['section']
+            recipe_info['licensing_info'] = 'Not Available'
+            recipe_info['homepage'] = event._depgraph['pn'][pn]['homepage']
+            recipe_info['bugtracker'] = event._depgraph['pn'][pn]['bugtracker']
+            recipe_info['file_path'] = file_name
+            recipe = self.orm_wrapper.get_update_recipe_object(recipe_info)
+            if 'inherits' in event._depgraph['pn'][pn].keys():
+                recipe.is_image = True in map(lambda x: x.endswith('image.bbclass'), event._depgraph['pn'][pn]['inherits'])
+            else:
+                recipe.is_image = False
+            if recipe.is_image:
+                for t in self.internal_state['targets']:
+                    if pn == t.target:
+                        t.is_image = True
+                        t.save()
+            self.internal_state['recipes'][pn] = recipe
+
+        # save recipe dependency
+        # buildtime
+        for recipe in event._depgraph['depends']:
+            try:
+                target = self.internal_state['recipes'][recipe]
+                for dep in event._depgraph['depends'][recipe]:
+                    dependency = self.internal_state['recipes'][dep]
+                    Recipe_Dependency.objects.get_or_create( recipe = target,
+                            depends_on = dependency, dep_type = Recipe_Dependency.TYPE_DEPENDS)
+            except KeyError:    # we'll not get recipes for key w/ values listed in ASSUME_PROVIDED
+                pass
+
+        # runtime
+        for recipe in event._depgraph['rdepends-pn']:
+            try:
+                target = self.internal_state['recipes'][recipe]
+                for dep in event._depgraph['rdepends-pn'][recipe]:
+                    dependency = self.internal_state['recipes'][dep]
+                    Recipe_Dependency.objects.get_or_create( recipe = target,
+                            depends_on = dependency, dep_type = Recipe_Dependency.TYPE_RDEPENDS)
+
+            except KeyError:    # we'll not get recipes for key w/ values listed in ASSUME_PROVIDED
+                pass
+
+        # save all task information
+        def _save_a_task(taskdesc):
+            spec = re.split(r'\.', taskdesc);
+            pn = ".".join(spec[0:-1])
+            taskname = spec[-1]
+            e = event
+            e.taskname = pn
+            recipe = self.internal_state['recipes'][pn]
+            task_info = self._get_task_information(e, recipe)
+            task_info['task_name'] = taskname
+            task_obj = self.orm_wrapper.get_update_task_object(task_info)
+            return task_obj
+
+        for taskdesc in event._depgraph['tdepends']:
+            target = _save_a_task(taskdesc)
+            for taskdesc1 in event._depgraph['tdepends'][taskdesc]:
+                dep = _save_a_task(taskdesc1)
+                Task_Dependency.objects.get_or_create( task = target, depends_on = dep )
+
+    def store_build_package_information(self, event):
+        package_info = event.data
+        self.orm_wrapper.save_build_package_information(self.internal_state['build'],
+                            package_info,
+                            self.internal_state['recipes'],
+                            )
+
+    def _store_log_information(self, level, text):
+        log_information = {}
+        log_information['build'] = self.internal_state['build']
+        log_information['level'] = level
+        log_information['message'] = text
+        self.orm_wrapper.create_logmessage(log_information)
+
+    def store_log_info(self, text):
+        self._store_log_information(LogMessage.INFO, text)
+
+    def store_log_warn(self, text):
+        self._store_log_information(LogMessage.WARNING, text)
+
+    def store_log_error(self, text):
+        self._store_log_information(LogMessage.ERROR, text)
+
+    def store_log_event(self, event):
+        # look up license files info from insane.bbclass
+        m = re.match("([^:]*): md5 checksum matched for ([^;]*)", event.msg)
+        if m:
+            (pn, fn) = m.groups()
+            self.internal_state['recipes'][pn].licensing_info = fn
+            self.internal_state['recipes'][pn].save()
+
+        if event.levelno < format.WARNING:
+            return
+        if not 'build' in self.internal_state:
+            return
+        log_information = {}
+        log_information['build'] = self.internal_state['build']
+        if event.levelno >= format.ERROR:
+            log_information['level'] = LogMessage.ERROR
+        elif event.levelno == format.WARNING:
+            log_information['level'] = LogMessage.WARNING
+        log_information['message'] = event.msg
+        log_information['pathname'] = event.pathname
+        log_information['lineno'] = event.lineno
+        self.orm_wrapper.create_logmessage(log_information)
+

bitbake/lib/bb/ui/crumbs/builddetailspage.py

@@ -185,6 +185,7 @@ class BuildDetailsPage (HobPage):
     def show_issues(self):
         self.num_of_issues += 1
         self.notebook.show_indicator_icon("Issues", self.num_of_issues)
+        self.notebook.queue_draw()
 
     def reset_issues(self):
         self.num_of_issues = 0
@@ -423,7 +424,7 @@ class BuildDetailsPage (HobPage):
         if "recipes" in action:
             self.builder.show_recipes()
         elif "packages" in action:
-            self.builder.show_packages(ask=False)
+            self.builder.show_packages()
         elif "image" in action:
             self.builder.show_configuration()
 

bitbake/lib/bb/ui/crumbs/builder.py

@@ -704,7 +704,6 @@ class Builder(gtk.Window):
         self.set_user_config_proxies()
 
     def set_user_config(self):
-        self.handler.reset_cooker()
         # set bb layers
         self.handler.set_bblayers(self.configuration.layers)
         # set local configuration
@@ -794,8 +793,8 @@ class Builder(gtk.Window):
                 self.generate_image_async(True)
 
     def show_error_dialog(self, msg):
-        lbl = "<b>Hob found an error</b>\n"
-        dialog = CrumbsMessageDialog(self, lbl, gtk.STOCK_DIALOG_ERROR, msg)
+        lbl = "<b>Hob found an error</b>"
+        dialog = CrumbsMessageDialog(self, lbl, gtk.MESSAGE_ERROR, msg)
         button = dialog.add_button("Close", gtk.RESPONSE_OK)
         HobButton.style_button(button)
         response = dialog.run()
@@ -811,10 +810,9 @@ class Builder(gtk.Window):
         dialog.destroy()
 
     def show_network_error_dialog(self):
-        lbl = "<b>Hob cannot connect to the network</b>\n"
-        msg = "Please check your network connection. If you are using a proxy server, please make sure it is configured correctly."
-        lbl = lbl + "%s\n\n" % glib.markup_escape_text(msg)
-        dialog = CrumbsMessageDialog(self, lbl, gtk.STOCK_DIALOG_ERROR)
+        lbl = "<b>Hob cannot connect to the network</b>"
+        msg = msg + "Please check your network connection. If you are using a proxy server, please make sure it is configured correctly."
+        dialog = CrumbsMessageDialog(self, lbl, gtk.MESSAGE_ERROR, msg)
         button = dialog.add_button("Close", gtk.RESPONSE_OK)
         HobButton.style_button(button)
         button = dialog.add_button("Proxy settings", gtk.RESPONSE_CANCEL)
@@ -1037,7 +1035,7 @@ class Builder(gtk.Window):
         self.build_failed()
 
     def handler_no_provider_cb(self, running_build, msg):
-        dialog = CrumbsMessageDialog(self, glib.markup_escape_text(msg), gtk.STOCK_DIALOG_INFO)
+        dialog = CrumbsMessageDialog(self, glib.markup_escape_text(msg), gtk.MESSAGE_INFO)
         button = dialog.add_button("Close", gtk.RESPONSE_OK)
         HobButton.style_button(button)
         dialog.run()
@@ -1100,9 +1098,10 @@ class Builder(gtk.Window):
     def build_packages(self):
         _, all_recipes = self.recipe_model.get_selected_recipes()
         if not all_recipes:
-            lbl = "<b>No selections made</b>\nYou have not made any selections"
-            lbl = lbl + " so there isn't anything to bake at this time."
-            dialog = CrumbsMessageDialog(self, lbl, gtk.STOCK_DIALOG_INFO)
+            lbl = "<b>No selections made</b>"
+            msg = "You have not made any selections"
+            msg = msg + " so there isn't anything to bake at this time."
+            dialog = CrumbsMessageDialog(self, lbl, gtk.MESSAGE_INFO, msg)
             button = dialog.add_button("Close", gtk.RESPONSE_OK)
             HobButton.style_button(button)
             dialog.run()
@@ -1113,9 +1112,10 @@ class Builder(gtk.Window):
     def build_image(self):
         selected_packages = self.package_model.get_selected_packages()
         if not selected_packages:      
-            lbl = "<b>No selections made</b>\nYou have not made any selections"
-            lbl = lbl + " so there isn't anything to bake at this time."
-            dialog = CrumbsMessageDialog(self, lbl, gtk.STOCK_DIALOG_INFO)
+            lbl = "<b>No selections made</b>"
+            msg = "You have not made any selections"
+            msg = msg + " so there isn't anything to bake at this time."
+            dialog = CrumbsMessageDialog(self, lbl, gtk.MESSAGE_INFO, msg)
             button = dialog.add_button("Close", gtk.RESPONSE_OK)
             HobButton.style_button(button)
             dialog.run()
@@ -1129,9 +1129,10 @@ class Builder(gtk.Window):
 
         # If no base image and no selected packages don't build anything
         if not (selected_packages or selected_image != self.recipe_model.__custom_image__):
-            lbl = "<b>No selections made</b>\nYou have not made any selections"
-            lbl = lbl + " so there isn't anything to bake at this time."
-            dialog = CrumbsMessageDialog(self, lbl, gtk.STOCK_DIALOG_INFO)
+            lbl = "<b>No selections made</b>"
+            msg = "You have not made any selections"
+            msg = msg + " so there isn't anything to bake at this time."
+            dialog = CrumbsMessageDialog(self, lbl, gtk.MESSAGE_INFO, msg)
             button = dialog.add_button("Close", gtk.RESPONSE_OK)
             HobButton.style_button(button)
             dialog.run()
@@ -1215,8 +1216,9 @@ class Builder(gtk.Window):
         response = dialog.run()
         if response == gtk.RESPONSE_YES:
             if not dialog.image_names:
-                lbl = "<b>No selections made</b>\nYou have not made any selections"
-                crumbs_dialog = CrumbsMessageDialog(self, lbl, gtk.STOCK_DIALOG_INFO)
+                lbl = "<b>No selections made</b>"
+                msg = "You have not made any selections"
+                crumbs_dialog = CrumbsMessageDialog(self, lbl, gtk.MESSAGE_INFO, msg)
                 button = crumbs_dialog.add_button("Close", gtk.RESPONSE_OK)
                 HobButton.style_button(button)
                 crumbs_dialog.run()
@@ -1303,7 +1305,7 @@ class Builder(gtk.Window):
     def deploy_image(self, image_name):
         if not image_name:
             lbl = "<b>Please select an image to deploy.</b>"
-            dialog = CrumbsMessageDialog(self, lbl, gtk.STOCK_DIALOG_INFO)
+            dialog = CrumbsMessageDialog(self, lbl, gtk.MESSAGE_INFO)
             button = dialog.add_button("Close", gtk.RESPONSE_OK)
             HobButton.style_button(button)
             dialog.run()
@@ -1349,8 +1351,8 @@ class Builder(gtk.Window):
 
     def runqemu_image(self, image_name, kernel_name):
         if not image_name or not kernel_name:
-            lbl = "<b>Please select an %s to launch in QEMU.</b>" % ("kernel" if image_name else "image")
-            dialog = CrumbsMessageDialog(self, lbl, gtk.STOCK_DIALOG_INFO)
+            lbl = "<b>Please select %s to launch in QEMU.</b>" % ("a kernel" if image_name else "an image")
+            dialog = CrumbsMessageDialog(self, lbl, gtk.MESSAGE_INFO)
             button = dialog.add_button("Close", gtk.RESPONSE_OK)
             HobButton.style_button(button)
             dialog.run()
@@ -1371,38 +1373,23 @@ class Builder(gtk.Window):
             cmdline += "runqemu " + kernel_path + " " + image_path + "\"\'"
             subprocess.Popen(shlex.split(cmdline))
         else:
-            lbl = "<b>Path error</b>\nOne of your paths is wrong,"
-            lbl = lbl + " please make sure the following paths exist:\n"
-            lbl = lbl + "image path:" + image_path + "\n"
-            lbl = lbl + "kernel path:" + kernel_path + "\n"
-            lbl = lbl + "source environment path:" + source_env_path + "\n"
-            lbl = lbl + "tmp path: " + tmp_path + "."
-            lbl = lbl + "You may be missing either xterm or vte for terminal services."
-            dialog = CrumbsMessageDialog(self, lbl, gtk.STOCK_DIALOG_ERROR)
+            lbl = "<b>Path error</b>"
+            msg = "One of your paths is wrong,"
+            msg = msg + " please make sure the following paths exist:\n"
+            msg = msg + "image path:" + image_path + "\n"
+            msg = msg + "kernel path:" + kernel_path + "\n"
+            msg = msg + "source environment path:" + source_env_path + "\n"
+            msg = msg + "tmp path: " + tmp_path + "."
+            msg = msg + "You may be missing either xterm or vte for terminal services."
+            dialog = CrumbsMessageDialog(self, lbl, gtk.MESSAGE_ERROR, msg)
             button = dialog.add_button("Close", gtk.RESPONSE_OK)
             HobButton.style_button(button)
             dialog.run()
             dialog.destroy()
 
-    def show_packages(self, ask=True):
-        _, selected_recipes = self.recipe_model.get_selected_recipes()
-        if selected_recipes and ask:
-            lbl = "<b>Package list may be incomplete!</b>\nDo you want to build selected recipes"
-            lbl = lbl + " to get a full list or just view the existing packages?"
-            dialog = CrumbsMessageDialog(self, lbl, gtk.STOCK_DIALOG_INFO)
-            button = dialog.add_button("View packages", gtk.RESPONSE_NO)
-            HobAltButton.style_button(button)
-            button = dialog.add_button("Build packages", gtk.RESPONSE_YES)
-            HobButton.style_button(button)
-            dialog.set_default_response(gtk.RESPONSE_YES)
-            response = dialog.run()
-            dialog.destroy()
-            if response == gtk.RESPONSE_YES:
-                self.generate_packages_async(True)
-            else:
-                self.switch_page(self.PACKAGE_SELECTION)
-        else:
-            self.switch_page(self.PACKAGE_SELECTION)
+    def show_packages(self):
+        self.package_details_page.refresh_tables()
+        self.switch_page(self.PACKAGE_SELECTION)
 
     def show_recipes(self):
         self.switch_page(self.RECIPE_SELECTION)
@@ -1415,26 +1402,28 @@ class Builder(gtk.Window):
 
     def stop_build(self):
         if self.stopping:
-            lbl = "<b>Force Stop build?</b>\nYou've already selected Stop once,"
-            lbl = lbl + " would you like to 'Force Stop' the build?\n\n"
-            lbl = lbl + "This will stop the build as quickly as possible but may"
-            lbl = lbl + " well leave your build directory in an  unusable state"
-            lbl = lbl + " that requires manual steps to fix.\n"
-            dialog = CrumbsMessageDialog(self, lbl, gtk.STOCK_DIALOG_WARNING)
+            lbl = "<b>Force Stop build?</b>"
+            msg = "You've already selected Stop once,"
+            msg = msg + " would you like to 'Force Stop' the build?\n\n"
+            msg = msg + "This will stop the build as quickly as possible but may"
+            msg = msg + " well leave your build directory in an  unusable state"
+            msg = msg + " that requires manual steps to fix."
+            dialog = CrumbsMessageDialog(self, lbl, gtk.MESSAGE_WARNING, msg)
             button = dialog.add_button("Cancel", gtk.RESPONSE_CANCEL)
             HobAltButton.style_button(button)
             button = dialog.add_button("Force Stop", gtk.RESPONSE_YES)
             HobButton.style_button(button)
         else:
-            lbl = "<b>Stop build?</b>\n\nAre you sure you want to stop this"
-            lbl = lbl + " build?\n\n'Stop' will stop the build as soon as all in"
-            lbl = lbl + " progress build tasks are finished. However if a"
-            lbl = lbl + " lengthy compilation phase is in progress this may take"
-            lbl = lbl + " some time.\n\n"
-            lbl = lbl + "'Force Stop' will stop the build as quickly as"
-            lbl = lbl + " possible but may well leave your build directory in an"
-            lbl = lbl + " unusable state that requires manual steps to fix."
-            dialog = CrumbsMessageDialog(self, lbl, gtk.STOCK_DIALOG_WARNING)
+            lbl = "<b>Stop build?</b>"
+            msg = "Are you sure you want to stop this"
+            msg = msg + " build?\n\n'Stop' will stop the build as soon as all in"
+            msg = msg + " progress build tasks are finished. However if a"
+            msg = msg + " lengthy compilation phase is in progress this may take"
+            msg = msg + " some time.\n\n"
+            msg = msg + "'Force Stop' will stop the build as quickly as"
+            msg = msg + " possible but may well leave your build directory in an"
+            msg = msg + " unusable state that requires manual steps to fix."
+            dialog = CrumbsMessageDialog(self, lbl, gtk.MESSAGE_WARNING, msg)
             button = dialog.add_button("Cancel", gtk.RESPONSE_CANCEL)
             HobAltButton.style_button(button)
             button = dialog.add_button("Force stop", gtk.RESPONSE_YES)

bitbake/lib/bb/ui/crumbs/hig/advancedsettingsdialog.py

@@ -183,8 +183,9 @@ class AdvancedSettingsDialog (CrumbsDialog, SettingsUIHelper):
         self.set_save_button_state()
         if self.get_num_checked_image_types() == 0:
             # Show an error dialog
-            lbl = "<b>Select an image type</b>\n\nYou need to select at least one image type."
-            dialog = CrumbsMessageDialog(self, lbl, gtk.STOCK_DIALOG_WARNING)
+            lbl = "<b>Select an image type</b>"
+            msg = "You need to select at least one image type."
+            dialog = CrumbsMessageDialog(self, lbl, gtk.MESSAGE_WARNING, msg)
             button = dialog.add_button("OK", gtk.RESPONSE_OK)
             HobButton.style_button(button)
             response = dialog.run()

bitbake/lib/bb/ui/crumbs/hig/crumbsmessagedialog.py

@@ -31,51 +31,28 @@ BitBake GUI's
 In summary: spacing = 12px, border-width = 6px
 """
 
-class CrumbsMessageDialog(CrumbsDialog):
+class CrumbsMessageDialog(gtk.MessageDialog):
     """
     A GNOME HIG compliant dialog widget.
     Add buttons with gtk.Dialog.add_button or gtk.Dialog.add_buttons
     """
-    def __init__(self, parent=None, label="", icon=gtk.STOCK_INFO, msg=""):
-        super(CrumbsMessageDialog, self).__init__("", parent, gtk.DIALOG_MODAL)
+    def __init__(self, parent = None, label="", dialog_type = gtk.MESSAGE_QUESTION, msg=""):
+        super(CrumbsMessageDialog, self).__init__(None,
+                                                  gtk.DIALOG_MODAL | gtk.DIALOG_DESTROY_WITH_PARENT,
+                                                  dialog_type,
+                                                  gtk.BUTTONS_NONE,
+                                                  None)
 
-        self.set_border_width(6)
-        self.vbox.set_property("spacing", 12)
-        self.action_area.set_property("spacing", 12)
-        self.action_area.set_property("border-width", 6)
+        self.set_skip_taskbar_hint(False)
 
-        first_column = gtk.HBox(spacing=12)
-        first_column.set_property("border-width", 6)
-        first_column.show()
-        self.vbox.add(first_column)
+        self.set_markup(label)
 
-        self.icon = gtk.Image()
-        # We have our own Info icon which should be used in preference of the stock icon
-        self.icon_chk = HobIconChecker()
-        self.icon.set_from_stock(self.icon_chk.check_stock_icon(icon), gtk.ICON_SIZE_DIALOG)
-        self.icon.set_property("yalign", 0.00)
-        self.icon.show()
-        first_column.pack_start(self.icon, expand=False, fill=True, padding=0)
-        
-        if 0 <= len(msg) < 200:
-            lbl = label + "%s" % glib.markup_escape_text(msg)
-            self.label_short = gtk.Label()
-            self.label_short.set_use_markup(True)
-            self.label_short.set_line_wrap(True)
-            self.label_short.set_markup(lbl)
-            self.label_short.set_property("yalign", 0.00)
-            self.label_short.show()
-            first_column.add(self.label_short)
+        if 0 <= len(msg) < 300:
+            self.format_secondary_markup(msg)
         else:
-            second_row = gtk.VBox(spacing=12)
-            second_row.set_property("border-width", 6)
-            self.label_long = gtk.Label()
-            self.label_long.set_use_markup(True)
-            self.label_long.set_line_wrap(True)
-            self.label_long.set_markup(label)
-            self.label_long.set_alignment(0.0, 0.0)
-            second_row.pack_start(self.label_long, expand=False, fill=False, padding=0)
-            self.label_long.show()
+            vbox = self.get_message_area()
+            vbox.set_border_width(1)
+            vbox.set_property("spacing", 12)
             self.textWindow = gtk.ScrolledWindow()
             self.textWindow.set_shadow_type(gtk.SHADOW_IN)
             self.textWindow.set_policy(gtk.POLICY_AUTOMATIC, gtk.POLICY_AUTOMATIC)
@@ -89,7 +66,5 @@ class CrumbsMessageDialog(CrumbsDialog):
             self.msgView.set_buffer(self.buf)
             self.textWindow.add(self.msgView)
             self.msgView.show()
-            second_row.add(self.textWindow)
+            vbox.add(self.textWindow)
             self.textWindow.show()
-            first_column.add(second_row)
-            second_row.show()

bitbake/lib/bb/ui/crumbs/hig/deployimagedialog.py

@@ -160,6 +160,7 @@ class DeployImageDialog (CrumbsDialog):
     def response_cb(self, dialog, response_id):
         if response_id == gtk.RESPONSE_YES:
             lbl = ''
+            msg = ''
             combo_item = self.usb_combo.get_active_text()
             if combo_item and combo_item != self.__dummy_usb__ and self.image_path:
                 cmdline = bb.ui.crumbs.utils.which_terminal()
@@ -172,15 +173,18 @@ class DeployImageDialog (CrumbsDialog):
                     if int(tmpfile.readline().strip()) == 0:
                         lbl = "<b>Deploy image successfully.</b>"
                     else:
-                        lbl = "<b>Failed to deploy image.</b>\nPlease check image <b>%s</b> exists and USB device <b>%s</b> is writable." % (self.image_path, combo_item)
+                        lbl = "<b>Failed to deploy image.</b>"
+                        msg = "Please check image <b>%s</b> exists and USB device <b>%s</b> is writable." % (self.image_path, combo_item)
                     tmpfile.close()
             else:
                 if not self.image_path:
-                    lbl = "<b>No selection made.</b>\nYou have not selected an image to deploy."
+                    lbl = "<b>No selection made.</b>"
+                    msg = "You have not selected an image to deploy."
                 else:
-                    lbl = "<b>No selection made.</b>\nYou have not selected a USB device."
+                    lbl = "<b>No selection made.</b>"
+                    msg = "You have not selected a USB device."
             if len(lbl):
-                crumbs_dialog = CrumbsMessageDialog(self, lbl, gtk.STOCK_DIALOG_INFO)
+                crumbs_dialog = CrumbsMessageDialog(self, lbl, gtk.MESSAGE_INFO, msg)
                 button = crumbs_dialog.add_button("Close", gtk.RESPONSE_OK)
                 HobButton.style_button(button)
                 crumbs_dialog.run()

bitbake/lib/bb/ui/crumbs/hig/layerselectiondialog.py

@@ -92,7 +92,8 @@ class LayerSelectionDialog (CrumbsDialog):
         path = dialog.get_filename()
         dialog.destroy()
 
-        lbl = "<b>Error</b>\nUnable to load layer <i>%s</i> because " % path
+        lbl = "<b>Error</b>"
+        msg = "Unable to load layer <i>%s</i> because " % path
         if response == gtk.RESPONSE_YES:
             import os
             import os.path
@@ -103,15 +104,15 @@ class LayerSelectionDialog (CrumbsDialog):
                 it = layer_store.iter_next(it)
 
             if not path:
-                lbl += "it is an invalid path."
+                msg += "it is an invalid path."
             elif not os.path.exists(path+"/conf/layer.conf"):
-                lbl += "there is no layer.conf inside the directory."
+                msg += "there is no layer.conf inside the directory."
             elif path in layers:
-                lbl += "it is already in loaded layers."
+                msg += "it is already in loaded layers."
             else:
                 layer_store.append([path])
                 return
-            dialog = CrumbsMessageDialog(parent, lbl)
+            dialog = CrumbsMessageDialog(parent, lbl, gtk.MESSAGE_ERROR, msg)
             dialog.add_button(gtk.STOCK_CLOSE, gtk.RESPONSE_OK)
             response = dialog.run()
             dialog.destroy()
@@ -132,12 +133,13 @@ class LayerSelectionDialog (CrumbsDialog):
         tree_selection.set_mode(gtk.SELECTION_SINGLE)
 
         # Allow enable drag and drop of rows including row move
+        dnd_internal_target = ''
+        dnd_targets = [(dnd_internal_target, gtk.TARGET_SAME_WIDGET, 0)]
         layer_tv.enable_model_drag_source( gtk.gdk.BUTTON1_MASK,
-            self.TARGETS,
-            gtk.gdk.ACTION_DEFAULT|
+            dnd_targets,
+            gtk.gdk.ACTION_MOVE)
+        layer_tv.enable_model_drag_dest(dnd_targets,
             gtk.gdk.ACTION_MOVE)
-        layer_tv.enable_model_drag_dest(self.TARGETS,
-            gtk.gdk.ACTION_DEFAULT)
         layer_tv.connect("drag_data_get", self.drag_data_get_cb)
         layer_tv.connect("drag_data_received", self.drag_data_received_cb)
 

bitbake/lib/bb/ui/crumbs/hig/propertydialog.py

@@ -105,13 +105,14 @@ class PropertyDialog(CrumbsDialog):
                 
         def create_package_visual_elements(self):
 
+                import json
+
                 name = self.properties['name']
                 binb = self.properties['binb']
                 size = self.properties['size']
                 recipe = self.properties['recipe']
-                file_list = self.properties['files_list']
+                file_list = json.loads(self.properties['files_list'])
 
-                file_list = file_list.strip("{}'")
                 files_temp = ''
                 paths_temp = ''
                 files_binb = []
@@ -180,58 +181,43 @@ class PropertyDialog(CrumbsDialog):
 
                 #################################### FILES BROUGHT BY PACKAGES ###################################
 
-                if file_list != '':
+                if file_list:
                 
                         self.textWindow = gtk.ScrolledWindow()
                         self.textWindow.set_shadow_type(gtk.SHADOW_IN)
                         self.textWindow.set_policy(gtk.POLICY_AUTOMATIC, gtk.POLICY_AUTOMATIC)
                         self.textWindow.set_size_request(100, 170)
 
-                        sstatemirrors_store = gtk.ListStore(str)
+                        packagefiles_store = gtk.ListStore(str)
 
-                        self.sstatemirrors_tv = gtk.TreeView()
-                        self.sstatemirrors_tv.set_rules_hint(True)
-                        self.sstatemirrors_tv.set_headers_visible(True)
-                        self.textWindow.add(self.sstatemirrors_tv)
+                        self.packagefiles_tv = gtk.TreeView()
+                        self.packagefiles_tv.set_rules_hint(True)
+                        self.packagefiles_tv.set_headers_visible(True)
+                        self.textWindow.add(self.packagefiles_tv)
 
                         self.cell1 = gtk.CellRendererText()
                         col1 = gtk.TreeViewColumn('Package files', self.cell1)
                         col1.set_cell_data_func(self.cell1, self.regex_field)
-                        self.sstatemirrors_tv.append_column(col1)
+                        self.packagefiles_tv.append_column(col1)
 
-                        for items in file_list.split(']'):
-                            if len(items) > 1:
-                                paths_temp = items.split(":")[0]
-                                paths_binb.append(paths_temp.strip(" ,'"))
-                                files_temp = items.split(":")[1]
-                                files_binb.append(files_temp.strip(" ['"))
+                        items = file_list.keys()
+                        items.sort()
+                        for item in items:
+                                fullpath = item
+                                while len(item) > 35:
+                                        item = item[:len(item)/2] + "" + item[len(item)/2+1:]
+                                if len(item) == 35:
+                                        item = item[:len(item)/2] + "..." + item[len(item)/2+3:]
+                                        self.tooltip_items[item] = fullpath
 
-                        unsorted_list = []
-                        
-                        for items in range(len(paths_binb)):
-                              if len(files_binb[items]) > 1:
-                                  for aduse in (files_binb[items].split(",")):
-                                        unsorted_list.append(paths_binb[items].split(name)[len(paths_binb[items].split(name))-1] + '/' + aduse.strip(" '"))
-                       
-                         
-                        unsorted_list.sort()
-                        for items in unsorted_list:
-                                temp = items
-                                while len(items) > 35:
-                                        items = items[:len(items)/2] + "" + items[len(items)/2+1:]
-                                if len(items) == 35:
-                                        items = items[:len(items)/2] + "..." + items[len(items)/2+3:]
-                                        self.tooltip_items[items] = temp
+                                packagefiles_store.append([str(item)])
 
-                                sstatemirrors_store.append([str(items)])
-                                
-                                
-                        self.sstatemirrors_tv.set_model(sstatemirrors_store)
+                        self.packagefiles_tv.set_model(packagefiles_store)
 
                         tips = gtk.Tooltips()
-                        tips.set_tip(self.sstatemirrors_tv, "")
-                        self.sstatemirrors_tv.connect("motion-notify-event", self.treeViewTooltip, tips, 0)
-                        self.sstatemirrors_tv.set_events(gtk.gdk.POINTER_MOTION_MASK)
+                        tips.set_tip(self.packagefiles_tv, "")
+                        self.packagefiles_tv.connect("motion-notify-event", self.treeViewTooltip, tips, 0)
+                        self.packagefiles_tv.set_events(gtk.gdk.POINTER_MOTION_MASK)
                         
                         self.vbox.add(self.textWindow)                                      
 

bitbake/lib/bb/ui/crumbs/hig/saveimagedialog.py

@@ -146,12 +146,11 @@ class SaveImageDialog (CrumbsDialog):
             self.show_invalid_input_error_dialog()
 
     def show_invalid_input_error_dialog(self):
-        lbl = "<b>Invalid characters in image recipe name</b>\n"
+        lbl = "<b>Invalid characters in image recipe name</b>"
         msg = "Image recipe names should be all lowercase and\n"
         msg += "include only alphanumeric characters. The only\n"
         msg += "special character you can use is the ASCII hyphen (-)."
-        lbl = lbl + "\n%s\n" % glib.markup_escape_text(msg)
-        dialog = CrumbsMessageDialog(self, lbl, gtk.STOCK_DIALOG_ERROR)
+        dialog = CrumbsMessageDialog(self, lbl, gtk.MESSAGE_ERROR, msg)
         button = dialog.add_button("Close", gtk.RESPONSE_OK)
         HobButton.style_button(button)
 

bitbake/lib/bb/ui/crumbs/hig/simplesettingsdialog.py

@@ -211,18 +211,19 @@ class SimpleSettingsDialog (CrumbsDialog, SettingsUIHelper):
 
     def response_cb(self, dialog, response_id):
         if response_id == gtk.RESPONSE_YES:
-            # Check that all proxy entries have a corresponding port
-            for proxy, port in zip(self.all_proxy_addresses, self.all_proxy_ports):
-                if proxy.get_text() and not port.get_text():
-                    lbl = "<b>Enter all port numbers</b>\n\n"
-                    msg = "Proxy servers require a port number. Please make sure you have entered a port number for each proxy server."
-                    dialog = CrumbsMessageDialog(self, lbl, gtk.STOCK_DIALOG_WARNING, msg)
-                    button = dialog.add_button("Close", gtk.RESPONSE_OK)
-                    HobButton.style_button(button)
-                    response = dialog.run()
-                    dialog.destroy()
-                    self.emit_stop_by_name("response")
-                    return
+            if self.proxy_checkbox.get_active():
+                # Check that all proxy entries have a corresponding port
+                for proxy, port in zip(self.all_proxy_addresses, self.all_proxy_ports):
+                    if proxy.get_text() and not port.get_text():
+                        lbl = "<b>Enter all port numbers</b>"
+                        msg = "Proxy servers require a port number. Please make sure you have entered a port number for each proxy server."
+                        dialog = CrumbsMessageDialog(self, lbl, gtk.MESSAGE_WARNING, msg)
+                        button = dialog.add_button("Close", gtk.RESPONSE_OK)
+                        HobButton.style_button(button)
+                        response = dialog.run()
+                        dialog.destroy()
+                        self.emit_stop_by_name("response")
+                        return
 
         self.configuration.dldir = self.dldir_text.get_text()
         self.configuration.sstatedir = self.sstatedir_text.get_text()

bitbake/lib/bb/ui/crumbs/hobeventhandler.py

@@ -98,6 +98,7 @@ class HobHandler(gobject.GObject):
 
         self.server = server
         self.error_msg = ""
+        self.lastCommand = ""
         self.initcmd = None
         self.parsing = False
 
@@ -112,6 +113,7 @@ class HobHandler(gobject.GObject):
             self.generating = False
 
     def runCommand(self, commandline):
+        self.lastCommand = commandline[0]
         try:
             result, error = self.server.runCommand(commandline)
             if error:
@@ -147,9 +149,7 @@ class HobHandler(gobject.GObject):
         elif next_command == self.SUB_MATCH_CLASS:
             self.runCommand(["findFilesMatchingInDir", "rootfs_", "classes"])
         elif next_command == self.SUB_PARSE_CONFIG:
-            self.runCommand(["enableDataTracking"])
-            self.runCommand(["parseConfigurationFiles", "conf/.hob.conf", ""])
-            self.runCommand(["disableDataTracking"])
+            self.runCommand(["resetCooker"])
         elif next_command == self.SUB_GNERATE_TGTS:
             self.runCommand(["generateTargetsTree", "classes/image.bbclass", []])
         elif next_command == self.SUB_GENERATE_PKGINFO:
@@ -204,7 +204,8 @@ class HobHandler(gobject.GObject):
             reparse = self.runCommand(["getVariable", "BB_INVALIDCONF"]) or None
             if reparse is True:
                 self.set_var_in_file("BB_INVALIDCONF", False, "local.conf")
-                self.runCommand(["parseConfigurationFiles", "conf/.hob.conf", ""])
+                self.runCommand(["setPrePostConfFiles", "conf/.hob.conf", ""])
+                self.commands_async.prepend(self.SUB_PARSE_CONFIG)
             self.run_next_command()
 
         elif isinstance(event, bb.event.SanityCheckFailed):
@@ -251,6 +252,10 @@ class HobHandler(gobject.GObject):
             self.current_phase = None
             self.run_next_command()
         elif isinstance(event, bb.command.CommandFailed):
+            if self.error_msg == "":
+                self.error_msg = "The command \"" + self.lastCommand
+                self.error_msg += "\" was sent to bitbake server but it failed. Please"
+                self.error_msg += " check the code executed by this command in bitbake."
             self.commands_async = []
             self.display_error()
         elif isinstance(event, (bb.event.ParseStarted,
@@ -298,14 +303,8 @@ class HobHandler(gobject.GObject):
         return
 
     def init_cooker(self):
-        self.runCommand(["initCooker"])
         self.runCommand(["createConfigFile", ".hob.conf"])
 
-    def reset_cooker(self):
-        self.runCommand(["enableDataTracking"])
-        self.runCommand(["resetCooker"])
-        self.runCommand(["disableDataTracking"])
-
     def set_extra_inherit(self, bbclass):
         inherits = self.runCommand(["getVariable", "INHERIT"]) or ""
         inherits = inherits + " " + bbclass
@@ -405,15 +404,17 @@ class HobHandler(gobject.GObject):
         self.run_next_command(self.NETWORK_TEST)
 
     def generate_configuration(self):
-        self.commands_async.append(self.SUB_PARSE_CONFIG)
+        self.runCommand(["setPrePostConfFiles", "conf/.hob.conf", ""])
         self.commands_async.append(self.SUB_PATH_LAYERS)
         self.commands_async.append(self.SUB_FILES_DISTRO)
         self.commands_async.append(self.SUB_FILES_MACH)
         self.commands_async.append(self.SUB_FILES_SDKMACH)
         self.commands_async.append(self.SUB_MATCH_CLASS)
+        self.commands_async.append(self.SUB_PARSE_CONFIG)
         self.run_next_command(self.GENERATE_CONFIGURATION)
 
     def generate_recipes(self):
+        self.runCommand(["setPrePostConfFiles", "conf/.hob.conf", ""])
         self.commands_async.append(self.SUB_PARSE_CONFIG)
         self.commands_async.append(self.SUB_GNERATE_TGTS)
         self.run_next_command(self.GENERATE_RECIPES)
@@ -423,6 +424,7 @@ class HobHandler(gobject.GObject):
         targets.extend(tgts)
         self.recipe_queue = targets
         self.default_task = default_task
+        self.runCommand(["setPrePostConfFiles", "conf/.hob.conf", ""])
         self.commands_async.append(self.SUB_PARSE_CONFIG)
         self.commands_async.append(self.SUB_BUILD_RECIPES)
         self.run_next_command(self.GENERATE_PACKAGES)
@@ -434,6 +436,7 @@ class HobHandler(gobject.GObject):
         self.package_queue = image_packages
         self.toolchain_packages = toolchain_packages
         self.default_task = default_task
+        self.runCommand(["setPrePostConfFiles", "conf/.hob.conf", ""])
         self.commands_async.append(self.SUB_PARSE_CONFIG)
         self.commands_async.append(self.SUB_BUILD_IMAGE)
         self.run_next_command(self.GENERATE_IMAGE)

bitbake/lib/bb/ui/crumbs/hoblistmodel.py

@@ -199,7 +199,9 @@ class PackageListModel(gtk.ListStore):
         return self.cmp_vals(val1, val2, user_data)
 
     def cmp_vals(self, val1, val2, user_data):
-        if val1.startswith(user_data) and not val2.startswith(user_data):
+        if val1 is None or val2 is None:
+            return 0
+        elif val1.startswith(user_data) and not val2.startswith(user_data):
             return -1
         elif not val1.startswith(user_data) and val2.startswith(user_data):
             return 1
@@ -249,7 +251,7 @@ class PackageListModel(gtk.ListStore):
             pkgv = getpkgvalue(pkginfo, 'PKGV', pkg)
             pkgr = getpkgvalue(pkginfo, 'PKGR', pkg)
             # PKGSIZE is artificial, will always be overridden with the package name if present
-            pkgsize = pkginfo.get('PKGSIZE_%s' % pkg, "0")
+            pkgsize = int(pkginfo.get('PKGSIZE_%s' % pkg, "0"))
             # PKG_%s is the renamed version
             pkg_rename = pkginfo.get('PKG_%s' % pkg, "")
             # The rest may be overridden or not
@@ -266,11 +268,10 @@ class PackageListModel(gtk.ListStore):
 
             allow_empty = getpkgvalue(pkginfo, 'ALLOW_EMPTY', pkg, "")
 
-            if pkgsize == "0" and not allow_empty:
+            if pkgsize == 0 and not allow_empty:
                 continue
 
-            # pkgsize is in KB
-            size = HobPage._size_to_string(HobPage._string_to_size(pkgsize + ' KB'))
+            size = HobPage._size_to_string(pkgsize)
             self.set(self.append(), self.COL_NAME, pkg, self.COL_VER, pkgv,
                      self.COL_REV, pkgr, self.COL_RNM, pkg_rename,
                      self.COL_SEC, section, self.COL_SUM, summary,
@@ -575,7 +576,9 @@ class RecipeListModel(gtk.ListStore):
         return self.cmp_vals(val1, val2, user_data)
 
     def cmp_vals(self, val1, val2, user_data):
-        if val1.startswith(user_data) and not val2.startswith(user_data):
+        if val1 is None or val2 is None:
+            return 0
+        elif val1.startswith(user_data) and not val2.startswith(user_data):
             return -1
         elif not val1.startswith(user_data) and val2.startswith(user_data):
             return 1

bitbake/lib/bb/ui/crumbs/hobwidget.py

@@ -734,7 +734,7 @@ class HobCellRendererController(gobject.GObject):
         self.current_angle_pos += self.step_angle
         if self.running_mode == self.MODE_CYCLE_RUNNING:
             if (self.current_angle_pos >= 1):
-                self.current_angle_pos = self.step_angle
+                self.current_angle_pos = 0
         else:
             if self.current_angle_pos > 1:
                 self.force_stop()

bitbake/lib/bb/ui/crumbs/imagedetailspage.py

@@ -355,9 +355,9 @@ class ImageDetailsPage (HobPage):
             vallist.append(base_image)
             i = 0
             for layer in layers:
-                varlist.append(" - ")
                 if i > layer_num_limit:
                     break
+                varlist.append(" - ")
                 i += 1
             vallist.append("")
             i = 0
@@ -655,7 +655,7 @@ class ImageDetailsPage (HobPage):
         self.builder.show_configuration()
 
     def edit_packages_button_clicked_cb(self, button):
-        self.builder.show_packages(ask=False)
+        self.builder.show_packages()
 
     def my_images_button_clicked_cb(self, button):
         self.builder.show_load_my_images_dialog()

bitbake/lib/bb/ui/crumbs/runningbuild.py

@@ -174,6 +174,15 @@ class RunningBuild (gobject.GObject):
                       color,
                       0))
 
+            # if there are warnings while processing a package
+            #  (parent), mark the task with warning color;
+            # in case there are errors, the updates will be
+            #  handled on TaskFailed.
+            if color == HobColors.WARNING and parent:
+                self.model.set(parent, self.model.COL_COLOR, color)
+                if task: #then we have a parent (package), and update it's color
+                    self.model.set(self.tasks_to_iter[(package, None)], self.model.COL_COLOR, color)
+
         elif isinstance(event, bb.build.TaskStarted):
             (package, task) = (event._package, event._task)
 
@@ -201,9 +210,10 @@ class RunningBuild (gobject.GObject):
 
             # Because this parent package now has an active child mark it as
             # such.
-            # @todo if parent is already in error, don't mark it green
-            self.model.set(parent, self.model.COL_ICON, "gtk-execute",
-                           self.model.COL_COLOR, HobColors.RUNNING)
+            self.model.set(parent, self.model.COL_ICON, "gtk-execute")
+            parent_color = self.model.get(parent, self.model.COL_COLOR)[0]
+            if parent_color != HobColors.ERROR and parent_color != HobColors.WARNING:
+                self.model.set(parent, self.model.COL_COLOR, HobColors.RUNNING)
 
             # Add an entry in the model for this task
             i = self.model.append (parent, (None,
@@ -246,20 +256,28 @@ class RunningBuild (gobject.GObject):
                     self.model.set(i, self.model.COL_ICON, icon,
                                    self.model.COL_COLOR, color)
             else:
-                icon = None
-                color = HobColors.OK
+                # Mark the parent package and the task as inactive,
+                # but make sure to preserve error, warnings and active
+                # states
+                parent_color = self.model.get(parent, self.model.COL_COLOR)[0]
+                task_color = self.model.get(current, self.model.COL_COLOR)[0]
 
                 # Mark the task as inactive
-                self.model.set(current, self.model.COL_ICON, icon,
-                               self.model.COL_COLOR, color)
+                self.model.set(current, self.model.COL_ICON, None)
+                if task_color != HobColors.ERROR:
+                    if task_color == HobColors.WARNING:
+                        self.model.set(current, self.model.COL_ICON, 'dialog-warning')
+                    else:
+                        self.model.set(current, self.model.COL_COLOR, HobColors.OK)
 
-                # Mark the parent package as inactive, but make sure to
-                # preserve error and active states
-                i = self.tasks_to_iter[(package, None)]
-                if self.model.get(parent, self.model.COL_ICON) != 'dialog-error':
-                    self.model.set(parent, self.model.COL_ICON, icon)
-                    if num_active == 0:
-                        self.model.set(parent, self.model.COL_COLOR, HobColors.OK)
+                # Mark the parent as inactive
+                if parent_color != HobColors.ERROR:
+                    if parent_color == HobColors.WARNING:
+                        self.model.set(parent, self.model.COL_ICON, "dialog-warning")
+                    else:
+                        self.model.set(parent, self.model.COL_ICON, None)
+                        if num_active == 0:
+                            self.model.set(parent, self.model.COL_COLOR, HobColors.OK)
 
             # Clear the iters and the pids since when the task goes away the
             # pid will no longer be used for messages

bitbake/lib/bb/ui/depexp.py

@@ -202,7 +202,7 @@ def main(server, eventHandler, params):
             return 1
         cmdline = cmdline['action']
         if not cmdline or cmdline[0] != "generateDotGraph":
-            print("This UI is only compatible with the -g option")
+            print("This UI requires the -g option")
             return 1
         ret, error = server.runCommand(["generateDepTreeEvent", cmdline[1], cmdline[2]])
         if error:

bitbake/lib/bb/ui/hob.py

@@ -46,7 +46,7 @@ from bb.ui.crumbs.hoblistmodel import RecipeListModel, PackageListModel
 from bb.ui.crumbs.hobeventhandler import HobHandler
 from bb.ui.crumbs.builder import Builder
 
-featureSet = [bb.cooker.CookerFeatures.HOB_EXTRA_CACHES]
+featureSet = [bb.cooker.CookerFeatures.HOB_EXTRA_CACHES, bb.cooker.CookerFeatures.BASEDATASTORE_TRACKING]
 
 def event_handle_idle_func(eventHandler, hobHandler):
     # Consume as many messages as we can in the time available to us

bitbake/lib/bb/ui/toasterui.py

@@ -0,0 +1,279 @@
+#
+# BitBake ToasterUI Implementation
+# based on (No)TTY UI Implementation by Richard Purdie
+#
+# Handling output to TTYs or files (no TTY)
+#
+# Copyright (C) 2006-2012 Richard Purdie
+# Copyright (C) 2013      Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from __future__ import division
+try:
+    import bb
+except RuntimeError as exc:
+    sys.exit(str(exc))
+
+from bb.ui import uihelper
+from bb.ui.buildinfohelper import BuildInfoHelper
+
+import bb.msg
+import copy
+import fcntl
+import logging
+import os
+import progressbar
+import signal
+import struct
+import sys
+import time
+import xmlrpclib
+
+featureSet = [bb.cooker.CookerFeatures.HOB_EXTRA_CACHES, bb.cooker.CookerFeatures.SEND_DEPENDS_TREE, bb.cooker.CookerFeatures.BASEDATASTORE_TRACKING]
+
+logger = logging.getLogger("BitBake")
+interactive = sys.stdout.isatty()
+
+
+
+def _log_settings_from_server(server):
+    # Get values of variables which control our output
+    includelogs, error = server.runCommand(["getVariable", "BBINCLUDELOGS"])
+    if error:
+        logger.error("Unable to get the value of BBINCLUDELOGS variable: %s" % error)
+        raise BaseException(error)
+    loglines, error = server.runCommand(["getVariable", "BBINCLUDELOGS_LINES"])
+    if error:
+        logger.error("Unable to get the value of BBINCLUDELOGS_LINES variable: %s" % error)
+        raise BaseException(error)
+    return includelogs, loglines
+
+def main(server, eventHandler, params ):
+
+    includelogs, loglines = _log_settings_from_server(server)
+
+    # verify and warn
+    build_history_enabled = True
+    inheritlist, error = server.runCommand(["getVariable", "INHERIT"])
+    if not "buildhistory" in inheritlist.split(" "):
+        logger.warn("buildhistory is not enabled. Please enable INHERIT += \"buildhistory\" to see image details.")
+        build_history_enabled = False
+
+    helper = uihelper.BBUIHelper()
+
+    console = logging.StreamHandler(sys.stdout)
+    format_str = "%(levelname)s: %(message)s"
+    format = bb.msg.BBLogFormatter(format_str)
+    bb.msg.addDefaultlogFilter(console)
+    console.setFormatter(format)
+    logger.addHandler(console)
+
+    if not params.observe_only:
+        logger.error("ToasterUI can only work in observer mode")
+        return
+
+
+    main.shutdown = 0
+    interrupted = False
+    return_value = 0
+    errors = 0
+    warnings = 0
+    taskfailures = []
+
+    buildinfohelper = BuildInfoHelper(server, build_history_enabled)
+    buildinfohelper.store_layer_info()
+
+
+    while True:
+        try:
+            event = eventHandler.waitEvent(0.25)
+
+            if event is None:
+                if main.shutdown > 0:
+                    break
+                continue
+
+            helper.eventHandler(event)
+
+            if isinstance(event, bb.event.BuildStarted):
+                buildinfohelper.store_started_build(event)
+
+            if isinstance(event, (bb.build.TaskStarted, bb.build.TaskSucceeded, bb.build.TaskFailedSilent)):
+                buildinfohelper.update_and_store_task(event)
+                continue
+
+            if isinstance(event, bb.event.LogExecTTY):
+                logger.warn(event.msg)
+                continue
+
+            if isinstance(event, logging.LogRecord):
+                buildinfohelper.store_log_event(event)
+                if event.levelno >= format.ERROR:
+                    errors = errors + 1
+                    return_value = 1
+                elif event.levelno == format.WARNING:
+                    warnings = warnings + 1
+                # For "normal" logging conditions, don't show note logs from tasks
+                # but do show them if the user has changed the default log level to
+                # include verbose/debug messages
+                if event.taskpid != 0 and event.levelno <= format.NOTE:
+                    continue
+
+                logger.handle(event)
+                continue
+
+            if isinstance(event, bb.build.TaskFailed):
+                buildinfohelper.update_and_store_task(event)
+                return_value = 1
+                logfile = event.logfile
+                if logfile and os.path.exists(logfile):
+                    bb.error("Logfile of failure stored in: %s" % logfile)
+                continue
+
+            # these events are unprocessed now, but may be used in the future to log
+            # timing and error informations from the parsing phase in Toaster
+            if isinstance(event, bb.event.ParseStarted):
+                continue
+            if isinstance(event, bb.event.ParseProgress):
+                continue
+            if isinstance(event, bb.event.ParseCompleted):
+                continue
+            if isinstance(event, bb.event.CacheLoadStarted):
+                continue
+            if isinstance(event, bb.event.CacheLoadProgress):
+                continue
+            if isinstance(event, bb.event.CacheLoadCompleted):
+                continue
+            if isinstance(event, bb.event.MultipleProviders):
+                continue
+            if isinstance(event, bb.event.NoProvider):
+                return_value = 1
+                errors = errors + 1
+                if event._runtime:
+                    r = "R"
+                else:
+                    r = ""
+
+                if event._dependees:
+                    text = "Nothing %sPROVIDES '%s' (but %s %sDEPENDS on or otherwise requires it)" % (r, event._item, ", ".join(event._dependees), r)
+                else:
+                    text = "Nothing %sPROVIDES '%s'" % (r, event._item)
+
+                logger.error(text)
+                if event._reasons:
+                    for reason in event._reasons:
+                        logger.error("%s", reason)
+                        text += reason
+                buildinfohelper.store_log_error(text)
+                continue
+
+            if isinstance(event, bb.event.ConfigParsed):
+                continue
+            if isinstance(event, bb.event.RecipeParsed):
+                continue
+
+            # end of saved events
+
+            if isinstance(event, (bb.runqueue.sceneQueueTaskStarted, bb.runqueue.runQueueTaskStarted, bb.runqueue.runQueueTaskSkipped)):
+                buildinfohelper.store_started_task(event)
+                continue
+
+            if isinstance(event, bb.runqueue.runQueueTaskCompleted):
+                buildinfohelper.update_and_store_task(event)
+                continue
+
+            if isinstance(event, bb.runqueue.runQueueTaskFailed):
+                buildinfohelper.update_and_store_task(event)
+                taskfailures.append(event.taskstring)
+                logger.error("Task %s (%s) failed with exit code '%s'",
+                             event.taskid, event.taskstring, event.exitcode)
+                continue
+
+            if isinstance(event, (bb.runqueue.sceneQueueTaskCompleted, bb.runqueue.sceneQueueTaskFailed)):
+                buildinfohelper.update_and_store_task(event)
+                continue
+
+
+            if isinstance(event, (bb.event.TreeDataPreparationStarted, bb.event.TreeDataPreparationCompleted)):
+                continue
+
+            if isinstance(event, (bb.event.BuildCompleted)):
+                buildinfohelper.read_target_package_dep_data(event)
+                buildinfohelper.update_build_information(event, errors, warnings, taskfailures)
+                continue
+
+            if isinstance(event, (bb.command.CommandCompleted,
+                                  bb.command.CommandFailed,
+                                  bb.command.CommandExit)):
+                if (isinstance(event, bb.command.CommandFailed)):
+                    event.levelno = format.ERROR
+                    event.msg = event.error
+                    event.pathname = ""
+                    event.lineno = 0
+                    buildinfohelper.store_log_event(event)
+                    errors += 1
+
+                buildinfohelper.update_build_information(event, errors, warnings, taskfailures)
+
+                # we start a new build info
+                errors = 0
+                warnings = 0
+                taskfailures = []
+                buildinfohelper = BuildInfoHelper(server, build_history_enabled)
+                buildinfohelper.store_layer_info()
+                continue
+
+            if isinstance(event, bb.event.MetadataEvent):
+                if event.type == "SinglePackageInfo":
+                    buildinfohelper.store_build_package_information(event)
+                continue
+
+            # ignore
+            if isinstance(event, (bb.event.BuildBase,
+                                  bb.event.StampUpdate,
+                                  bb.event.RecipePreFinalise,
+                                  bb.runqueue.runQueueEvent,
+                                  bb.runqueue.runQueueExitWait,
+                                  bb.event.OperationProgress,
+                                  bb.command.CommandFailed,
+                                  bb.command.CommandExit,
+                                  bb.command.CommandCompleted,
+                                  bb.cooker.CookerExit)):
+                continue
+
+            if isinstance(event, bb.event.DepTreeGenerated):
+                buildinfohelper.store_dependency_information(event)
+                continue
+
+            logger.error("Unknown event: %s", event)
+
+        except EnvironmentError as ioerror:
+            # ignore interrupted io
+            if ioerror.args[0] == 4:
+                pass
+        except KeyboardInterrupt:
+            main.shutdown = 1
+            pass
+        except Exception as e:
+            logger.error(e)
+            import traceback
+            traceback.print_exc()
+            pass
+
+    if interrupted:
+        if return_value == 0:
+            return_value = 1
+
+    return return_value

bitbake/lib/bb/utils.py

@@ -354,6 +354,9 @@ def better_exec(code, context, text = None, realfile = "<code>"):
         code = better_compile(code, realfile, realfile)
     try:
         exec(code, get_context(), context)
+    except bb.BBHandledException:
+        # Error already shown so passthrough
+        raise
     except Exception as e:
         (t, value, tb) = sys.exc_info()
 
@@ -793,22 +796,28 @@ def copyfile(src, dest, newmtime = None, sstat = None):
         newmtime = sstat[stat.ST_MTIME]
     return newmtime
 
-def which(path, item, direction = 0):
+def which(path, item, direction = 0, history = False):
     """
     Locate a file in a PATH
     """
 
+    hist = []
     paths = (path or "").split(':')
     if direction != 0:
         paths.reverse()
 
     for p in paths:
         next = os.path.join(p, item)
+        hist.append(next)
         if os.path.exists(next):
             if not os.path.isabs(next):
                 next = os.path.abspath(next)
+            if history:
+                return next, hist
             return next
 
+    if history:
+        return "", hist
     return ""
 
 def to_boolean(string, default=None):

bitbake/lib/prserv/serv.py

@@ -13,6 +13,7 @@ except ImportError:
 import bb.server.xmlrpc
 import prserv
 import prserv.db
+import errno
 
 logger = logging.getLogger("BitBake.PRserv")
 
@@ -280,8 +281,18 @@ def stop_daemon(host, port):
         if pid:
             if os.path.exists(pidfile):
                 os.remove(pidfile)
-            os.kill(pid,signal.SIGTERM)
-            time.sleep(0.1)
+
+            wait_timeout = 0
+            while is_running(pid) and wait_timeout < 10:
+                print("Waiting for pr-server to exit.")
+                time.sleep(0.5)
+                wait_timeout += 1
+
+            if is_running(pid):
+                print("Sending SIGTERM to pr-server.")
+                os.kill(pid,signal.SIGTERM)
+                time.sleep(0.1)
+
     except OSError as e:
         err = str(e)
         if err.find("No such process") <= 0:
@@ -289,6 +300,14 @@ def stop_daemon(host, port):
 
     return 0
 
+def is_running(pid):
+    try:
+        os.kill(pid, 0)
+    except OSError as err:
+        if err.errno == errno.ESRCH:
+            return False
+    return True
+
 def is_local_special(host, port):
     if host.strip().upper() == 'localhost'.upper() and (not port):
         return True

bitbake/lib/toaster/bldviewer/api.py

@@ -0,0 +1,37 @@
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2013        Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from django.conf.urls import patterns, include, url
+
+
+urlpatterns = patterns('bldviewer.views',
+        url(r'^builds$', 'model_explorer',  {'model_name':'build'}, name='builds'),
+        url(r'^targets$', 'model_explorer',  {'model_name':'target'}, name='targets'),
+        url(r'^tasks$', 'model_explorer', {'model_name':'task'}, name='task'),
+        url(r'^task_dependencies$', 'model_explorer',  {'model_name':'task_dependency'}, name='task_dependencies'),
+        url(r'^packages$', 'model_explorer',  {'model_name':'build_package'}, name='build_packages'),
+        url(r'^package_dependencies$', 'model_explorer',  {'model_name':'build_package_dependency'}, name='build_package_dependencies'),
+        url(r'^target_packages$', 'model_explorer',  {'model_name':'target_package'}, name='target_packages'),
+        url(r'^package_files$', 'model_explorer',  {'model_name':'build_file'}, name='build_files'),
+        url(r'^layers$', 'model_explorer', {'model_name':'layer'}, name='layer'),
+        url(r'^layerversions$', 'model_explorer', {'model_name':'layerversion'}, name='layerversion'),
+        url(r'^recipes$', 'model_explorer', {'model_name':'recipe'}, name='recipe'),
+        url(r'^recipe_dependencies$', 'model_explorer',  {'model_name':'recipe_dependency'}, name='recipe_dependencies'),
+        url(r'^variables$', 'model_explorer',  {'model_name':'variable'}, name='variables'),
+        url(r'^logmessages$', 'model_explorer',  {'model_name':'logmessage'}, name='logmessages'),
+)

bitbake/lib/toaster/bldviewer/templates/base.html

@@ -0,0 +1,30 @@
+<!DOCTYPE html>
+{% load static %}
+<html>
+	<head>
+		<title>Toaster Simple Explorer</title>
+<script src="{% static 'js/jquery-2.0.3.js' %}">
+</script>
+<script src="{% static 'js/bootstrap.js' %}">
+</script>
+<link href="{% static 'css/bootstrap.css' %}" rel="stylesheet" type="text/css">
+	</head>
+
+<body style="height: 100%">
+<div style="width:100%; height: 100%; position:absolute">
+<div style="width: 100%; height: 3em" class="nav">
+    <ul class="nav nav-tabs">
+        <li><a href="{% url all-builds %}">All Builds</a></li>
+        <li><a href="{% url all-layers %}">All Layers</a></li>
+    </ul>
+</div>
+
+<div style="overflow-y:scroll; width: 100%; position: absolute; top: 3em; bottom:70px ">
+{% block pagecontent %}
+{% endblock %}
+</div>
+<div class="navbar" style="position: absolute; bottom: 0; width:100%"><br/>About Toaster | Yocto Project </div>
+</div>
+</body>
+</html>
+

bitbake/lib/toaster/bldviewer/templates/basebuildpage.html

@@ -0,0 +1,17 @@
+{% extends "basetable.html" %}
+
+{% block pagename %}
+<ul class="nav nav-tabs" style="display: inline-block">
+  <li><a>Build {{build.target_set.all|join:"&nbsp;"}} at {{build.started_on}} : </a></li>
+  <li><a href="{% url task build.id %}"> Tasks </a></li>
+  <li><a href="{% url bpackage build.id %}"> Build Packages </a></li>
+    {% for t in build.target_set.all %}
+        {% if t.is_image %}
+  <li><a href="{% url tpackage build.id t.pk %}"> Packages for {{t.target}} </a> </li>
+        {% endif %}
+    {% endfor %}
+  <li><a href="{% url configuration build.id %}"> Configuration </a> </li>
+</ul>
+     <h1>Toaster - Build {% block pagetitle %} {% endblock %}</h1>
+{% endblock %}
+

bitbake/lib/toaster/bldviewer/templates/basetable.html

@@ -0,0 +1,46 @@
+{% extends "base.html" %}
+
+{% block pagecontent %}
+<script>
+function showhideTableColumn(i, sh) {
+    if (sh)
+        $('td:nth-child('+i+'),th:nth-child('+i+')').show();
+    else
+        $('td:nth-child('+i+'),th:nth-child('+i+')').hide();
+}
+
+
+function filterTableRows(test) {
+    if (test.length > 0) {
+        var r = test.split(/[ ,]+/).map(function (e) { return new RegExp(e, 'i') });
+        $('tr.data').map( function (i, el) {
+            (! r.map(function (j) { return j.test($(el).html())}).reduce(function (c, p) { return c && p;} )) ? $(el).hide() : $(el).show();
+        });
+    } else
+    {
+        $('tr.data').show();
+    }
+}
+</script>
+<div style="margin-bottom: 0.5em">
+
+    {% block pagename %}
+    {% endblock %}
+    <div align="left" style="display:inline-block; width: 40%; margin-left: 2em"> Search: <input type="search" id="filterstring" style="width: 80%" onkeyup="filterTableRows($('#filterstring').val())" autocomplete="off">
+    </div>
+    {% if hideshowcols %}
+    <div align="right" style="display: inline-block; width: 40%">Show/Hide columns:
+    {% for i in hideshowcols %}
+        <span>{{i.name}} <input type="checkbox" id="ct{{i.name}}" onchange="showhideTableColumn({{i.order}}, $('#ct{{i.name}}').is(':checked'))" checked autocomplete="off"></span> |
+    {% endfor %}
+    </div>
+    {% endif %}
+</div>
+<div>
+    <table class="table table-striped table-condensed" style="width:95%">
+{% block pagetable %}
+{% endblock %}
+    </table>
+</div>
+
+{% endblock %}

bitbake/lib/toaster/bldviewer/templates/bfile.html

@@ -0,0 +1,24 @@
+{% extends "basebuildpage.html" %}
+
+{% block pagetitle %}Files for package {{files.0.bpackage.name}} {% endblock %}
+{% block pagetable %}
+    {% if not files %}
+        <p>No files were recorded for this package!</p>
+    {% else %}
+
+            <tr>
+            <th>Name</th>
+            <th>Size (Bytes)</th>
+            </tr>
+
+            {% for file in files %}
+
+            <tr class="data">
+            <td>{{file.path}}</td>
+            <td>{{file.size}}</td>
+
+            {% endfor %}
+
+    {% endif %}
+
+{% endblock %}

bitbake/lib/toaster/bldviewer/templates/bpackage.html

@@ -0,0 +1,44 @@
+{% extends "basebuildpage.html" %}
+
+{% block pagetitle %}Packages{% endblock %}
+{% block pagetable %}
+    {% if not packages %}
+        <p>No packages were recorded for this target!</p>
+    {% else %}
+
+            <tr>
+            <th>Name</th>
+            <th>Version</th>
+            <th>Recipe</th>
+            <th>Summary</th>
+            <th>Section</th>
+            <th>Description</th>
+            <th>Size on host disk (Bytes)</th>
+            <th>License</th>
+            <th>Dependencies List (all)</th>
+            </tr>
+
+            {% for package in packages %}
+
+            <tr class="data">
+                <td><a name="#{{package.name}}" href="{% url bfile build.pk package.pk %}">{{package.name}} ({{package.filelist_bpackage.count}} files)</a></td>
+                <td>{{package.version}}-{{package.revision}}</td>
+                <td><a href="{% url layer_versions_recipes package.recipe.layer_version_id %}#{{package.recipe.name}}">{{package.recipe.name}}</a>{{package.package_name}}</a></td>
+
+            <td>{{package.summary}}</td>
+            <td>{{package.section}}</td>
+            <td>{{package.description}}</td>
+            <td>{{package.size}}</td>
+            <td>{{package.license}}</td>
+            <td>
+        <div style="height: 3em; overflow:auto">
+            {% for bpd in package.bpackage_dependencies_package.all %}
+                {{bpd.dep_type}}: {{bpd.depends_on}} <br/>
+            {% endfor %}
+        </div>
+            </td>
+            {% endfor %}
+
+    {% endif %}
+
+{% endblock %}

bitbake/lib/toaster/bldviewer/templates/build.html

@@ -0,0 +1,43 @@
+{% extends "basetable.html" %}
+
+{% block pagename %}
+    <h1>Toaster - Builds</h1>
+{% endblock %}
+
+{% block pagetable %}
+
+    {% load projecttags %}
+        <tr>
+            <th>Outcome</th>
+            <th>Started On</th>
+            <th>Completed On</th>
+            <th>Target</th>
+            <th>Machine</th>
+            <th>Time</th>
+            <th>Errors</th>
+            <th>Warnings</th>
+            <th>Output</th>
+            <th>Log</th>
+            <th>Bitbake Version</th>
+            <th>Build Name</th>
+        </tr>
+        {% for build in builds %}
+        <tr class="data">
+            <td><a href="{% url configuration build.id %}">{{build.get_outcome_display}}</a></td>
+            <td>{{build.started_on}}</td>
+            <td>{{build.completed_on}}</td>
+            <td>{% for t in build.target_set.all %}<a href="{% url tpackage build.id t.id %}">{{t.target}}</a>{% if t.is_image %} (Img){% endif %}<br/>{% endfor %}</td>
+            <td>{{build.machine}}</td>
+            <td>{% time_difference build.started_on build.completed_on %}</td>
+            <td>{{build.errors_no}}:{% if  build.errors_no %}{% for error in logs %}{% if error.build == build %}{% if error.level == 2 %}<p>{{error.message}}</p>{% endif %}{% endif %}{% endfor %}{% else %}None{% endif %}</td>
+            <td>{{build.warnings_no}}:{% if  build.warnings_no %}{% for warning in logs %}{% if warning.build == build %}{% if warning.level == 1 %}<p>{{warning.message}}</p>{% endif %}{% endif %}{% endfor %}{% else %}None{% endif %}</td>
+            <td>{% if build.outcome == 0 %}{% for t in build.target_set.all %}{% if t.is_image %}{{build.image_fstypes}}{% endif %}{% endfor %}{% endif %}</td>
+            <td>{{build.cooker_log_path}}</td>
+            <td>{{build.bitbake_version}}</td>
+            <td>{{build.build_name}}</td>
+        </tr>
+
+        {% endfor %}
+{% endblock %}
+
+

bitbake/lib/toaster/bldviewer/templates/configuration.html

@@ -0,0 +1,22 @@
+{% extends "basebuildpage.html" %}
+
+{% block pagetitle %}Configuration{% endblock %}
+{% block pagetable %}
+
+            <tr>
+            <th>Name</th>
+            <th>Description</th>
+            <th>Definition history</th>
+            <th>Value</th>
+            </tr>
+
+            {% for variable in configuration %}
+
+            <tr class="data">
+                <td>{{variable.variable_name}}</td>
+                <td>{% if variable.description %}{{variable.description}}{% endif %}</td>
+                <td>{% for vh in variable.variablehistory_set.all %}{{vh.operation}} in {{vh.file_name}}:{{vh.line_number}}<br/>{%endfor%}</td>
+                <td>{{variable.variable_value}}</td>
+            {% endfor %}
+
+{% endblock %}

bitbake/lib/toaster/bldviewer/templates/layer.html

@@ -0,0 +1,34 @@
+{% extends "basetable.html" %}
+
+{% block pagename %}
+    <h1>Toaster - Layers</h1>
+{% endblock %}
+
+{% block pagetable %}
+    {% load projecttags %}
+
+        <tr>
+            <th>Name</th>
+            <th>Local Path</th>
+            <th>Layer Index URL</th>
+            <th>Known Versions</th>
+        </tr>
+
+        {% for layer in layers %}
+
+        <tr class="data">
+            <td>{{layer.name}}</td>
+            <td>{{layer.local_path}}</td>
+            <td><a href='{{layer.layer_index_url}}'>{{layer.layer_index_url}}</a></td>
+            <td><table>
+            {% for lv in layer.versions %}
+                <tr><td>
+        <a href="{% url layer_versions_recipes lv.id %}">({{lv.priority}}){{lv.branch}}:{{lv.commit}} ({{lv.count}} recipes)</a>
+                </td></tr>
+            {% endfor %}
+            </table></td>
+        </tr>
+
+        {% endfor %}
+
+{% endblock %}

bitbake/lib/toaster/bldviewer/templates/package.html

@@ -0,0 +1,36 @@
+{% extends "basebuildpage.html" %}
+
+{% block pagetable %}
+    {% if not packages %}
+        <p>No packages were recorded for this target!</p>
+    {% else %}
+
+            <tr>
+            <th>Name</th>
+            <th>Version</th>
+            <th>Size (Bytes)</th>
+            <th>Recipe</th>
+            <th>Depends on</th>
+            </tr>
+
+            {% for package in packages %}
+
+            <tr class="data">
+                <td><a name="#{{package.name}}">{{package.name}}</a></td>
+                <td>{{package.version}}</td>
+                <td>{{package.size}}</td>
+                <td>{%if package.recipe %}<a name="{{package.recipe.name}}.{{package.package_name}}">
+                <a href="{% url layer_versions_recipes package.recipe.layer_version_id %}#{{package.recipe.name}}">{{package.recipe.name}}</a>{{package.package_name}}</a>{%endif%}</td>
+                <td>
+            <div style="height: 4em; overflow:auto">
+                    {% for d in package.tpackage_dependencies_package.all %}
+                    <a href="#{{d.name}}">{{d.depends_on.name}}</a><br/>
+                    {% endfor %}
+            </div>
+                </td>
+
+            {% endfor %}
+
+    {% endif %}
+
+{% endblock %}

bitbake/lib/toaster/bldviewer/templates/recipe.html

@@ -0,0 +1,53 @@
+{% extends "basetable.html" %}
+
+{% block pagename %}
+<ul class="nav nav-tabs" style="display: inline-block">
+  <li><a>Layer  {{layer_version.layer.name}}&nbsp;:&nbsp;{{layer_version.branch}}&nbsp;:&nbsp;{{layer_version.commit}}&nbsp;:&nbsp;{{layer_version.priority}}</a></li>
+</ul>
+    <h1>Toaster - Recipes for a Layer</h1>
+{% endblock %}
+
+{% block pagetable %}
+    {% load projecttags %}
+
+        <tr>
+        </tr>
+            <th>Name</th>
+            <th>Version</th>
+            <th>Summary</th>
+            <th>Description</th>
+            <th>Section</th>
+            <th>License</th>
+            <th>License file</th>
+            <th>Homepage</th>
+            <th>Bugtracker</th>
+            <th>Author</th>
+            <th>File_path</th>
+            <th style="width: 30em">Recipe Dependency</th>
+
+
+        {% for recipe in recipes %}
+
+        <tr class="data">
+            <td><a name="{{recipe.name}}">{{recipe.name}}</a></td>
+            <td>{{recipe.version}}</td>
+            <td>{{recipe.summary}}</td>
+            <td>{{recipe.description}}</td>
+            <td>{{recipe.section}}</td>
+            <td>{{recipe.license}}</td>
+            <td>{{recipe.licensing_info}}</td>
+            <td>{{recipe.homepage}}</td>
+            <td>{{recipe.bugtracker}}</td>
+            <td>{{recipe.file_path}}</td>
+            <td>
+        <div style="height: 5em; overflow:auto">
+            {% for rr in recipe.r_dependencies_recipe.all %}
+                <a href="#{{rr.depends_on.name}}">{{rr.depends_on.name}}</a><br/>
+            {% endfor %}
+        </div>
+            </td>
+        </tr>
+
+        {% endfor %}
+
+{% endblock %}

bitbake/lib/toaster/bldviewer/templates/task.html

@@ -0,0 +1,63 @@
+{% extends "basebuildpage.html" %}
+
+{% block pagetitle %}Tasks{% endblock %}
+{% block pagetable %}
+    {% if not tasks %}
+        <p>No tasks were executed in this build!</p>
+    {% else %}
+
+            <tr>
+            <th>Order</th>
+            <th>Task</th>
+            <th>Recipe Version</th>
+            <th>Task Type</th>
+            <th>Checksum</th>
+            <th>Outcome</th>
+            <th>Message</th>
+            <th>Logfile</th>
+            <th>Time</th>
+            <th>CPU usage</th>
+            <th>Disk I/O</th>
+            <th>Script type</th>
+            <th>File path</th>
+            <th>Depends</th>
+            </tr>
+
+            {% for task in tasks %}
+
+            <tr class="data">
+                <td>{{task.order}}</td>
+                <td><a name="{{task.recipe.name}}.{{task.task_name}}">
+                <a href="{% url layer_versions_recipes task.recipe.layer_version_id %}#{{task.recipe.name}}">{{task.recipe.name}}</a>.{{task.task_name}}</a></td>
+                <td>{{task.recipe.version}}</td>
+
+                {% if task.task_executed %}
+                <td>Executed</td>
+                {% else %}
+                <td>Prebuilt</td>
+                {% endif %}
+
+                <td>{{task.sstate_checksum}}</td>
+                <td>{{task.get_outcome_display}}{% if task.provider %}</br>(by <a href="#{{task.provider.recipe.name}}.{{task.provider.task_name}}">{{task.provider.recipe.name}}.{{task.provider.task_name}}</a>){% endif %}</td>
+                <td><p>{{task.message}}</td>
+                <td><a target="_fileview" href="file:///{{task.logfile}}">{{task.logfile}}</a></td>
+                <td>{{task.elapsed_time}}</td>
+                <td>{{task.cpu_usage}}</td>
+                <td>{{task.disk_io}}</td>
+                <td>{{task.get_script_type_display}}</td>
+                <td><a target="_fileview" href="file:///{{task.recipe.file_path}}">{{task.recipe.file_path}}</a></td>
+                <td>
+            <div style="height: 3em; overflow:auto">
+                {% for tt in task.task_dependencies_task.all %}
+                    <a href="#{{tt.depends_on.recipe.name}}.{{tt.depends_on.task_name}}">
+                    {{tt.depends_on.recipe.name}}.{{tt.depends_on.task_name}}</a><br/>
+                {% endfor %}
+            </div>
+                </td>
+                </tr>
+
+            {% endfor %}
+
+    {% endif %}
+
+{% endblock %}

bitbake/lib/toaster/bldviewer/templatetags/projecttags.py

@@ -0,0 +1,26 @@
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2013        Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from datetime import datetime
+from django import template
+
+register = template.Library()
+
+@register.simple_tag
+def time_difference(start_time, end_time):
+    return end_time - start_time

bitbake/lib/toaster/bldviewer/urls.py

@@ -0,0 +1,32 @@
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2013        Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from django.conf.urls import patterns, include, url
+from django.views.generic.simple import redirect_to
+
+urlpatterns = patterns('bldviewer.views',
+        url(r'^builds/$', 'build', name='all-builds'),
+        url(r'^build/(?P<build_id>\d+)/task/$', 'task', name='task'),
+        url(r'^build/(?P<build_id>\d+)/packages/$', 'bpackage', name='bpackage'),
+        url(r'^build/(?P<build_id>\d+)/package/(?P<package_id>\d+)/files/$', 'bfile', name='bfile'),
+        url(r'^build/(?P<build_id>\d+)/target/(?P<target_id>\d+)/packages/$', 'tpackage', name='tpackage'),
+        url(r'^build/(?P<build_id>\d+)/configuration/$', 'configuration', name='configuration'),
+        url(r'^layers/$', 'layer', name='all-layers'),
+        url(r'^layerversions/(?P<layerversion_id>\d+)/recipes/.*$', 'layer_versions_recipes', name='layer_versions_recipes'),
+        url(r'^$', redirect_to, {'url': 'builds/'}),
+)

bitbake/lib/toaster/bldviewer/views.py

@@ -0,0 +1,260 @@
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2013        Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import operator
+
+from django.db.models import Q
+from django.shortcuts import render
+from orm.models import Build, Target, Task, Layer, Layer_Version, Recipe, Target_Package, LogMessage, Variable
+from orm.models import Task_Dependency, Recipe_Dependency, Build_Package, Build_File, Build_Package_Dependency
+from django.views.decorators.cache import cache_control
+
+@cache_control(no_store=True)
+def build(request):
+    template = 'build.html'
+    build_info = Build.objects.all()
+
+    logs = LogMessage.objects.all()
+
+    context = {'builds': build_info, 'logs': logs ,
+        'hideshowcols' : [
+                {'name': 'Output', 'order':10},
+                {'name': 'Log', 'order':11},
+            ]}
+
+    return render(request, template, context)
+
+
+def _find_task_revdep(task):
+    tp = []
+    for p in Task_Dependency.objects.filter(depends_on=task):
+        tp.append(p.task);
+    return tp
+
+def _find_task_provider(task):
+    task_revdeps = _find_task_revdep(task)
+    for tr in task_revdeps:
+        if tr.outcome != Task.OUTCOME_COVERED:
+            return tr
+    for tr in task_revdeps:
+        trc = _find_task_provider(tr)
+        if trc is not None:
+            return trc
+    return None
+
+def task(request, build_id):
+    template = 'task.html'
+
+    tasks = Task.objects.filter(build=build_id)
+
+    for t in tasks:
+        if t.outcome == Task.OUTCOME_COVERED:
+            t.provider = _find_task_provider(t)
+
+    context = {'build': Build.objects.filter(pk=build_id)[0], 'tasks': tasks}
+
+    return render(request, template, context)
+
+def configuration(request, build_id):
+    template = 'configuration.html'
+    variables = Variable.objects.filter(build=build_id)
+    context = {'build': Build.objects.filter(pk=build_id)[0], 'configuration' : variables}
+    return render(request, template, context)
+
+def bpackage(request, build_id):
+    template = 'bpackage.html'
+    packages = Build_Package.objects.filter(build = build_id)
+    context = {'build': Build.objects.filter(pk=build_id)[0], 'packages' : packages}
+    return render(request, template, context)
+
+def bfile(request, build_id, package_id):
+    template = 'bfile.html'
+    files = Build_File.objects.filter(bpackage = package_id)
+    context = {'build': Build.objects.filter(pk=build_id)[0], 'files' : files}
+    return render(request, template, context)
+
+def tpackage(request, build_id, target_id):
+    template = 'package.html'
+
+    packages = Target_Package.objects.filter(target=target_id)
+
+    context = {'build' : Build.objects.filter(pk=build_id)[0],'packages': packages}
+
+    return render(request, template, context)
+
+def layer(request):
+    template = 'layer.html'
+    layer_info = Layer.objects.all()
+
+    for li in layer_info:
+        li.versions = Layer_Version.objects.filter(layer = li)
+        for liv in li.versions:
+            liv.count = Recipe.objects.filter(layer_version__id = liv.id).count()
+
+    context = {'layers': layer_info}
+
+    return render(request, template, context)
+
+
+def layer_versions_recipes(request, layerversion_id):
+    template = 'recipe.html'
+    recipes = Recipe.objects.filter(layer_version__id = layerversion_id)
+
+    context = {'recipes': recipes,
+            'layer_version' : Layer_Version.objects.filter( id = layerversion_id )[0]
+    }
+
+    return render(request, template, context)
+
+#### API
+
+import json
+from django.core import serializers
+from django.http import HttpResponse, HttpResponseBadRequest
+
+
+def model_explorer(request, model_name):
+
+    DESCENDING = 'desc'
+    response_data = {}
+    model_mapping = {
+        'build': Build,
+        'target': Target,
+        'target_package': Target_Package,
+        'task': Task,
+        'task_dependency': Task_Dependency,
+        'package': Build_Package,
+        'layer': Layer,
+        'layerversion': Layer_Version,
+        'recipe': Recipe,
+        'recipe_dependency': Recipe_Dependency,
+        'build_package': Build_Package,
+        'build_package_dependency': Build_Package_Dependency,
+        'build_file': Build_File,
+        'variable': Variable,
+        'logmessage': LogMessage,
+        }
+
+    if model_name not in model_mapping.keys():
+        return HttpResponseBadRequest()
+
+    model = model_mapping[model_name]
+
+    try:
+        limit = int(request.GET.get('limit', 0))
+    except ValueError:
+        limit = 0
+
+    try:
+        offset = int(request.GET.get('offset', 0))
+    except ValueError:
+        offset = 0
+
+    ordering_string, invalid = _validate_input(request.GET.get('orderby', ''),
+                                               model)
+    if invalid:
+        return HttpResponseBadRequest()
+
+    filter_string, invalid = _validate_input(request.GET.get('filter', ''),
+                                             model)
+    if invalid:
+        return HttpResponseBadRequest()
+
+    search_term = request.GET.get('search', '')
+
+    if filter_string:
+        filter_terms = _get_filtering_terms(filter_string)
+        try:
+            queryset = model.objects.filter(**filter_terms)
+        except ValueError:
+            queryset = []
+    else:
+        queryset = model.objects.all()
+
+    if search_term:
+        queryset = _get_search_results(search_term, queryset, model)
+
+    if ordering_string and queryset:
+        column, order = ordering_string.split(':')
+        if order.lower() == DESCENDING:
+            queryset = queryset.order_by('-' + column)
+        else:
+            queryset = queryset.order_by(column)
+
+    if offset and limit:
+        queryset = queryset[offset:(offset+limit)]
+    elif offset:
+        queryset = queryset[offset:]
+    elif limit:
+        queryset = queryset[:limit]
+
+    if queryset:
+        response_data['count'] = queryset.count()
+    else:
+        response_data['count'] = 0
+
+    response_data['list'] = serializers.serialize('json', queryset)
+
+    return HttpResponse(json.dumps(response_data),
+                        content_type='application/json')
+
+def _get_filtering_terms(filter_string):
+
+    search_terms = filter_string.split(":")
+    keys = search_terms[0].split(',')
+    values = search_terms[1].split(',')
+
+    return dict(zip(keys, values))
+
+def _validate_input(input, model):
+
+    invalid = 0
+
+    if input:
+        input_list = input.split(":")
+
+        # Check we have only one colon
+        if len(input_list) != 2:
+            invalid = 1
+            return None, invalid
+
+        # Check we have an equal number of terms both sides of the colon
+        if len(input_list[0].split(',')) != len(input_list[1].split(',')):
+            invalid = 1
+            return None, invalid
+
+        # Check we are looking for a valid field
+        valid_fields = model._meta.get_all_field_names()
+        for field in input_list[0].split(','):
+            if field not in valid_fields:
+                invalid = 1
+                return None, invalid
+
+    return input, invalid
+
+def _get_search_results(search_term, queryset, model):
+    search_objects = []
+    for st in search_term.split(" "):
+        q_map = map(lambda x: Q(**{x+'__icontains': st}),
+                model.search_allowed_fields)
+
+        search_objects.append(reduce(operator.or_, q_map))
+    search_object = reduce(operator.and_, search_objects)
+    queryset = queryset.filter(search_object)
+
+    return queryset

bitbake/lib/toaster/manage.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+import os
+import sys
+
+if __name__ == "__main__":
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "toastermain.settings")
+
+    from django.core.management import execute_from_command_line
+
+    execute_from_command_line(sys.argv)

bitbake/lib/toaster/orm/models.py

@@ -0,0 +1,265 @@
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2013        Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from django.db import models
+from django.utils.encoding import python_2_unicode_compatible
+
+
+class Build(models.Model):
+    SUCCEEDED = 0
+    FAILED = 1
+    IN_PROGRESS = 2
+
+    BUILD_OUTCOME = (
+        (SUCCEEDED, 'Succeeded'),
+        (FAILED, 'Failed'),
+        (IN_PROGRESS, 'In Progress'),
+    )
+
+    search_allowed_fields = ['machine',
+                             'cooker_log_path']
+
+    machine = models.CharField(max_length=100)
+    image_fstypes = models.CharField(max_length=100)
+    distro = models.CharField(max_length=100)
+    distro_version = models.CharField(max_length=100)
+    started_on = models.DateTimeField()
+    completed_on = models.DateTimeField()
+    outcome = models.IntegerField(choices=BUILD_OUTCOME, default=IN_PROGRESS)
+    errors_no = models.IntegerField(default=0)
+    warnings_no = models.IntegerField(default=0)
+    cooker_log_path = models.CharField(max_length=500)
+    build_name = models.CharField(max_length=100)
+    bitbake_version = models.CharField(max_length=50)
+
+@python_2_unicode_compatible
+class Target(models.Model):
+    search_allowed_fields = ['target', 'image_fstypes', 'file_name']
+    build = models.ForeignKey(Build)
+    target = models.CharField(max_length=100)
+    is_image = models.BooleanField(default = False)
+    file_name = models.CharField(max_length=100)
+    file_size = models.IntegerField()
+
+    def __str__(self):
+        return self.target
+
+
+class Task(models.Model):
+
+    SSTATE_NA = 0
+    SSTATE_MISS = 1
+    SSTATE_FAILED = 2
+    SSTATE_RESTORED = 3
+
+    SSTATE_RESULT = (
+        (SSTATE_NA, 'Not Applicable'), # For rest of tasks, but they still need checking.
+        (SSTATE_MISS, 'Missing'), # it is a miss
+        (SSTATE_FAILED, 'Failed'), # there was a pkg, but the script failed
+        (SSTATE_RESTORED, 'Restored'), # succesfully restored
+    )
+
+    CODING_NA = 0
+    CODING_NOEXEC = 1
+    CODING_PYTHON = 2
+    CODING_SHELL = 3
+
+    TASK_CODING = (
+        (CODING_NA, 'N/A'),
+        (CODING_NOEXEC, 'NoExec'),
+        (CODING_PYTHON, 'Python'),
+        (CODING_SHELL, 'Shell'),
+    )
+
+    OUTCOME_SUCCESS = 0
+    OUTCOME_COVERED = 1
+    OUTCOME_SSTATE = 2
+    OUTCOME_EXISTING = 3
+    OUTCOME_FAILED = 4
+    OUTCOME_NA = 5
+
+    TASK_OUTCOME = (
+        (OUTCOME_SUCCESS, 'Succeeded'),
+        (OUTCOME_COVERED, 'Covered'),
+        (OUTCOME_SSTATE, 'Sstate'),
+        (OUTCOME_EXISTING, 'Existing'),
+        (OUTCOME_FAILED, 'Failed'),
+        (OUTCOME_NA, 'Not Available'),
+    )
+
+    build = models.ForeignKey(Build, related_name='task_build')
+    order = models.IntegerField(null=True)
+    task_executed = models.BooleanField(default=False) # True means Executed, False means Prebuilt
+    outcome = models.IntegerField(choices=TASK_OUTCOME, default=OUTCOME_NA)
+    sstate_checksum = models.CharField(max_length=100, blank=True)
+    path_to_sstate_obj = models.FilePathField(max_length=500, blank=True)
+    recipe = models.ForeignKey('Recipe', related_name='build_recipe')
+    task_name = models.CharField(max_length=100)
+    source_url = models.FilePathField(max_length=255, blank=True)
+    work_directory = models.FilePathField(max_length=255, blank=True)
+    script_type = models.IntegerField(choices=TASK_CODING, default=CODING_NA)
+    line_number = models.IntegerField(default=0)
+    disk_io = models.IntegerField(null=True)
+    cpu_usage = models.DecimalField(max_digits=6, decimal_places=2, null=True)
+    elapsed_time = models.CharField(max_length=50, default=0)
+    sstate_result = models.IntegerField(choices=SSTATE_RESULT, default=SSTATE_NA)
+    message = models.CharField(max_length=240)
+    logfile = models.FilePathField(max_length=255, blank=True)
+
+    class Meta:
+        ordering = ('order', 'recipe' ,)
+
+
+class Task_Dependency(models.Model):
+    task = models.ForeignKey(Task, related_name='task_dependencies_task')
+    depends_on = models.ForeignKey(Task, related_name='task_dependencies_depends')
+
+
+class Build_Package(models.Model):
+    build = models.ForeignKey('Build')
+    recipe = models.ForeignKey('Recipe', null=True)
+    name = models.CharField(max_length=100)
+    version = models.CharField(max_length=100, blank=True)
+    revision = models.CharField(max_length=32, blank=True)
+    summary = models.CharField(max_length=200, blank=True)
+    description = models.CharField(max_length=200, blank=True)
+    size = models.IntegerField(default=0)
+    section = models.CharField(max_length=80, blank=True)
+    license = models.CharField(max_length=80, blank=True)
+
+class Build_Package_Dependency(models.Model):
+    TYPE_RDEPENDS = 0
+    TYPE_RPROVIDES = 1
+    TYPE_RRECOMMENDS = 2
+    TYPE_RSUGGESTS = 3
+    TYPE_RREPLACES = 4
+    TYPE_RCONFLICTS = 5
+    DEPENDS_TYPE = (
+        (TYPE_RDEPENDS, "rdepends"),
+        (TYPE_RPROVIDES, "rprovides"),
+        (TYPE_RRECOMMENDS, "rrecommends"),
+        (TYPE_RSUGGESTS, "rsuggests"),
+        (TYPE_RREPLACES, "rreplaces"),
+        (TYPE_RCONFLICTS, "rconflicts"),
+    )
+    package = models.ForeignKey(Build_Package, related_name='bpackage_dependencies_package')
+    depends_on = models.CharField(max_length=100)   # soft dependency
+    dep_type = models.IntegerField(choices=DEPENDS_TYPE)
+
+
+class Target_Package(models.Model):
+    target = models.ForeignKey('Target')
+    recipe = models.ForeignKey('Recipe', null=True)
+    name = models.CharField(max_length=100)
+    version = models.CharField(max_length=100, blank=True)
+    size = models.IntegerField()
+
+
+class Target_Package_Dependency(models.Model):
+    TYPE_DEPENDS = 0
+    TYPE_RDEPENDS = 1
+    TYPE_RECOMMENDS = 2
+
+    DEPENDS_TYPE = (
+        (TYPE_DEPENDS, "depends"),
+        (TYPE_RDEPENDS, "rdepends"),
+        (TYPE_RECOMMENDS, "recommends"),
+    )
+    package = models.ForeignKey(Target_Package, related_name='tpackage_dependencies_package')
+    depends_on = models.ForeignKey(Target_Package, related_name='tpackage_dependencies_depends')
+    dep_type = models.IntegerField(choices=DEPENDS_TYPE)
+
+
+class Build_File(models.Model):
+    bpackage = models.ForeignKey(Build_Package, related_name='filelist_bpackage')
+    path = models.FilePathField(max_length=255, blank=True)
+    size = models.IntegerField()
+
+class Target_File(models.Model):
+    tpackage = models.ForeignKey(Target_Package, related_name='filelist_tpackage')
+    path = models.FilePathField(max_length=255, blank=True)
+    size = models.IntegerField()
+
+
+class Recipe(models.Model):
+    name = models.CharField(max_length=100, blank=True)
+    version = models.CharField(max_length=100, blank=True)
+    layer_version = models.ForeignKey('Layer_Version', related_name='recipe_layer_version')
+    summary = models.CharField(max_length=100, blank=True)
+    description = models.CharField(max_length=100, blank=True)
+    section = models.CharField(max_length=100, blank=True)
+    license = models.CharField(max_length=200, blank=True)
+    licensing_info = models.TextField(blank=True)
+    homepage = models.URLField(blank=True)
+    bugtracker = models.URLField(blank=True)
+    file_path = models.FilePathField(max_length=255)
+
+
+class Recipe_Dependency(models.Model):
+    TYPE_DEPENDS = 0
+    TYPE_RDEPENDS = 1
+
+    DEPENDS_TYPE = (
+        (TYPE_DEPENDS, "depends"),
+        (TYPE_RDEPENDS, "rdepends"),
+    )
+    recipe = models.ForeignKey(Recipe, related_name='r_dependencies_recipe')
+    depends_on = models.ForeignKey(Recipe, related_name='r_dependencies_depends')
+    dep_type = models.IntegerField(choices=DEPENDS_TYPE)
+
+class Layer(models.Model):
+    name = models.CharField(max_length=100)
+    local_path = models.FilePathField(max_length=255)
+    layer_index_url = models.URLField()
+
+
+class Layer_Version(models.Model):
+    layer = models.ForeignKey(Layer, related_name='layer_version_layer')
+    branch = models.CharField(max_length=50)
+    commit = models.CharField(max_length=100)
+    priority = models.IntegerField()
+
+
+class Variable(models.Model):
+    build = models.ForeignKey(Build, related_name='variable_build')
+    variable_name = models.CharField(max_length=100)
+    variable_value = models.TextField(blank=True)
+    changed = models.BooleanField(default=False)
+    human_readable_name = models.CharField(max_length=200)
+    description = models.TextField(blank=True)
+
+class VariableHistory(models.Model):
+    variable = models.ForeignKey(Variable)
+    file_name = models.FilePathField(max_length=255)
+    line_number = models.IntegerField(null=True)
+    operation = models.CharField(max_length=16)
+
+class LogMessage(models.Model):
+    INFO = 0
+    WARNING = 1
+    ERROR = 2
+
+    LOG_LEVEL = ( (INFO, "info"),
+            (WARNING, "warn"),
+            (ERROR, "error") )
+
+    build = models.ForeignKey(Build)
+    level = models.IntegerField(choices=LOG_LEVEL, default=INFO)
+    message=models.CharField(max_length=240)
+    pathname = models.FilePathField(max_length=255, blank=True)
+    lineno = models.IntegerField(null=True)

bitbake/lib/toaster/toastergui/templates/index.html

@@ -0,0 +1,13 @@
+<html>
+<head>
+	<title>GUI Page</title>
+</head>
+<body>
+{% load staticfiles %}
+
+<img src="{% static "/static/images/yocto.jpg" %}" alt="Yocto"/>
+
+This is your basic index page!
+</body>
+
+</html>

bitbake/lib/toaster/toastergui/urls.py

@@ -0,0 +1,27 @@
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2013        Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import os
+
+from django.conf import settings
+from django.conf.urls import patterns, include, url
+
+
+urlpatterns = patterns('toastergui.views',
+        url(r'^$', 'guihome', name='guihome'),
+)

bitbake/lib/toaster/toastergui/views.py

@@ -0,0 +1,26 @@
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2013        Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from django.shortcuts import render
+from orm.models import Build, Task
+
+
+def guihome(request):
+    template = 'index.html'
+
+    return render(request, template)

bitbake/lib/toaster/toastermain/settings.py

@@ -0,0 +1,191 @@
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2013        Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+# Django settings for Toaster project.
+
+DEBUG = True
+TEMPLATE_DEBUG = DEBUG
+
+ADMINS = (
+    # ('Your Name', 'your_email@example.com'),
+)
+
+MANAGERS = ADMINS
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
+        'NAME': 'toaster.sqlite',                      # Or path to database file if using sqlite3.
+        'USER': '',
+        'PASSWORD': '',
+        'HOST': '127.0.0.1',                      # Empty for localhost through domain sockets or '127.0.0.1' for localhost through TCP.
+        'PORT': '3306',                      # Set to empty string for default.
+    }
+}
+
+# Hosts/domain names that are valid for this site; required if DEBUG is False
+# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
+ALLOWED_HOSTS = []
+
+# Local time zone for this installation. Choices can be found here:
+# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
+# although not all choices may be available on all operating systems.
+# In a Windows environment this must be set to your system time zone.
+
+# Always use local computer's time zone
+import time
+TIME_ZONE = time.tzname[0]
+
+# Language code for this installation. All choices can be found here:
+# http://www.i18nguy.com/unicode/language-identifiers.html
+LANGUAGE_CODE = 'en-us'
+
+SITE_ID = 1
+
+# If you set this to False, Django will make some optimizations so as not
+# to load the internationalization machinery.
+USE_I18N = True
+
+# If you set this to False, Django will not format dates, numbers and
+# calendars according to the current locale.
+USE_L10N = True
+
+# If you set this to False, Django will not use timezone-aware datetimes.
+USE_TZ = True
+
+# Absolute filesystem path to the directory that will hold user-uploaded files.
+# Example: "/var/www/example.com/media/"
+MEDIA_ROOT = ''
+
+# URL that handles the media served from MEDIA_ROOT. Make sure to use a
+# trailing slash.
+# Examples: "http://example.com/media/", "http://media.example.com/"
+MEDIA_URL = ''
+
+# Absolute path to the directory static files should be collected to.
+# Don't put anything in this directory yourself; store your static files
+# in apps' "static/" subdirectories and in STATICFILES_DIRS.
+# Example: "/var/www/example.com/static/"
+STATIC_ROOT = ''
+
+# URL prefix for static files.
+# Example: "http://example.com/static/", "http://static.example.com/"
+STATIC_URL = '/static/'
+
+# Additional locations of static files
+STATICFILES_DIRS = (
+    # Put strings here, like "/home/html/static" or "C:/www/django/static".
+    # Always use forward slashes, even on Windows.
+    # Don't forget to use absolute paths, not relative paths.
+)
+
+# List of finder classes that know how to find static files in
+# various locations.
+STATICFILES_FINDERS = (
+    'django.contrib.staticfiles.finders.FileSystemFinder',
+    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
+#    'django.contrib.staticfiles.finders.DefaultStorageFinder',
+)
+
+# Make this unique, and don't share it with anybody.
+SECRET_KEY = 'NOT_SUITABLE_FOR_HOSTED_DEPLOYMENT'
+
+# List of callables that know how to import templates from various sources.
+TEMPLATE_LOADERS = (
+    'django.template.loaders.filesystem.Loader',
+    'django.template.loaders.app_directories.Loader',
+#     'django.template.loaders.eggs.Loader',
+)
+
+MIDDLEWARE_CLASSES = (
+    'django.middleware.common.CommonMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    # Uncomment the next line for simple clickjacking protection:
+    # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
+)
+
+ROOT_URLCONF = 'toastermain.urls'
+
+# Python dotted path to the WSGI application used by Django's runserver.
+WSGI_APPLICATION = 'toastermain.wsgi.application'
+
+TEMPLATE_DIRS = (
+    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
+    # Always use forward slashes, even on Windows.
+    # Don't forget to use absolute paths, not relative paths.
+)
+
+INSTALLED_APPS = (
+    #'django.contrib.auth',
+    #'django.contrib.contenttypes',
+    #'django.contrib.sessions',
+    #'django.contrib.sites',
+    #'django.contrib.messages',
+    'django.contrib.staticfiles',
+    # Uncomment the next line to enable the admin:
+    # 'django.contrib.admin',
+    # Uncomment the next line to enable admin documentation:
+    # 'django.contrib.admindocs',
+    'orm',
+    'toastermain',
+    'bldviewer',
+    'toastergui',
+)
+
+# A sample logging configuration. The only tangible logging
+# performed by this configuration is to send an email to
+# the site admins on every HTTP 500 error when DEBUG=False.
+# See http://docs.djangoproject.com/en/dev/topics/logging for
+# more details on how to customize your logging configuration.
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'filters': {
+        'require_debug_false': {
+            '()': 'django.utils.log.RequireDebugFalse'
+        }
+    },
+    'handlers': {
+        'mail_admins': {
+            'level': 'ERROR',
+            'filters': ['require_debug_false'],
+            'class': 'django.utils.log.AdminEmailHandler'
+        }
+    },
+    'loggers': {
+        'django.request': {
+            'handlers': ['mail_admins'],
+            'level': 'ERROR',
+            'propagate': True,
+        },
+    }
+}
+
+# If we're using sqlite, we need to tweak the performance a bit
+from django.db.backends.signals import connection_created
+def activate_synchronous_off(sender, connection, **kwargs):
+    if connection.vendor == 'sqlite':
+        cursor = connection.cursor()
+        cursor.execute('PRAGMA synchronous = 0;')
+connection_created.connect(activate_synchronous_off)
+#
+
+

bitbake/lib/toaster/toastermain/urls.py

@@ -0,0 +1,41 @@
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2013        Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from django.conf.urls import patterns, include, url
+from django.views.generic.simple import redirect_to
+from django.views.decorators.cache import never_cache
+
+
+# Uncomment the next two lines to enable the admin:
+# from django.contrib import admin
+# admin.autodiscover()
+
+urlpatterns = patterns('',
+    url(r'^simple/', include('bldviewer.urls')),
+    url(r'^api/1.0/', include('bldviewer.api')),
+    url(r'^gui/', include('toastergui.urls')),
+    url(r'^$', never_cache(redirect_to), {'url': '/simple/'}),
+    # Examples:
+    # url(r'^toaster/', include('toaster.foo.urls')),
+
+    # Uncomment the admin/doc line below to enable admin documentation:
+    # url(r'^admin/doc/', include('django.contrib.admindocs.urls')),
+
+    # Uncomment the next line to enable the admin:
+    # url(r'^admin/', include(admin.site.urls)),
+)

bitbake/lib/toaster/toastermain/wsgi.py

@@ -0,0 +1,32 @@
+"""
+WSGI config for Toaster project.
+
+This module contains the WSGI application used by Django's development server
+and any production WSGI deployments. It should expose a module-level variable
+named ``application``. Django's ``runserver`` and ``runfcgi`` commands discover
+this application via the ``WSGI_APPLICATION`` setting.
+
+Usually you will have the standard Django WSGI application here, but it also
+might make sense to replace the whole Django WSGI application with a custom one
+that later delegates to the Django one. For example, you could introduce WSGI
+middleware here, or combine a Django application with an application of another
+framework.
+
+"""
+import os
+
+# We defer to a DJANGO_SETTINGS_MODULE already in the environment. This breaks
+# if running multiple sites in the same mod_wsgi process. To fix this, use
+# mod_wsgi daemon mode with each site in its own daemon process, or use
+# os.environ["DJANGO_SETTINGS_MODULE"] = "Toaster.settings"
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "toastermain.settings")
+
+# This application object is used by any WSGI server configured to use this
+# file. This includes Django's development server, if the WSGI_APPLICATION
+# setting points here.
+from django.core.wsgi import get_wsgi_application
+application = get_wsgi_application()
+
+# Apply WSGI middleware here.
+# from helloworld.wsgi import HelloWorldApplication
+# application = HelloWorldApplication(application)

documentation/adt-manual/adt-command.xml

@@ -8,23 +8,32 @@
     <para>
         Recall that earlier the manual discussed how to use an existing toolchain
         tarball that had been installed into the default installation
-        directory, <filename>/opt/poky</filename>, which is outside of the
+        directory, <filename>/opt/poky/&DISTRO;</filename>, which is outside of the
         <ulink url='&YOCTO_DOCS_DEV_URL;#build-directory'>Build Directory</ulink>
         (see the section "<link linkend='using-an-existing-toolchain-tarball'>Using a Cross-Toolchain Tarball)</link>".
         And, that sourcing your architecture-specific environment setup script
         initializes a suitable cross-toolchain development environment.
-        During the setup, locations for the compiler, QEMU scripts, QEMU binary,
+    </para>
+
+    <para>
+        During this setup, locations for the compiler, QEMU scripts, QEMU binary,
         a special version of <filename>pkgconfig</filename> and other useful
         utilities are added to the <filename>PATH</filename> variable.
-        Variables to assist <filename>pkgconfig</filename> and <filename>autotools</filename>
-        are also defined so that,
-        for example, <filename>configure.sh</filename> can find pre-generated
-        test results for tests that need target hardware on which to run.
-        These conditions allow you to easily use the toolchain outside of the
-        OpenEmbedded build environment on both autotools-based projects and
-        Makefile-based projects.
+        Also, variables to assist
+        <filename>pkgconfig</filename> and <filename>autotools</filename>
+        are also defined so that, for example, <filename>configure.sh</filename>
+        can find pre-generated test results for tests that need target hardware
+        on which to run.
     </para>
 
+    <para>
+        Collectively, these conditions allow you to easily use the toolchain
+        outside of the OpenEmbedded build environment on both autotools-based
+        projects and Makefile-based projects.
+        This chapter provides information for both these types of projects.
+    </para>
+
+
 <section id='autotools-based-projects'>
 <title>Autotools-Based Projects</title>
 
@@ -179,7 +188,7 @@
                 If <filename>configure</filename> script results in problems recognizing the
                 <filename>--with-libtool-sysroot=&lt;sysroot-dir&gt;</filename> option,
                 regenerate the script to enable the support by doing the following and then
-                re-running the script:
+                run the script again:
                 <literallayout class='monospaced'>
      $ libtoolize --automake
      $ aclocal -I ${OECORE_NATIVE_SYSROOT}/usr/share/aclocal \

documentation/adt-manual/adt-intro.xml

@@ -22,8 +22,8 @@
     and an introduction to the <trademark class='trade'>Eclipse</trademark> IDE
     Yocto Plug-in.
     <note>
-        The ADT is distribution-neutral and does not require the Yocto 
-        Project reference distribution, which is called Poky.  
+        The ADT is distribution-neutral and does not require the Yocto
+        Project reference distribution, which is called Poky.
         This manual, however, uses examples that use the Poky distribution.
     </note>
 </para>
@@ -43,7 +43,7 @@
         <itemizedlist>
             <listitem><para>An architecture-specific cross-toolchain and matching
                 sysroot both built by the OpenEmbedded build system.
-                The toolchain and sysroot are based on a 
+                The toolchain and sysroot are based on a
                 <ulink url='&YOCTO_DOCS_DEV_URL;#metadata'>Metadata</ulink>
                 configuration and extensions,
                 which allows you to cross-develop on the host machine for the target hardware.
@@ -149,8 +149,7 @@
                     that causes skips in audio,
                     stutters in your desktop experience, or situations that overload your server
                     even when you have plenty of CPU power left.
-                    You can find out more about LatencyTOP at
-                    <ulink url='https://latencytop.org/'></ulink>.</para></listitem>
+                    </para></listitem>
                 <listitem><para><emphasis>PowerTOP:</emphasis> Helps you determine what
                     software is using the most power.
                     You can find out more about PowerTOP at

documentation/adt-manual/adt-manual.xml

@@ -66,6 +66,16 @@
                 <date>October 2013</date>
                 <revremark>Released with the Yocto Project 1.5 Release.</revremark>
             </revision>
+            <revision>
+                <revnumber>1.5.1</revnumber>
+                <date>Sometime in 2013</date>
+                <revremark>Released with the Yocto Project 1.5.1 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>1.6</revnumber>
+                <date>Sometime 2014</date>
+                <revremark>Released with the Yocto Project 1.6 Release.</revremark>
+            </revision>
        </revhistory>
 
     <copyright>
@@ -79,11 +89,10 @@
         the terms of the <ulink type="http" url="http://creativecommons.org/licenses/by-sa/2.0/uk/">Creative Commons Attribution-Share Alike 2.0 UK: England &amp; Wales</ulink> as published by Creative Commons.
       </para>
       <note>
-          Due to production processes, there could be differences between the Yocto Project
-          documentation bundled in the release tarball and the
-          <ulink url='&YOCTO_DOCS_ADT_URL;'>Yocto Project Application Developer's Guide</ulink> on
-          the <ulink url='&YOCTO_HOME_URL;'>Yocto Project</ulink> website.
-          For the latest version of this manual, see the manual on the website.
+          For the latest version of this manual associated with this
+          Yocto Project release, see the
+          <ulink url='&YOCTO_DOCS_ADT_URL;'>Yocto Project Application Developer's Guide</ulink>
+          from the Yocto Project website.
       </note>
 
     </legalnotice>

documentation/adt-manual/adt-prepare.xml

@@ -80,39 +80,48 @@
 
             <para>
                 The ADT Installer is contained in the ADT Installer tarball.
-                You can download the tarball into any directory from the
-                <ulink url='&YOCTO_DL_URL;/releases'>Index of Releases</ulink>, specifically
-                at
-                <ulink url='&YOCTO_ADTINSTALLER_DL_URL;'></ulink>.
-                Or, you can use BitBake to generate the tarball inside the existing
-                <ulink url='&YOCTO_DOCS_DEV_URL;#build-directory'>Build Directory</ulink>.
-            </para>
-
-            <para>
-                If you use BitBake to generate the ADT Installer tarball, you must
-                <filename>source</filename> the environment setup script
-                (<ulink url='&YOCTO_DOCS_REF_URL;#structure-core-script'><filename>&OE_INIT_FILE;</filename></ulink>
-                or
-                <ulink url='&YOCTO_DOCS_REF_URL;#structure-memres-core-script'><filename>oe-init-build-env-memres</filename></ulink>)
-                located in the Source Directory before running the
-                BitBake command that creates the tarball.
-            </para>
-
-            <para>
-                The following example commands download the Poky tarball, set up the
-                <ulink url='&YOCTO_DOCS_DEV_URL;#source-directory'>Source Directory</ulink>,
-                set up the environment while also creating the default Build Directory,
-                and run the BitBake command that results in the tarball
-                <filename>~/yocto-project/build/tmp/deploy/sdk/adt_installer.tar.bz2</filename>:
-                <literallayout class='monospaced'>
+                You can get the tarball using either of these methods:
+                <itemizedlist>
+                    <listitem><para><emphasis>Download the Tarball:</emphasis>
+                        You can download the tarball from
+                        <ulink url='&YOCTO_ADTINSTALLER_DL_URL;'></ulink> into
+                        any directory.</para></listitem>
+                    <listitem><para><emphasis>Build the Tarball:</emphasis>
+                        You can use BitBake to generate the tarball inside an
+                        existing
+                        <ulink url='&YOCTO_DOCS_DEV_URL;#build-directory'>Build Directory</ulink>.
+                        </para>
+                        <para>If you use BitBake to generate the ADT Installer
+                        tarball, you must <filename>source</filename> the
+                        environment setup script
+                        (<ulink url='&YOCTO_DOCS_REF_URL;#structure-core-script'><filename>&OE_INIT_FILE;</filename></ulink>
+                        or
+                        <ulink url='&YOCTO_DOCS_REF_URL;#structure-memres-core-script'><filename>oe-init-build-env-memres</filename></ulink>)
+                        located in the Source Directory before running the
+                        BitBake command that creates the tarball.</para>
+                        <para>The following example commands establish
+                        the
+                        <ulink url='&YOCTO_DOCS_DEV_URL;#source-directory'>Source Directory</ulink>,
+                        check out the current release branch, set up the
+                        build environment while also creating the default
+                        Build Directory, and run the BitBake command that
+                        results in the tarball
+                        <filename>poky/build/tmp/deploy/sdk/adt_installer.tar.bz2</filename>:
+                        <note>
+                            Before using BitBake to build the ADT tarball, be
+                            sure to make sure your
+                            <filename>local.conf</filename> file is properly
+                            configured.
+                        </note>
+                        <literallayout class='monospaced'>
      $ cd ~
-     $ mkdir yocto-project
-     $ cd yocto-project
-     $ wget &YOCTO_RELEASE_DL_URL;/&YOCTO_POKY_TARBALL;
-     $ tar xjf &YOCTO_POKY_TARBALL;
-     $ source &OE_INIT_PATH;
+     $ git clone git://git.yoctoproject.org/poky
+     $ cd poky
+     $ git checkout -b &DISTRO_NAME; origin/&DISTRO_NAME;
+     $ source &OE_INIT_FILE;
      $ bitbake adt-installer
-                </literallayout>
+                        </literallayout></para></listitem>
+                </itemizedlist>
             </para>
         </section>
 
@@ -127,7 +136,7 @@
                 a top-level directory named <filename>adt-installer</filename>:
                 <literallayout class='monospaced'>
      $ cd ~
-     $ cp ~/poky/build/tmp/deploy/sdk/adt_installer.tar.bz2 $HOME
+     $ cp poky/build/tmp/deploy/sdk/adt_installer.tar.bz2 $HOME
      $ tar -xjf adt_installer.tar.bz2
                 </literallayout>
                 Unpacking it creates the directory <filename>adt-installer</filename>,
@@ -197,7 +206,7 @@
                 When you run the installer, the environment must use a
                 host <filename>gcc</filename>:
                 <literallayout class='monospaced'>
-     $ cd ~/adt-installer
+     $ cd adt-installer
      $ ./adt_installer
                 </literallayout>
                 Once the installer begins to run, you are asked to enter the
@@ -268,7 +277,7 @@
                             target, go into the <filename>x86_64</filename>
                             folder and download the following installer:
                             <literallayout class='monospaced'>
-     poky-eglibc-x86_64-core-image-sato-i586-&DISTRO;.sh
+     poky-eglibc-x86_64-core-image-sato-i586-toolchain-&DISTRO;.sh
                             </literallayout></para></listitem>
                         <listitem><para>Build your own toolchain installer.
                             For cases where you cannot use an installer
@@ -279,27 +288,32 @@
                     </itemizedlist></para></listitem>
                 <listitem><para>Once you have the installer, run it to install
                     the toolchain.
-                    You must change the permissions on the toolchain installer
-                    script so that it is executable.</para>
+                    <note>
+                        You must change the permissions on the toolchain
+                        installer script so that it is executable.
+                    </note></para>
                     <para>The following command shows how to run the installer
                     given a toolchain tarball for a 64-bit x86 development host
                     system and a 32-bit x86 target architecture.
                     The example assumes the toolchain installer is located
                     in <filename>~/Downloads/</filename>.
                     <literallayout class='monospaced'>
-     $ ~/Downloads/poky-eglibc-x86_64-core-image-sato-i586-&DISTRO;.sh
+     $ ~/Downloads/poky-eglibc-x86_64-core-image-sato-i586-toolchain-&DISTRO;.sh
                     </literallayout>
-                    <note>
-                        If you do not have write permissions for the directory
-                        into which you are installing the toolchain, the
-                        toolchain installer notifies you and exits.
-                        Be sure you have write permissions in the directory and
-                        run the installer again.
-                    </note>
-                    Once the tarball is expanded, the cross-toolchain is
+                    The first thing the installer prompts you for is the
+                    directory into which you want to install the toolchain.
+                    The default directory used is
+                    <filename>/opt/poky/&DISTRO;</filename>.
+                    If you do not have write permissions for the directory
+                    into which you are installing the toolchain, the
+                    toolchain installer notifies you and exits.
+                    Be sure you have write permissions in the directory and
+                    run the installer again.</para>
+                    <para>When the script finishes, the cross-toolchain is
                     installed.
                     You will notice environment setup files for the
-                    cross-toolchain in the directory.</para></listitem>
+                    cross-toolchain in the installation directory.
+                    </para></listitem>
             </orderedlist>
         </para>
     </section>
@@ -320,37 +334,43 @@
         <para>
             Follow these steps to generate the toolchain into the Build Directory:
             <orderedlist>
-                <listitem><para>Source the environment setup script
-                    <filename>&OE_INIT_FILE;</filename> located in the
+                <listitem><para><emphasis>Set up the Build Environment:</emphasis>
+                    Source the OpenEmbedded build environment setup
+                    script (i.e.
+                    <ulink url='&YOCTO_DOCS_REF_URL;#structure-core-script'><filename>&OE_INIT_FILE;</filename></ulink>
+                    or
+                    <ulink url='&YOCTO_DOCS_REF_URL;#structure-memres-core-script'><filename>oe-init-build-env-memres</filename></ulink>)
+                    located in the
                     <ulink url='&YOCTO_DOCS_DEV_URL;#source-directory'>Source Directory</ulink>.
                     </para></listitem>
-                <listitem><para>At this point, you should be sure that the
+                <listitem><para><emphasis>Check your Local Configuration File:</emphasis>
+                    At this point, you should be sure that the
                     <ulink url='&YOCTO_DOCS_REF_URL;#var-MACHINE'><filename>MACHINE</filename></ulink> variable
                     in the <filename>local.conf</filename> file found in the
                     <filename>conf</filename> directory of the Build Directory
                     is set for the target architecture.
-                    Comments within the <filename>local.conf</filename> file list the values you
-                    can use for the <filename>MACHINE</filename> variable.
-                    <note>You can populate the Build Directory with the cross-toolchains for more
-                        than a single architecture.
-                        You just need to edit the <filename>MACHINE</filename> variable in the
-                        <filename>local.conf</filename> file and re-run the BitBake
-                        command.</note></para></listitem>
-                <listitem><para>Run <filename>bitbake meta-ide-support</filename> to complete the
-                    cross-toolchain generation.
-                    <note>If you change out of your working directory after you
-                        <filename>source</filename> the environment setup script and before you run
-                        the BitBake command, the command might not work.
-                        Be sure to run the BitBake command immediately
-                        after checking or editing the <filename>local.conf</filename> but without
-                        changing out of your working directory.</note>
-                    Once the BitBake command finishes,
-                    the cross-toolchain is generated and populated within the Build Directory.
-                    You will notice environment setup files for the cross-toolchain in the
-                    Build Directory in the <filename>tmp</filename> directory.
-                    Setup script filenames contain the strings <filename>environment-setup</filename>.</para>
-                    <para>Be aware that when you use this method to install the toolchain you still need
-                    to separately extract and install the sysroot filesystem.
+                    Comments within the <filename>local.conf</filename> file
+                    list the values you can use for the
+                    <filename>MACHINE</filename> variable.
+                    <note>
+                        You can populate the Build Directory with the
+                        cross-toolchains for more than a single architecture.
+                        You just need to edit the <filename>MACHINE</filename>
+                        variable in the <filename>local.conf</filename> file and
+                        re-run the BitBake command.
+                    </note></para></listitem>
+                <listitem><para><emphasis>Generate the Cross-Toolchain:</emphasis>
+                    Run <filename>bitbake meta-ide-support</filename> to
+                    complete the cross-toolchain generation.
+                    Once the BitBake command finishes, the cross-toolchain is
+                    generated and populated within the Build Directory.
+                    You will notice environment setup files for the
+                    cross-toolchain that contain the string
+                    "<filename>environment-setup</filename>" in the
+                    Build Directory's <filename>tmp</filename> folder.</para>
+                    <para>Be aware that when you use this method to install the
+                    toolchain, you still need to separately extract and install
+                    the sysroot filesystem.
                     For information on how to do this, see the
                     "<link linkend='extracting-the-root-filesystem'>Extracting the Root Filesystem</link>" section.
                     </para></listitem>
@@ -437,8 +457,8 @@
             application from within the
             Eclipse IDE, you must have an image that contains the Yocto Target Communication
             Framework (TCF) agent (<filename>tcf-agent</filename>).
-            By default, the Yocto Project provides only one type pre-built image that contains the
-            <filename>tcf-agent</filename>.
+            By default, the Yocto Project provides only one type of pre-built
+            image that contains the <filename>tcf-agent</filename>.
             And, those images are SDK (e.g.<filename>core-image-sato-sdk</filename>).
         </para>
 
@@ -467,13 +487,16 @@
                             the following commands:
                             <literallayout class='monospaced'>
      $ git clone http://git.eclipse.org/gitroot/tcf/org.eclipse.tcf.agent.git
-     $ cd agent
+     $ cd org.eclipse.tcf.agent/agent
                             </literallayout></para></listitem>
-                        <listitem><para>Modify the <filename>Makefile.inc</filename> file
+                        <listitem><para>Locate the
+                            <filename>Makefile.inc</filename> file inside the
+                            <filename>agent</filename> folder and modify it
                             for the cross-compilation environment by setting the
                             <filename>OPSYS</filename> and
                             <ulink url='&YOCTO_DOCS_REF_URL;#var-MACHINE'><filename>MACHINE</filename></ulink>
-                            variables according to your target.</para></listitem>
+                            variables according to your target.
+                            </para></listitem>
                         <listitem><para>Use the cross-development tools to build the
                             <filename>tcf-agent</filename>.
                             Before you "Make" the file, be sure your cross-tools are set up first.
@@ -493,32 +516,63 @@
         <title>Extracting the Root Filesystem</title>
 
         <para>
-            You must extract the root filesystem if you want to boot the image using NFS
-            or you want to use the root filesystem as the target sysroot.
-            For example, the Eclipse IDE environment with the Eclipse Yocto Plug-in installed allows you
-            to use QEMU to boot under NFS.
-            Another example is if you want to develop your target application using the
-            root filesystem as the target sysroot.
+            If you install your toolchain by hand or build it using BitBake and
+            you need a root filesystem, you need to extract it separately.
+            If you use the ADT Installer to install the ADT, the root
+            filesystem is automatically extracted and installed.
+        </para>
+
+        <para>
+            Here are some cases where you need to extract the root filesystem:
+            <itemizedlist>
+                <listitem><para>You want to boot the image using NFS.
+                    </para></listitem>
+                <listitem><para>You want to use the root filesystem as the
+                    target sysroot.
+                    For example, the Eclipse IDE environment with the Eclipse
+                    Yocto Plug-in installed allows you to use QEMU to boot
+                    under NFS.</para></listitem>
+                <listitem><para>You want to develop your target application
+                    using the root filesystem as the target sysroot.
+                    </para></listitem>
+            </itemizedlist>
         </para>
 
         <para>
             To extract the root filesystem, first <filename>source</filename>
-            the cross-development environment setup script and then
-            use the <filename>runqemu-extract-sdk</filename> command on the
+            the cross-development environment setup script.
+            If you built the toolchain in the Build Directory, you will find
+            the toolchain environment script in the
+            <filename>tmp</filename> directory.
+            If you installed the toolchain by hand, the environment setup
+            script is located in <filename>/opt/poky/&DISTRO;</filename>.
+        </para>
+
+        <para>
+            After sourcing the environment script, use the
+            <filename>runqemu-extract-sdk</filename> command and provide the
             filesystem image.
-            For example, the following commands set up the environment and then extract
-            the root filesystem from a previously built filesystem image tarball named
-            <filename>core-image-sato-sdk-qemux86-2011091411831.rootfs.tar.bz2</filename>.
-            The example extracts the root filesystem into the <filename>$HOME/qemux86-sato</filename>
-            directory:
+        </para>
+
+        <para>
+            Following is an example.
+            The second command sets up the environment.
+            In this case, the setup script is located in the
+            <filename>/opt/poky/&DISTRO;</filename> directory.
+            The third command extracts the root filesystem from a previously
+            built filesystem that is located in the
+            <filename>~/Downloads</filename> directory.
+            Furthermore, this command extracts the root filesystem into the
+            <filename>qemux86-sato</filename> directory:
             <literallayout class='monospaced'>
-     $ source $HOME/toolchain_dir/environment-setup-i586-poky-linux
+     $ cd ~
+     $ source /opt/poky/&DISTRO;/environment-setup-i586-poky-linux
      $ runqemu-extract-sdk \
-        ~Downloads/core-image-sato-sdk-qemux86-2011091411831.rootfs.tar.bz2 \
+        ~/Downloads/core-image-sato-sdk-qemux86-2011091411831.rootfs.tar.bz2 \
         $HOME/qemux86-sato
             </literallayout>
-            In this case, you could now point to the target sysroot at
-            <filename>$HOME/qemux86-sato</filename>.
+            You could now point to the target sysroot at
+            <filename>qemux86-sato</filename>.
         </para>
     </section>
 </section>
@@ -528,30 +582,26 @@
 
     <para>
         As an alternative to locating and downloading a toolchain installer,
-        you can build the toolchain installer if you have a
-        <ulink url='&YOCTO_DOCS_DEV_URL;#build-directory'>Build Directory</ulink>.
+        you can build the toolchain installer one of two ways if you have a
+        <ulink url='&YOCTO_DOCS_DEV_URL;#build-directory'>Build Directory</ulink>:
+        <itemizedlist>
+            <listitem><para>Use <filename>bitbake meta-toolchain</filename>.
+                This method requires you to still install the target
+                sysroot by installing and extracting it separately.
+                For information on how to install the sysroot, see the
+                "<link linkend='extracting-the-root-filesystem'>Extracting the Root Filesystem</link>"
+                section.</para></listitem>
+            <listitem><para>Use
+                <filename>bitbake image -c populate_sdk</filename>.
+                This method has significant advantages over the previous method
+                because it results in a toolchain installer that contains the
+                sysroot that matches your target root filesystem.
+                </para></listitem>
+        </itemizedlist>
     </para>
 
     <para>
-        You can build the toolchain
-        installer using <filename>bitbake meta-toolchain</filename>.
-        This method requires you to still install the target
-        sysroot by installing and extracting it separately.
-        For information on how to install the sysroot, see the
-        "<link linkend='extracting-the-root-filesystem'>Extracting the Root Filesystem</link>" section.
-    </para>
-
-    <para>
-        A final method of building the toolchain installer exists that has
-        significant advantages over the previous method.
-        This method results in a toolchain installer that contains the sysroot
-        that matches your target root filesystem.
-        To build this installer, use the
-        <filename>bitbake image -c populate_sdk</filename> command.
-    </para>
-
-    <para>
-         Remember, before using any <filename>bitbake</filename> command, you
+         Remember, before using any BitBake command, you
          must source the build environment setup script
          (i.e.
          <ulink url='&YOCTO_DOCS_REF_URL;#structure-core-script'><filename>&OE_INIT_FILE;</filename></ulink>
@@ -567,12 +617,27 @@
          variable is correctly set if you are building a toolchain designed to
          run on an architecture that differs from your current development host
          machine (i.e. the build machine).
-     </para>
+    </para>
 
-     <para>
-         When the BitBake command completes, the toolchain installer will be in
-         <filename>tmp/deploy/sdk</filename> in the Build Directory.
-     </para>
+    <para>
+        When the BitBake command completes, the toolchain installer will be in
+        <filename>tmp/deploy/sdk</filename> in the Build Directory.
+        <note>
+            By default, this toolchain does not build static binaries.
+            If you want to use the toolchain to build these types of libraries,
+            you need to be sure your image has the appropriate static
+            development libraries.
+            Use the
+            <ulink url='&YOCTO_DOCS_REF_URL;#var-IMAGE_INSTALL'><filename>IMAGE_INSTALL</filename></ulink>
+            variable inside your <filename>local.conf</filename> file to
+            install the appropriate library packages.
+            Following is an example using <filename>eglibc</filename> static
+            development libraries:
+            <literallayout class='monospaced'>
+     IMAGE_INSTALL_append = " eglibc-staticdev"
+            </literallayout>
+        </note>
+    </para>
 </section>
 
 </chapter>

documentation/bsp-guide/bsp-guide.xml

@@ -78,6 +78,16 @@
                 <date>October 2013</date>
                 <revremark>Released with the Yocto Project 1.5 Release.</revremark>
             </revision>
+            <revision>
+                <revnumber>1.5.1</revnumber>
+                <date>Sometime in 2013</date>
+                <revremark>Released with the Yocto Project 1.5.1 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>1.6</revnumber>
+                <date>Sometime 2014</date>
+                <revremark>Released with the Yocto Project 1.6 Release.</revremark>
+            </revision>
         </revhistory>
 
     <copyright>
@@ -91,11 +101,10 @@
         the terms of the <ulink type="http" url="http://creativecommons.org/licenses/by-nc-sa/2.0/uk/">Creative Commons Attribution-Non-Commercial-Share Alike 2.0 UK: England &amp; Wales</ulink> as published by Creative Commons.
       </para>
       <note>
-          Due to production processes, there could be differences between the Yocto Project
-          documentation bundled in the release tarball and the
-          <ulink url='&YOCTO_DOCS_BSP_URL;'>Yocto Project Board Support Package (BSP) Developer's Guide</ulink> on
-          the <ulink url='&YOCTO_HOME_URL;'>Yocto Project</ulink> website.
-          For the latest version of this manual, see the manual on the website.
+          For the latest version of this manual associated with this
+          Yocto Project release, see the
+          <ulink url='&YOCTO_DOCS_BSP_URL;'>Yocto Project Board Support Package (BSP) Developer's Guide</ulink>
+          from the Yocto Project website.
       </note>
     </legalnotice>
 

documentation/bsp-guide/bsp.xml

@@ -394,12 +394,13 @@
 
             <para>
                 To use an include file, you simply include them in the machine configuration file.
-                For example, the Crown Bay BSP <filename>crownbay.conf</filename> has the
+                For example, the Crown Bay BSP <filename>crownbay.conf</filename> contains the
                 following statements:
                 <literallayout class='monospaced'>
      require conf/machine/include/tune-atom.inc
      require conf/machine/include/ia32-base.inc
      require conf/machine/include/meta-intel.inc
+     require conf/machine/include/meta-intel-emgd.inc
                 </literallayout>
             </para>
             </section>
@@ -420,7 +421,7 @@
                 <filename>formfactor_0.0.bbappend</filename> file, which is an
                 append file used to augment the recipe that starts the build.
                 Furthermore, there are machine-specific settings used during the
-                build that are defined by the <filename>machconfig</filename>.
+                build that are defined by the <filename>machconfig</filename> file.
                 In the Crown Bay example, two <filename>machconfig</filename> files
                 exist: one that supports the Intel® Embedded Media and Graphics
                 Driver (Intel® EMGD) and one that does not:
@@ -460,7 +461,7 @@
                 (VESA) graphics):
                 <literallayout class='monospaced'>
      meta-crownbay/recipes-graphics/xorg-xserver/xserver-xf86-config_0.1.bbappend
-     meta-crownbay/recipes-graphics/xorg-xserver/xserver-xf86-config/crownbay-noemgd/xorg.conf
+     meta-crownbay/recipes-graphics/xorg-xserver/xserver-xf86-config/crownbay/xorg.conf
                 </literallayout>
             </para>
             </section>
@@ -561,7 +562,7 @@
             <para>
                 For example, suppose you had some configuration options in a file called
                 <filename>network_configs.cfg</filename>.
-                You can place that file inside a directory named <filename>/linux-yocto</filename> and then add
+                You can place that file inside a directory named <filename>linux-yocto</filename> and then add
                 a <filename>SRC_URI</filename> statement such as the following to the append file.
                 When the OpenEmbedded build system builds the kernel, the configuration options are
                 picked up and applied.
@@ -747,7 +748,7 @@
                                 <listitem><para>Instructions on how to boot the BSP build from
                                     the BSP layer.</para></listitem>
                                 <listitem><para>Instructions on how to boot the binary images
-                                    contained in the <filename>/binary</filename> directory,
+                                    contained in the <filename>binary</filename> directory,
                                     if present.</para></listitem>
                                 <listitem><para>Information on any known bugs or issues that users
                                     should know about when either building or booting the BSP
@@ -758,7 +759,7 @@
                             <filename>meta-&lt;bsp_name&gt;</filename> directory.
                             This file specifies exactly where you can find the sources used to
                             generate the binary images contained in the
-                            <filename>/binary</filename> directory, if present.
+                            <filename>binary</filename> directory, if present.
                             See the
                             <ulink url='&YOCTO_GIT_URL;/cgit.cgi/meta-intel/tree/meta-fri2/README.sources'><filename>README.sources</filename></ulink>
                             file for the Fish River Island 2 BSP in the <filename>meta-fri2</filename> BSP layer

documentation/dev-manual/dev-manual-model.xml

@@ -155,10 +155,11 @@
                     For more information on BSP layers, see the
                     "<ulink url='&YOCTO_DOCS_BSP_URL;#bsp-layers'>BSP Layers</ulink>" section in the
                     Yocto Project Board Support Package (BSP) Developer's Guide.</para>
-                    <note>Four BSPs exist that are part of the
-                    Yocto Project release: <filename>genericx86</filename>, <filename>beagleboard</filename>,
+                    <note>Five BSPs exist that are part of the
+                    Yocto Project release: <filename>genericx86</filename>, <filename>genericx86-64</filename>,
+                    <filename>beagleboard</filename>,
                     <filename>mpc8315e</filename>, and <filename>routerstationpro</filename>.
-                    The recipes and configurations for these four BSPs are located and dispersed
+                    The recipes and configurations for these five BSPs are located and dispersed
                     within the <link linkend='source-directory'>Source Directory</link>.
                     On the other hand, BSP layers for Chief River, Crown Bay,
                     Crystal Forest, Emenlow, Fish River Island 2, Jasper Forest, N450, NUC DC3217IYE,
@@ -218,10 +219,10 @@
         <para>
             You can view a video presentation on "Building Custom Embedded Images with Yocto"
             at <ulink url='http://free-electrons.com/blog/elc-2011-videos'>Free Electrons</ulink>.
-            You can also find supplemental information in
+            You can also find supplemental information in the
             <ulink url='&YOCTO_DOCS_BSP_URL;'>
-            The Board Support Package (BSP) Development Guide</ulink>.
-            Finally, there is wiki page write up of the example also located
+            Yocto Project Board Support Package (BSP) Developer's Guide</ulink>.
+            Finally, there is a wiki page write up of the example also located
             <ulink url='&YOCTO_WIKI_URL;/wiki/Transcript:_creating_one_generic_Atom_BSP_from_another'>
             here</ulink> that you might find helpful.
        </para>
@@ -418,8 +419,9 @@
                         If you have never built the kernel you are interested in, you need to run
                         an initial build to establish local kernel source files.</para>
                         <para>If you are building an image for the first time, you need to get the build
-                        environment ready by sourcing
-                        the environment setup script.
+                        environment ready by sourcing an environment setup script
+                        (e.g. <filename>oe-init-build-env</filename> or
+                        <filename>oe-init-build-env-memres</filename>).
                         You also need to be sure two key configuration files
                         (<filename>local.conf</filename> and <filename>bblayers.conf</filename>)
                         are configured appropriately.</para>
@@ -1424,9 +1426,6 @@
                         "<ulink url='&YOCTO_DOCS_PROF_URL;#profile-manual-perf'>perf</ulink>"
                         section in the Yocto Project Profiling and Tracing
                         Manual.
-                        For information on LatencyTOP, see the
-                        <ulink url='https://latencytop.org/'>LatencyTOP</ulink>
-                        website.
                         </para></listitem>
                 </itemizedlist>
             </para>
@@ -1470,7 +1469,14 @@
                             the "Clone from Yocto Git Repository" box, which
                             would execute a <filename>git clone</filename>
                             command to get the project's Metadata files.
-                            </para></listitem>
+                            <note>
+                                Do not specify your BitBake Commander project
+                                location as your Eclipse workspace.
+                                Doing so causes an error indicating that the
+                                current project overlaps the location of
+                                another project.
+                                This error occurs even if no such project exits.
+                            </note></para></listitem>
                         <listitem><para>Select <filename>Finish</filename> to
                             create the project.</para></listitem>
                     </orderedlist>
@@ -1639,7 +1645,7 @@
             <filename>meta/conf/bitbake.conf</filename> configuration file in the
             <link linkend='source-directory'>Source Directory</link>:
             <literallayout class='monospaced'>
-     S = ${WORKDIR}/${BP}
+     S = "${WORKDIR}/${BP}"
             </literallayout>
             You should be aware that many recipes override the <filename>S</filename> variable.
             For example, recipes that fetch their source from Git usually set
@@ -1649,7 +1655,7 @@
                 <ulink url='&YOCTO_DOCS_REF_URL;#var-BP'><filename>BP</filename></ulink>
                 represents the base recipe name, which consists of the name and version:
                 <literallayout class='monospaced'>
-     BP = ${BPN}-${PV}
+     BP = "${BPN}-${PV}"
                 </literallayout>
             </note>
         </para>
@@ -1670,7 +1676,7 @@
             the following is the work directory for the <filename>acl</filename> recipe that
             creates the <filename>acl</filename> package:
             <literallayout class='monospaced'>
-     ~/poky/build/tmp/work/i586-poky-linux/acl/2.2.51-r3/
+     poky/build/tmp/work/i586-poky-linux/acl/2.2.51-r3/
             </literallayout>
         </para>
 
@@ -1686,8 +1692,8 @@
             for the <filename>acl</filename> package that is being
             built for a MIPS-based device:
             <literallayout class='monospaced'>
-     ~/poky/build/tmp/work/mips-poky-linux/acl/2.2.51-r2
-     ~/poky/build/tmp/work/mips-poky-linux/acl/2.2.51-r2/acl-2.2.51
+     poky/build/tmp/work/mips-poky-linux/acl/2.2.51-r2
+     poky/build/tmp/work/mips-poky-linux/acl/2.2.51-r2/acl-2.2.51
             </literallayout>
         </para>
 
@@ -1698,6 +1704,7 @@
             <ulink url='&YOCTO_DOCS_REF_URL;#var-TMPDIR'><filename>TMPDIR</filename></ulink>,
             <ulink url='&YOCTO_DOCS_REF_URL;#var-TOPDIR'><filename>TOPDIR</filename></ulink>,
             <ulink url='&YOCTO_DOCS_REF_URL;#var-PACKAGE_ARCH'><filename>PACKAGE_ARCH</filename></ulink>,
+            <ulink url='&YOCTO_DOCS_REF_URL;#var-MULTIMACH_TARGET_SYS'><filename>MULTIMACH_TARGET_SYS</filename></ulink>,
             <ulink url='&YOCTO_DOCS_REF_URL;#var-TARGET_OS'><filename>TARGET_OS</filename></ulink>,
             <ulink url='&YOCTO_DOCS_REF_URL;#var-PN'><filename>PN</filename></ulink>,
             <ulink url='&YOCTO_DOCS_REF_URL;#var-PV'><filename>PV</filename></ulink>,

documentation/dev-manual/dev-manual-newbie.xml

@@ -406,38 +406,34 @@
     <title>Yocto Project Source Repositories</title>
 
     <para>
-        The Yocto Project team maintains complete source repositories for all Yocto Project files
-        at <ulink url='&YOCTO_GIT_URL;/cgit/cgit.cgi'></ulink>.
-        This web-based source code browser is organized into categories by function such as
-        IDE Plugins, Matchbox, Poky, Yocto Linux Kernel, and so forth.
-        From the interface, you can click on any particular item in the "Name" column and
-        see the URL at the bottom of the page that you need to clone a Git repository for
-        that particular item.
-        Having a local Git repository of the Source Directory (poky) allows you to
-        make changes, contribute to the history, and ultimately enhance the Yocto Project's
-        tools, Board Support Packages, and so forth.
+        The Yocto Project team maintains complete source repositories for all
+        Yocto Project files at
+        <ulink url='&YOCTO_GIT_URL;/cgit/cgit.cgi'></ulink>.
+        This web-based source code browser is organized into categories by
+        function such as IDE Plugins, Matchbox, Poky, Yocto Linux Kernel, and
+        so forth.
+        From the interface, you can click on any particular item in the "Name"
+        column and see the URL at the bottom of the page that you need to clone
+        a Git repository for that particular item.
+        Having a local Git repository of the Source Directory (poky) allows
+        you to make changes, contribute to the history, and ultimately enhance
+        the Yocto Project's tools, Board Support Packages, and so forth.
     </para>
 
     <para>
-        Conversely, if you are a developer that is not interested in contributing back to the
-        Yocto Project, you have the ability to simply download and extract release tarballs
-        and use them within the Yocto Project environment.
-        All that is required is a particular release of the Yocto Project and
-        your application source code.
-    </para>
-
-    <para>
-        For any supported release of Yocto Project, you can go to the
+        For any supported release of Yocto Project, you can also go to the
         <ulink url='&YOCTO_HOME_URL;'>Yocto Project Website</ulink> and
-        select the "Downloads" tab and get a tarball of the release.
-        You can also go to this site to download any supported BSP tarballs.
-        Unpacking the tarball gives you a hierarchical Source Directory that lets you develop
-        using the Yocto Project.
-    </para>
-
-    <para>
-        Once you are set up through either tarball extraction or a checkout of Git repositories,
-        you are ready to develop.
+        select the "Downloads" tab and get a released tarball of the
+        <filename>poky</filename> repository or any supported BSP tarballs.
+        Unpacking these tarballs gives you a snapshot of the released
+        files.
+        <note>
+            The recommended method for setting up the Yocto Project
+            <link linkend='source-directory'>Source Directory</link> and the
+            files for supported BSPs (e.g., <filename>meta-intel</filename>) is to
+            use <link linkend='git'>Git</link> to create a local copy of the
+            upstream repositories.
+        </note>
     </para>
 
     <para>
@@ -514,27 +510,40 @@
                 The <ulink url='&YOCTO_DOCS_REF_URL;#var-TOPDIR'><filename>TOPDIR</filename></ulink>
                 variable points to the Build Directory.</para>
 
-                <para>You have a lot of flexibility when creating the Build Directory.
-                Following are some examples that show how to create the directory:
+                <para>
+                    You have a lot of flexibility when creating the Build
+                    Directory.
+                    Following are some examples that show how to create the
+                    directory.
+                    The examples assume your
+                    <link linkend='source-directory'>Source Directory</link> is
+                    named <filename>poky</filename>:
                    <itemizedlist>
-                        <listitem><para>Create the Build Directory in your current working directory
-                            and name it <filename>build</filename>.
-                            This is the default behavior.
+                        <listitem><para>Create the Build Directory inside your
+                            Source Directory and let the name of the Build
+                            Directory default to <filename>build</filename>:
                             <literallayout class='monospaced'>
-     $ source &OE_INIT_PATH;
+     $ cd $HOME/poky
+     $ source &OE_INIT_FILE;
                             </literallayout></para></listitem>
-                        <listitem><para>Provide a directory path and specifically name the build
-                            directory.
-                            This next example creates a Build Directory named <filename>YP-&POKYVERSION;</filename>
-                            in your home directory within the directory <filename>mybuilds</filename>.
-                            If <filename>mybuilds</filename> does not exist, the directory is created for you:
+                        <listitem><para>Create the Build Directory inside your
+                            home directory and specifically name it
+                            <filename>test-builds</filename>:
                             <literallayout class='monospaced'>
-     $ source &OE_INIT_PATH; $HOME/mybuilds/YP-&POKYVERSION;
+     $ cd $HOME
+     $ source poky/&OE_INIT_FILE; test-builds
                             </literallayout></para></listitem>
-                        <listitem><para>Provide an existing directory to use as the Build Directory
-                            and use the default <filename>build</filename> name.
+                        <listitem><para>Provide a directory path and
+                            specifically name the build directory.
+                            Any intermediate folders in the pathname must
+                            exist.
+                            This next example creates a Build Directory named
+                            <filename>YP-&POKYVERSION;</filename>
+                            in your home directory within the existing
+                            directory <filename>mybuilds</filename>:
                             <literallayout class='monospaced'>
-     $ source &OE_INIT_PATH; $HOME/mybuilds/
+     $cd $HOME
+     $ source $HOME/poky/&OE_INIT_FILE; $HOME/mybuilds/YP-&POKYVERSION;
                             </literallayout></para></listitem>
                     </itemizedlist>
                 </para></listitem>
@@ -647,6 +656,17 @@
                 <ulink url='&YOCTO_DOCS_REF_URL;#var-PV'><filename>PV</filename></ulink>, and
                 <ulink url='&YOCTO_DOCS_REF_URL;#var-PE'><filename>PE</filename></ulink>).
                 </para></listitem>
+            <listitem><para><emphasis>Package Groups:</emphasis>
+                Arbitrary groups of software Recipes.
+                You use package groups to hold recipes that, when built,
+                usually accomplish a single task.
+                For example, a package group could contain the recipes for a
+                company’s proprietary or value-add software.
+                Or, the package group could contain the recipes that enable
+                graphics.
+                A package group is really just another recipe.
+                Because package group files are recipes, they end with the
+                <filename>.bb</filename> filename extension.</para></listitem>
             <listitem><para id='poky'><emphasis>Poky:</emphasis> The term "poky" can mean several things.
                 In its most general sense, it is an open-source project that was initially developed
                 by OpenedHand.  With OpenedHand, poky was developed off of the existing OpenEmbedded
@@ -666,63 +686,69 @@
                 use the <filename>.bb</filename> file extension.</para></listitem>
             <listitem>
                 <para id='source-directory'><emphasis>Source Directory:</emphasis>
-                This term refers to the directory structure created as a result of either downloading
-                and unpacking a Yocto Project release tarball or creating a local copy of
-                the <filename>poky</filename> Git repository
-                <filename>git://git.yoctoproject.org/poky</filename>.
-                Sometimes you might hear the term "poky directory" used to refer to this
-                directory structure.
+                This term refers to the directory structure created as a result
+                of creating a local copy of the <filename>poky</filename> Git
+                repository <filename>git://git.yoctoproject.org/poky</filename>
+                or expanding a released <filename>poky</filename> tarball.
+                Sometimes you might hear the term "poky directory" used to refer
+                to this directory structure.
                 <note>
-                    The OpenEmbedded build system does not support file or directory names that
-                    contain spaces.
-                    Be sure that the Source Directory you use does not contain these types
-                    of names.
+                    The OpenEmbedded build system does not support file or
+                    directory names that contain spaces.
+                    Be sure that the Source Directory you use does not contain
+                    these types of names.
                 </note></para>
-                <para>The Source Directory contains BitBake, Documentation, Metadata and
-                other files that all support the Yocto Project.
-                Consequently, you must have the Source Directory in place on your development
-                system in order to do any development using the Yocto Project.</para>
 
-                <para>For tarball expansion, the name of the top-level directory of the Source Directory
-                is derived from the Yocto Project release tarball.
-                For example, downloading and unpacking <filename>&YOCTO_POKY_TARBALL;</filename>
-                results in a Source Directory whose top-level folder is named
-                <filename>&YOCTO_POKY;</filename>.
-                If you create a local copy of the Git repository, you can name the repository
-                anything you like.
-                Throughout much of the documentation, <filename>poky</filename> is used as the name of
-                the top-level folder of the local copy of the poky Git repository.
-                So, for example, cloning the <filename>poky</filename> Git repository results in a
-                local Git repository whose top-level folder is also named <filename>poky</filename>.</para>
+                <para>The Source Directory contains BitBake, Documentation,
+                Metadata and other files that all support the Yocto Project.
+                Consequently, you must have the Source Directory in place on
+                your development system in order to do any development using
+                the Yocto Project.</para>
 
-                <para>It is important to understand the differences between the Source Directory created
-                by unpacking a released tarball as compared to cloning
+                <para>When you create a local copy of the Git repository, you
+                can name the repository anything you like.
+                Throughout much of the documentation, <filename>poky</filename>
+                is used as the name of the top-level folder of the local copy of
+                the poky Git repository.
+                So, for example, cloning the <filename>poky</filename> Git
+                repository results in a local Git repository whose top-level
+                folder is also named <filename>poky</filename>.</para>
+
+                <para>While it is not recommended that you use tarball expansion
+                to setup the Source Directory, if you do, the top-level
+                directory name of the Source Directory is derived from the
+                Yocto Project release tarball.
+                For example, downloading and unpacking
+                <filename>&YOCTO_POKY_TARBALL;</filename> results in a
+                Source Directory whose root folder is named
+                <filename>&YOCTO_POKY;</filename>.</para>
+
+                <para>It is important to understand the differences between the
+                Source Directory created by unpacking a released tarball as
+                compared to cloning
                 <filename>git://git.yoctoproject.org/poky</filename>.
-                When you unpack a tarball, you have an exact copy of the files based on the time of
-                release - a fixed release point.
-                Any changes you make to your local files in the Source Directory are on top of the release.
-                On the other hand, when you clone the <filename>poky</filename> Git repository, you have an
-                active development repository.
-                In this case, any local changes you make to the Source Directory can be later applied
-                to active development branches of the upstream <filename>poky</filename> Git
+                When you unpack a tarball, you have an exact copy of the files
+                based on the time of release - a fixed release point.
+                Any changes you make to your local files in the Source Directory
+                are on top of the release and will remain local only.
+                On the other hand, when you clone the <filename>poky</filename>
+                Git repository, you have an active development repository with
+                access to the upstream repository's branches and tags.
+                In this case, any local changes you make to the local
+                Source Directory can be later applied to active development
+                branches of the upstream <filename>poky</filename> Git
                 repository.</para>
 
-                <para>Finally, if you want to track a set of local changes while starting from the same point
-                as a release tarball, you can create a local Git branch that
-                reflects the exact copy of the files at the time of their release.
-                You do this by using Git tags that are part of the repository.</para>
-
-                <para>For more information on concepts related to Git repositories, branches, and tags,
-                see the
+                <para>For more information on concepts related to Git
+                repositories, branches, and tags, see the
                 "<link linkend='repositories-tags-and-branches'>Repositories, Tags, and Branches</link>"
                 section.</para></listitem>
-            <listitem><para><emphasis>Tasks:</emphasis> Arbitrary groups of software Recipes.
-                You use tasks to hold recipes that, when built, usually accomplish a single task.
-                For example, a task could contain the recipes for a company’s proprietary or value-add software.
-                Or, the task could contain the recipes that enable graphics.
-                A task is really just another recipe.
-                Because task files are recipes, they end with the <filename>.bb</filename> filename
-                extension.</para></listitem>
+            <listitem><para><emphasis>Task:</emphasis>
+                A unit of execution for BitBake (e.g.
+                <filename>do_compile</filename>,
+                <filename>do_fetch</filename>, <filename>do_patch</filename>,
+                and so forth).
+                </para></listitem>
             <listitem><para><emphasis>Upstream:</emphasis> A reference to source code or repositories
                 that are not local to the development system but located in a master area that is controlled
                 by the maintainer of the source code.
@@ -807,7 +833,8 @@
     <title>Git</title>
 
     <para>
-        The Yocto Project uses Git, which is a free, open source distributed version control system.
+        The Yocto Project makes extensive use of Git,
+        which is a free, open source distributed version control system.
         Git supports distributed development, non-linear development, and can handle large projects.
         It is best that you have some fundamental understanding of how Git tracks projects and
         how to work with Git if you are going to use the Yocto Project for development.
@@ -911,7 +938,7 @@
             local working branch based on a branch name,
             your local environment matches the "tip" of that development branch
             at the time you created your local branch, which could be
-            different than the files at the time of a similarly named release.
+            different from the files at the time of a similarly named release.
             In other words, creating and checking out a local branch based on the
             <filename>&DISTRO_NAME;</filename> branch name is not the same as
             cloning and checking out the <filename>master</filename> branch.
@@ -1004,7 +1031,7 @@
                     will allow the change, and for ultimately pushing the change from your local Git repository
                     into the project’s upstream (or master) repository.</para></listitem>
                 <listitem><para><emphasis><filename>git status</filename>:</emphasis> Reports any modified files that
-                    possibly need staged and committed.</para></listitem>
+                    possibly need to be staged and committed.</para></listitem>
                 <listitem><para><emphasis><filename>git checkout &lt;branch-name&gt;</filename>:</emphasis> Changes
                     your working branch.
                     This command is analogous to "cd".</para></listitem>
@@ -1281,10 +1308,9 @@
             <listitem><para><emphasis>Board Support Package (BSP) README Files:</emphasis>
                 For BSP maintainers of supported BSPs, you can examine
                 individual BSP <filename>README</filename> files.
-                Alternatively, you can examine the
-                <filename>MAINTAINERS</filename> file, which is found in the
-                <filename>meta-intel</filename>, for a list of all supported
-                BSP maintainers.
+                In addition, some layers (such as the <filename>meta-intel</filename> layer),
+                include a <filename>MAINTAINERS</filename> file which contains
+                a list of all supported BSP maintainers for that layer.
                 </para></listitem>
             <listitem><para><emphasis>Search by File:</emphasis>
                 Using <link linkend='git'>Git</link>, you can enter the
@@ -1459,8 +1485,8 @@
                     <para>For help on using these scripts, simply provide the
                     <filename>-h</filename> argument as follows:
                     <literallayout class='monospaced'>
-     $ ~/poky/scripts/create-pull-request -h
-     $ ~/poky/scripts/send-pull-request -h
+     $ poky/scripts/create-pull-request -h
+     $ poky/scripts/send-pull-request -h
                     </literallayout></para></listitem>
             </itemizedlist>
         </para>

documentation/dev-manual/dev-manual-start.xml

@@ -55,12 +55,12 @@
     <title>Getting Set Up</title>
 
     <para>
-        Here is what you need to get set up to use the Yocto Project:
+        Here is what you need to use the Yocto Project:
         <itemizedlist>
             <listitem><para><emphasis>Host System:</emphasis>  You should have a reasonably current
                 Linux-based host system.
                 You will have the best results with a recent release of Fedora,
-                OpenSUSE, Debian, Ubuntu, or CentOS as these releases are frequently tested against the Yocto Project
+                openSUSE, Debian, Ubuntu, or CentOS as these releases are frequently tested against the Yocto Project
                 and officially supported.
                 For a list of the distributions under validation and their status, see the
                 "<ulink url='&YOCTO_DOCS_REF_URL;#detailed-supported-distros'>Supported Linux Distributions</ulink>" section
@@ -70,7 +70,7 @@
                 You should also have about 100 gigabytes of free disk space for building images.
                 </para></listitem>
             <listitem><para><emphasis>Packages:</emphasis>  The OpenEmbedded build system
-                requires certain packages exist on your development system (e.g. Python 2.6 or 2.7).
+                requires that certain packages exist on your development system (e.g. Python 2.6 or 2.7).
                 See "<ulink url='&YOCTO_DOCS_QS_URL;#packages'>The Packages</ulink>"
                 section in the Yocto Project Quick Start and the
                 "<ulink url='&YOCTO_DOCS_REF_URL;#required-packages-for-the-host-development-system'>Required Packages for the Host Development System</ulink>"
@@ -79,47 +79,33 @@
                 them for the supported distributions.
                 </para></listitem>
             <listitem id='local-yp-release'><para><emphasis>Yocto Project Release:</emphasis>
-                You need a release of the Yocto Project.
-                You set that up with a local <link linkend='source-directory'>Source Directory</link>
-                one of two ways depending on whether you
-                are going to contribute back into the Yocto Project or not.
-                <note>
-                    Regardless of the method you use, this manual refers to the resulting local
-                    hierarchical set of files as the "Source Directory."
+                You need a release of the Yocto Project installed locally on
+                your development system.
+                This local area is referred to as the
+                <link linkend='source-directory'>Source Directory</link>
+                and is created when you use
+                <link linkend='git'>Git</link> to clone a local copy
+                of the upstream <filename>poky</filename> repository,
+                or when you download an official release of the corresponding tarball.</para>
+                <para>Working from a copy of the upstream repository allows you
+                to contribute back into the Yocto Project or simply work with
+                the latest software on a development branch.
+                Because Git maintains and creates an upstream repository with
+                a complete history of changes and you are working with a local
+                clone of that repository, you have access to all the Yocto
+                Project development branches and tag names used in the upstream
+                repository.</para>
+                <note>You can view the Yocto Project Source Repositories at
+                    <ulink url='&YOCTO_GIT_URL;/cgit.cgi'></ulink>
                 </note>
-                <itemizedlist>
-                    <listitem><para><emphasis>Tarball Extraction:</emphasis>
-                        If you are not going to contribute back into the Yocto
-                        Project, you can simply go to the
-                        <ulink url='&YOCTO_HOME_URL;'>Yocto Project Website</ulink>,
-                        select the "Downloads" tab, and choose what you want.
-                        Once you have the tarball, just extract it into a
-                        directory of your choice.</para>
-                        <para>For example, the following command extracts the
-                        Yocto Project &DISTRO; release tarball
-                        into the current working directory and sets up the local Source Directory
-                        with a top-level folder named <filename>&YOCTO_POKY;</filename>:
-                        <literallayout class='monospaced'>
-     $ tar xfj &YOCTO_POKY_TARBALL;
-                        </literallayout></para>
-                        <para>This method does not produce a local Git repository.
-                        Instead, you simply end up with a snapshot of the release.</para></listitem>
-                    <listitem><para><emphasis>Git Repository Method:</emphasis>  If you are going to be contributing
-                        back into the Yocto Project or you simply want to keep up
-                        with the latest developments, you should use Git commands to set up a local
-                        Git repository of the upstream <filename>poky</filename> source repository.
-                        Doing so creates a repository with a complete history of changes and allows
-                        you to easily submit your changes upstream to the project.
-                        Because you clone the repository, you have access to all the Yocto Project development
-                        branches and tag names used in the upstream repository.</para>
-                        <note>You can view the Yocto Project Source Repositories at
-                        <ulink url='&YOCTO_GIT_URL;/cgit.cgi'></ulink></note>
-                        <para>The following transcript shows how to clone the <filename>poky</filename>
-                        Git repository into the current working directory.
-                        The command creates the local repository in a directory named <filename>poky</filename>.
-                        For information on Git used within the Yocto Project, see the
-                        "<link linkend='git'>Git</link>" section.
-                        <literallayout class='monospaced'>
+                <para>The following transcript shows how to clone the
+                <filename>poky</filename> Git repository into the current
+                working directory.
+                The command creates the local repository in a directory
+                named <filename>poky</filename>.
+                For information on Git used within the Yocto Project, see
+                the "<link linkend='git'>Git</link>" section.
+                <literallayout class='monospaced'>
      $ git clone git://git.yoctoproject.org/poky
      Cloning into 'poky'...
      remote: Counting objects: 203728, done.
@@ -127,16 +113,17 @@
      remote: Total 203728 (delta 147444), reused 202891 (delta 146614)
      Receiving objects: 100% (203728/203728), 95.54 MiB | 308 KiB/s, done.
      Resolving deltas: 100% (147444/147444), done.
-                        </literallayout></para>
-                        <para>For another example of how to set up your own local Git repositories, see this
-                        <ulink url='&YOCTO_WIKI_URL;/wiki/Transcript:_from_git_checkout_to_meta-intel_BSP'>
-                        wiki page</ulink>, which describes how to create both <filename>poky</filename>
-                        and <filename>meta-intel</filename> Git repositories.</para></listitem>
-                </itemizedlist></para></listitem>
+                </literallayout></para>
+                <para>For another example of how to set up your own local Git
+                repositories, see this
+                <ulink url='&YOCTO_WIKI_URL;/wiki/Transcript:_from_git_checkout_to_meta-intel_BSP'>
+                wiki page</ulink>, which describes how to create both
+                <filename>poky</filename> and <filename>meta-intel</filename>
+                Git repositories.</para></listitem>
             <listitem id='local-kernel-files'><para><emphasis>Yocto Project Kernel:</emphasis>
                 If you are going to be making modifications to a supported Yocto Project kernel, you
                 need to establish local copies of the source.
-                You can find Git repositories of supported Yocto Project Kernels organized under
+                You can find Git repositories of supported Yocto Project kernels organized under
                 "Yocto Linux Kernel" in the Yocto Project Source Repositories at
                 <ulink url='&YOCTO_GIT_URL;/cgit.cgi'></ulink>.</para>
                 <para>This setup can involve creating a bare clone of the Yocto Project kernel and then
@@ -198,21 +185,20 @@
                </literallayout></para></listitem>
            <listitem><para id='supported-board-support-packages-(bsps)'><emphasis>Supported Board
                 Support Packages (BSPs):</emphasis>
-                The Yocto Project provides a layer called <filename>meta-intel</filename> and
-                it is maintained in its own separate Git repository.
-                The <filename>meta-intel</filename> layer contains many supported
-                <ulink url='&YOCTO_DOCS_BSP_URL;#bsp-layers'>BSP Layers</ulink>.</para>
-                <para>Similar considerations exist for setting up the <filename>meta-intel</filename>
-                layer.
-                You can get set up for BSP development one of two ways: tarball extraction or
-                with a local Git repository.
-                It is a good idea to use the same method that you used to set up the Source Directory.
-                Regardless of the method you use, the Yocto Project uses the following BSP layer
-                naming scheme:
+                The Yocto Project provides a layer called
+                <filename>meta-intel</filename> and it is maintained in its own
+                separate Git repository.
+                The <filename>meta-intel</filename> layer contains many
+                supported
+                <ulink url='&YOCTO_DOCS_BSP_URL;#bsp-layers'>BSP Layers</ulink>.
+                </para>
+                <para>The Yocto Project uses the following BSP layer naming
+                scheme:
                 <literallayout class='monospaced'>
      meta-&lt;BSP_name&gt;
                 </literallayout>
-                where <filename>&lt;BSP_name&gt;</filename> is the recognized BSP name.
+                where <filename>&lt;BSP_name&gt;</filename> is the recognized
+                BSP name.
                 Here are some examples:
                 <literallayout class='monospaced'>
      meta-crownbay
@@ -221,30 +207,21 @@
                 </literallayout>
                 See the
                 "<ulink url='&YOCTO_DOCS_BSP_URL;#bsp-layers'>BSP Layers</ulink>"
-                section in the Yocto Project Board Support Package (BSP) Developer's Guide for more
-                information on BSP Layers.
-                <itemizedlist>
-                    <listitem><para><emphasis>Tarball Extraction:</emphasis>  You can download any released
-                        BSP tarball from the same "Downloads" page of the
-                        <ulink url='&YOCTO_HOME_URL;'>Yocto Project Website</ulink>
-                        to get the Yocto Project release.
-                        Once on the "Download" page, look to the right of the
-                        page and scroll down to find the BSP tarballs.</para>
-                        <para>Once you have the tarball, just extract it into a directory of your choice.
-                        Again, this method just produces a snapshot of the BSP layer in the form
-                        of a hierarchical directory structure.</para></listitem>
-                    <listitem><para><emphasis>Git Repository Method:</emphasis>  If you are working
-                        with a local Git repository for your Source Directory, you should also use this method
-                        to set up the <filename>meta-intel</filename> Git repository.
-                        You can locate the <filename>meta-intel</filename> Git repository in the
-                        "Yocto Metadata Layers" area of the Yocto Project Source Repositories at
-                        <ulink url='&YOCTO_GIT_URL;/cgit.cgi'></ulink>.</para>
-                        <para>Typically, you set up the <filename>meta-intel</filename> Git repository inside
-                        the Source Directory.
-                        For example, the following transcript shows the steps to clone the
-                        <filename>meta-intel</filename>
-                        Git repository inside the local <filename>poky</filename> Git repository.
-                        <literallayout class='monospaced'>
+                section in the Yocto Project Board Support Package (BSP)
+                Developer's Guide for more information on BSP Layers.</para>
+                <para>You can locate the <filename>meta-intel</filename> Git
+                repository in the "Yocto Metadata Layers" area of the Yocto
+                Project Source Repositories at
+                <ulink url='&YOCTO_GIT_URL;/cgit.cgi'></ulink>.</para>
+                <para>Using
+                <link linkend='git'>Git</link> to create a local clone of the
+                upstream repository can be helpful if you are working with
+                BSPs.
+                Typically, you set up the <filename>meta-intel</filename>
+                Git repository inside the Source Directory.
+                For example, the following transcript shows the steps to clone
+                <filename>meta-intel</filename>.
+                <literallayout class='monospaced'>
      $ cd ~/poky
      $ git clone git://git.yoctoproject.org/meta-intel.git
      Cloning into 'meta-intel'...
@@ -253,13 +230,12 @@
      remote: Total 7366 (delta 3997), reused 7299 (delta 3930)
      Receiving objects: 100% (7366/7366), 2.31 MiB | 95 KiB/s, done.
      Resolving deltas: 100% (3997/3997), done.
-                        </literallayout></para>
-                        <para>The same
-                        <ulink url='&YOCTO_WIKI_URL;/wiki/Transcript:_from_git_checkout_to_meta-intel_BSP'>wiki page</ulink>
-                        referenced earlier covers how to
-                        set up the <filename>meta-intel</filename> Git repository.
-                        </para></listitem>
-                </itemizedlist></para></listitem>
+                </literallayout></para>
+                <para>The same
+                <ulink url='&YOCTO_WIKI_URL;/wiki/Transcript:_from_git_checkout_to_meta-intel_BSP'>wiki page</ulink>
+                referenced earlier covers how to set up the
+                <filename>meta-intel</filename> Git repository.
+                </para></listitem>
             <listitem><para><emphasis>Eclipse Yocto Plug-in:</emphasis>  If you are developing
                 applications using the Eclipse Integrated Development Environment (IDE),
                 you will need this plug-in.

documentation/dev-manual/dev-manual.xml

@@ -56,6 +56,16 @@
                 <date>October 2013</date>
                 <revremark>Released with the Yocto Project 1.5 Release.</revremark>
             </revision>
+            <revision>
+                <revnumber>1.5.1</revnumber>
+                <date>Sometime in 2013</date>
+                <revremark>Released with the Yocto Project 1.5.1 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>1.6</revnumber>
+                <date>Sometime 2014</date>
+                <revremark>Released with the Yocto Project 1.6 Release.</revremark>
+            </revision>
         </revhistory>
 
     <copyright>
@@ -72,11 +82,10 @@
       </para>
 
       <note>
-          Due to production processes, there could be differences between the Yocto Project
-          documentation bundled in the release tarball and the
-          <ulink url='&YOCTO_DOCS_DEV_URL;'>Yocto Project Development Manual</ulink> on
-          the <ulink url='&YOCTO_HOME_URL;'>Yocto Project</ulink> website.
-          For the latest version of this manual, see the manual on the website.
+          For the latest version of this manual associated with this
+          Yocto Project release, see the
+          <ulink url='&YOCTO_DOCS_DEV_URL;'>Yocto Project Development Manual</ulink>
+          from the Yocto Project website.
       </note>
     </legalnotice>