ref-manual: Updates to the migrating to YP 1.7 section.

Some minor wording changes and a new section added for local.conf
QEMU changes.  Also, reordered some sections.

(From yocto-docs rev: 65207b6afa6df7d82cd3482d61f10b308da6fac7)

Signed-off-by: Scott Rifenbark <scott.m.rifenbark@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

README.hardware

@@ -52,6 +52,13 @@ The following boards are supported by the meta-yocto-bsp layer:
 For more information see the board's section below. The appropriate MACHINE
 variable value corresponding to the board is given in brackets.
 
+Reference Board Maintenance
+===========================
+
+Send pull requests, patches, comments or questions about meta-yocto-bsps to poky@yoctoproject.org
+
+Maintainers: Kevin Hao <kexin.hao@windriver.com>
+             Bruce Ashfield <bruce.ashfield@windriver.com>
 
 Consumer Devices
 ================
@@ -317,6 +324,22 @@ Load the kernel and dtb (device tree blob), and boot the system as follows:
  => tftp 2000000 uImage-mpc8315e-rdb.dtb
  => bootm 1000000 - 2000000
 
+--- Booting from JFFS2 root ---
+
+ 1. First boot the board with NFS root.
+
+ 2. Erase the MTD partition which will be used as root:
+
+    $ flash_eraseall  /dev/mtd3
+
+ 3. Copy the JFFS2 image to the MTD partition:
+
+    $ flashcp core-image-minimal-mpc8315e-rdb.jffs2 /dev/mtd3
+
+ 4. Then reboot the board and set up the environment in U-Boot:
+
+    => setenv bootargs root=/dev/mtdblock3 rootfstype=jffs2 console=ttyS0,115200
+
 
 Ubiquiti Networks EdgeRouter Lite (edgerouter)
 ==============================================

bitbake/bin/bitbake

@@ -41,7 +41,7 @@ from bb import ui
 from bb import server
 from bb import cookerdata
 
-__version__ = "1.22.0"
+__version__ = "1.24.0"
 logger = logging.getLogger("BitBake")
 
 # Python multiprocessing requires /dev/shm
@@ -139,8 +139,8 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
         parser.add_option("-n", "--dry-run", help = "Don't execute, just go through the motions.",
                    action = "store_true", dest = "dry_run", default = False)
 
-        parser.add_option("-S", "--dump-signatures", help = "Dump out the signature construction information, with no task execution. Parameters are passed to the signature handling code, use 'none' if no specific handler is required.",
-                   action = "append", dest = "dump_signatures", default = [])
+        parser.add_option("-S", "--dump-signatures", help = "Dump out the signature construction information, with no task execution. The SIGNATURE_HANDLER parameter is passed to the handler. Two common values are none and printdiff but the handler may define more/less. none means only dump the signature, printdiff means compare the dumped signature with the cached one.",
+                   action = "append", dest = "dump_signatures", default = [], metavar="SIGNATURE_HANDLER")
 
         parser.add_option("-p", "--parse-only", help = "Quit after parsing the BB recipes.",
                    action = "store_true", dest = "parse_only", default = False)
@@ -148,7 +148,7 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
         parser.add_option("-s", "--show-versions", help = "Show current and preferred versions of all recipes.",
                    action = "store_true", dest = "show_versions", default = False)
 
-        parser.add_option("-e", "--environment", help = "Show the global or per-package environment complete with information about where variables were set/changed.",
+        parser.add_option("-e", "--environment", help = "Show the global or per-recipe environment complete with information about where variables were set/changed.",
                    action = "store_true", dest = "show_environment", default = False)
 
         parser.add_option("-g", "--graphviz", help = "Save dependency tree information for the specified targets in the dot syntax.",
@@ -169,6 +169,9 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
         parser.add_option("-t", "--servertype", help = "Choose which server to use, process or xmlrpc.",
                    action = "store", dest = "servertype")
 
+        parser.add_option("", "--token", help = "Specify the connection token to be used when connecting to a remote server.",
+                   action = "store", dest = "xmlrpctoken")
+
         parser.add_option("", "--revisions-changed", help = "Set the exit code depending on whether upstream floating revisions have changed or not.",
                    action = "store_true", dest = "revisions_changed", default = False)
 
@@ -200,6 +203,31 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
             options.servertype = "xmlrpc"
             options.remote_server = os.environ["BBSERVER"]
 
+        if "BBTOKEN" in os.environ:
+            options.xmlrpctoken = os.environ["BBTOKEN"]
+
+        # if BBSERVER says to autodetect, let's do that
+        if options.remote_server:
+            [host, port] = options.remote_server.split(":", 2)
+            port = int(port)
+            # use automatic port if port set to -1, means read it from
+            # the bitbake.lock file; this is a bit tricky, but we always expect
+            # to be in the base of the build directory if we need to have a
+            # chance to start the server later, anyway
+            if port == -1:
+                lock_location = "./bitbake.lock"
+                # we try to read the address at all times; if the server is not started,
+                # we'll try to start it after the first connect fails, below
+                try:
+                    lf = open(lock_location, 'r')
+                    remotedef = lf.readline()
+                    [host, port] = remotedef.split(":")
+                    port = int(port)
+                    lf.close()
+                    options.remote_server = remotedef
+                except Exception as e:
+                    sys.exit("Failed to read bitbake.lock (%s), invalid port" % str(e))
+
         return options, targets[1:]
 
 
@@ -279,6 +307,9 @@ def main():
     if configParams.observe_only and (not configParams.remote_server or configParams.bind):
         sys.exit("FATAL: '--observe-only' can only be used by UI clients connecting to a server.\n")
 
+    if configParams.kill_server and not configParams.remote_server:
+        sys.exit("FATAL: '--kill-server' can only be used to terminate a remote server")
+
     if "BBDEBUG" in os.environ:
         level = int(os.environ["BBDEBUG"])
         if level > configuration.debug:
@@ -307,30 +338,17 @@ def main():
         bb.event.ui_queue = []
     else:
         # we start a stub server that is actually a XMLRPClient that connects to a real server
-        server = servermodule.BitBakeXMLRPCClient(configParams.observe_only)
+        server = servermodule.BitBakeXMLRPCClient(configParams.observe_only, configParams.xmlrpctoken)
         server.saveConnectionDetails(configParams.remote_server)
-        server.saveConnectionConfigParams(configParams)
+
 
     if not configParams.server_only:
-        if configParams.status_only:
-            try:
-                server_connection = server.establishConnection(featureset)
-            except:
-                sys.exit(1)
-            if not server_connection:
-                sys.exit(1)
-            server_connection.terminate()
-            sys.exit(0)
-
-        # Setup a connection to the server (cooker)
-        server_connection = server.establishConnection(featureset)
-        if not server_connection:
-            if configParams.kill_server:
-                bb.fatal("Server already killed")
-            configParams.bind = configParams.remote_server
-            start_server(servermodule, configParams, configuration, featureset)
-            bb.event.ui_queue = []
+        try:
             server_connection = server.establishConnection(featureset)
+        except Exception as e:
+            if configParams.kill_server:
+                sys.exit(0)
+            bb.fatal("Could not connect to server %s: %s" % (configParams.remote_server, str(e)))
 
         # Restore the environment in case the UI needs it
         for k in cleanedvars:
@@ -338,6 +356,16 @@ def main():
 
         logger.removeHandler(handler)
 
+
+        if configParams.status_only:
+            server_connection.terminate()
+            sys.exit(0)
+
+        if configParams.kill_server:
+            server_connection.connection.terminateServer()
+            bb.event.ui_queue = []
+            sys.exit(0)
+
         try:
             return ui_module.main(server_connection.connection, server_connection.events, configParams)
         finally:

bitbake/bin/bitbake-diffsigs

@@ -118,5 +118,5 @@ else:
             logger.error('Invalid signature data - ensure you are specifying sigdata/siginfo files')
             sys.exit(1)
 
-    if output:
-        print '\n'.join(output)
+        if output:
+            print '\n'.join(output)

bitbake/bin/bitbake-layers

@@ -556,7 +556,7 @@ Recipes are listed with the bbappends that apply to them as subitems.
                 continue
 
             basename = os.path.basename(filename)
-            appends = self.bbhandler.cooker.collection.appendlist.get(basename)
+            appends = self.bbhandler.cooker.collection.get_file_appends(basename)
             if appends:
                 appended.append((basename, list(appends)))
             else:
@@ -566,26 +566,29 @@ Recipes are listed with the bbappends that apply to them as subitems.
     def do_show_cross_depends(self, args):
         """figure out the dependency between recipes that crosses a layer boundary.
 
-usage: show-cross-depends [-f]
+usage: show-cross-depends [-f] [-i layer1[,layer2[,layer3...]]]
 
 Figure out the dependency between recipes that crosses a layer boundary.
 
 Options:
   -f   show full file path
+  -i   ignore dependencies on items in the specified layer(s)
 
 NOTE:
 The .bbappend file can impact the dependency.
 """
-        self.init_bbhandler()
+        import optparse
 
-        show_filenames = False
-        for arg in args.split():
-            if arg == '-f':
-                show_filenames = True
-            else:
-                sys.stderr.write("show-cross-depends: invalid option %s\n" % arg)
-                self.do_help('')
-                return
+        parser = optparse.OptionParser(usage="show-cross-depends [-f] [-i layer1[,layer2[,layer3...]]]")
+        parser.add_option("-f", "",
+                action="store_true", dest="show_filenames")
+        parser.add_option("-i", "",
+                action="store", dest="ignore_layers", default="")
+
+        options, args = parser.parse_args(sys.argv)
+        ignore_layers = options.ignore_layers.split(',')
+
+        self.init_bbhandler()
 
         pkg_fn = self.bbhandler.cooker_data.pkg_fn
         bbpath = str(self.bbhandler.config_data.getVar('BBPATH', True))
@@ -593,6 +596,8 @@ The .bbappend file can impact the dependency.
         self.include_re = re.compile(r"include\s+(.+)")
         self.inherit_re = re.compile(r"inherit\s+(.+)")
 
+        global_inherit = (self.bbhandler.config_data.getVar('INHERIT', True) or "").split()
+
         # The bb's DEPENDS and RDEPENDS
         for f in pkg_fn:
             f = bb.cache.Cache.virtualfn2realfn(f)[0]
@@ -607,7 +612,7 @@ The .bbappend file can impact the dependency.
                             self.bbhandler.config_data,
                             self.bbhandler.cooker_data,
                             self.bbhandler.cooker_data.pkg_pn)
-                    self.check_cross_depends("DEPENDS", layername, f, best[3], show_filenames)
+                    self.check_cross_depends("DEPENDS", layername, f, best[3], options.show_filenames, ignore_layers)
 
             # The RDPENDS
             all_rdeps = self.bbhandler.cooker_data.rundeps[f].values()
@@ -621,10 +626,33 @@ The .bbappend file can impact the dependency.
             for rdep in all_rdeps:
                 all_p = bb.providers.getRuntimeProviders(self.bbhandler.cooker_data, rdep)
                 if all_p:
+                    if f in all_p:
+                        # The recipe provides this one itself, ignore
+                        continue
                     best = bb.providers.filterProvidersRunTime(all_p, rdep,
                                     self.bbhandler.config_data,
                                     self.bbhandler.cooker_data)[0][0]
-                    self.check_cross_depends("RDEPENDS", layername, f, best, show_filenames)
+                    self.check_cross_depends("RDEPENDS", layername, f, best, options.show_filenames, ignore_layers)
+
+            # The RRECOMMENDS
+            all_rrecs = self.bbhandler.cooker_data.runrecs[f].values()
+            # Remove the duplicated or null one.
+            sorted_rrecs = {}
+            # The all_rrecs is the list in list, so we need two for loops
+            for k1 in all_rrecs:
+                for k2 in k1:
+                    sorted_rrecs[k2] = 1
+            all_rrecs = sorted_rrecs.keys()
+            for rrec in all_rrecs:
+                all_p = bb.providers.getRuntimeProviders(self.bbhandler.cooker_data, rrec)
+                if all_p:
+                    if f in all_p:
+                        # The recipe provides this one itself, ignore
+                        continue
+                    best = bb.providers.filterProvidersRunTime(all_p, rrec,
+                                    self.bbhandler.config_data,
+                                    self.bbhandler.cooker_data)[0][0]
+                    self.check_cross_depends("RRECOMMENDS", layername, f, best, options.show_filenames, ignore_layers)
 
             # The inherit class
             cls_re = re.compile('classes/')
@@ -634,9 +662,12 @@ The .bbappend file can impact the dependency.
                     # The inherits' format is [classes/cls, /path/to/classes/cls]
                     # ignore the classes/cls.
                     if not cls_re.match(cls):
+                        classname = os.path.splitext(os.path.basename(cls))[0]
+                        if classname in global_inherit:
+                            continue
                         inherit_layername = self.get_file_layer(cls)
-                        if inherit_layername != layername:
-                            if not show_filenames:
+                        if inherit_layername != layername and not inherit_layername in ignore_layers:
+                            if not options.show_filenames:
                                 f_short = self.remove_layer_prefix(f)
                                 cls = self.remove_layer_prefix(cls)
                             else:
@@ -656,7 +687,7 @@ The .bbappend file can impact the dependency.
                     if pv_re.search(needed_file) and f in self.bbhandler.cooker_data.pkg_pepvpr:
                         pv = self.bbhandler.cooker_data.pkg_pepvpr[f][1]
                         needed_file = re.sub(r"\${PV}", pv, needed_file)
-                    self.print_cross_files(bbpath, keyword, layername, f, needed_file, show_filenames)
+                    self.print_cross_files(bbpath, keyword, layername, f, needed_file, options.show_filenames, ignore_layers)
                 line = fnfile.readline()
             fnfile.close()
 
@@ -683,21 +714,22 @@ The .bbappend file can impact the dependency.
                                 bbclass=".bbclass"
                             # Find a 'require/include xxxx'
                             if m:
-                                self.print_cross_files(bbpath, keyword, layername, f, m.group(1) + bbclass, show_filenames)
+                                self.print_cross_files(bbpath, keyword, layername, f, m.group(1) + bbclass, options.show_filenames, ignore_layers)
                             line = ffile.readline()
                         ffile.close()
 
-    def print_cross_files(self, bbpath, keyword, layername, f, needed_filename, show_filenames):
+    def print_cross_files(self, bbpath, keyword, layername, f, needed_filename, show_filenames, ignore_layers):
         """Print the depends that crosses a layer boundary"""
         needed_file = bb.utils.which(bbpath, needed_filename)
         if needed_file:
             # Which layer is this file from
             needed_layername = self.get_file_layer(needed_file)
-            if needed_layername != layername:
+            if needed_layername != layername and not needed_layername in ignore_layers:
                 if not show_filenames:
                     f = self.remove_layer_prefix(f)
                     needed_file = self.remove_layer_prefix(needed_file)
                 logger.plain("%s %s %s" %(f, keyword, needed_file))
+
     def match_inherit(self, line):
         """Match the inherit xxx line"""
         return (self.inherit_re.match(line), "inherits")
@@ -711,11 +743,11 @@ The .bbappend file can impact the dependency.
             keyword = "includes"
         return (m, keyword)
 
-    def check_cross_depends(self, keyword, layername, f, needed_file, show_filenames):
+    def check_cross_depends(self, keyword, layername, f, needed_file, show_filenames, ignore_layers):
         """Print the DEPENDS/RDEPENDS file that crosses a layer boundary"""
         best_realfn = bb.cache.Cache.virtualfn2realfn(needed_file)[0]
         needed_layername = self.get_file_layer(best_realfn)
-        if needed_layername != layername:
+        if needed_layername != layername and not needed_layername in ignore_layers:
             if not show_filenames:
                 f = self.remove_layer_prefix(f)
                 best_realfn = self.remove_layer_prefix(best_realfn)

bitbake/bin/bitbake-worker

@@ -12,10 +12,18 @@ import errno
 import signal
 
 # Users shouldn't be running this code directly
-if len(sys.argv) != 2 or sys.argv[1] != "decafbad":
+if len(sys.argv) != 2 or not sys.argv[1].startswith("decafbad"):
     print("bitbake-worker is meant for internal execution by bitbake itself, please don't use it standalone.")
     sys.exit(1)
 
+profiling = False
+if sys.argv[1] == "decafbadbad":
+    profiling = True
+    try:
+        import cProfile as profile
+    except:
+        import profile
+
 logger = logging.getLogger("BitBake")
 
 try:
@@ -81,6 +89,11 @@ def workerlog_write(msg):
         lf.write(msg)
         lf.flush()
 
+def sigterm_handler(signum, frame):
+    signal.signal(signal.SIGTERM, signal.SIG_DFL)
+    os.killpg(0, signal.SIGTERM)
+    sys.exit()
+
 def fork_off_task(cfg, data, workerdata, fn, task, taskname, appends, taskdepdata, quieterrors=False):
     # We need to setup the environment BEFORE the fork, since
     # a fork() or exec*() activates PSEUDO...
@@ -129,10 +142,11 @@ def fork_off_task(cfg, data, workerdata, fn, task, taskname, appends, taskdepdat
         bb.msg.fatal("RunQueue", "fork failed: %d (%s)" % (e.errno, e.strerror))
 
     if pid == 0:
+        def child():
             global worker_pipe
             pipein.close()
 
-            signal.signal(signal.SIGTERM, signal.SIG_DFL)
+            signal.signal(signal.SIGTERM, sigterm_handler)
 
             # Save out the PID so that the event can include it the
             # events
@@ -154,15 +168,11 @@ def fork_off_task(cfg, data, workerdata, fn, task, taskname, appends, taskdepdat
             data.setVar("BUILDNAME", workerdata["buildname"])
             data.setVar("DATE", workerdata["date"])
             data.setVar("TIME", workerdata["time"])
-            bb.parse.siggen.set_taskdata(workerdata["hashes"], workerdata["hash_deps"], workerdata["sigchecksums"])
+            bb.parse.siggen.set_taskdata(workerdata["sigdata"])
             ret = 0
             try:
                 the_data = bb.cache.Cache.loadDataFull(fn, appends, data)
                 the_data.setVar('BB_TASKHASH', workerdata["runq_hash"][task])
-                for h in workerdata["hashes"]:
-                    the_data.setVar("BBHASH_%s" % h, workerdata["hashes"][h])
-                for h in workerdata["hash_deps"]:
-                    the_data.setVar("BBHASHDEPS_%s" % h, workerdata["hash_deps"][h])
 
                 # exported_vars() returns a generator which *cannot* be passed to os.environ.update() 
                 # successfully. We also need to unset anything from the environment which shouldn't be there 
@@ -183,11 +193,22 @@ def fork_off_task(cfg, data, workerdata, fn, task, taskname, appends, taskdepdat
                     logger.critical(str(exc))
                 os._exit(1)
             try:
-                if not cfg.dry_run:
-                    ret = bb.build.exec_task(fn, taskname, the_data, cfg.profile)
-                os._exit(ret)
+                if cfg.dry_run:
+                    return 0
+                return bb.build.exec_task(fn, taskname, the_data, cfg.profile)
             except:
                 os._exit(1)
+        if not profiling:
+            os._exit(child())
+        else:
+            profname = "profile-%s.log" % (fn.replace("/", "-") + "-" + taskname)
+            prof = profile.Profile()
+            try: 
+                ret = profile.Profile.runcall(prof, child)
+            finally:
+                prof.dump_stats(profname)
+                bb.utils.process_profilelog(profname)
+                os._exit(ret)
     else:
         for key, value in envbackup.iteritems():
             if value is None:
@@ -362,7 +383,16 @@ class BitbakeWorker(object):
 
 try:
     worker = BitbakeWorker(sys.stdin)
-    worker.serve()
+    if not profiling:
+        worker.serve()
+    else:
+        profname = "profile-worker.log"
+        prof = profile.Profile()
+        try:
+            profile.Profile.runcall(prof, worker.serve)
+        finally:
+            prof.dump_stats(profname)
+            bb.utils.process_profilelog(profname)
 except BaseException as e:
     if not normalexit:
         import traceback

bitbake/bin/toaster

@@ -16,9 +16,15 @@
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
 
-# This script enables toaster event logging and
-# starts bitbake resident server
-# use as:  source toaster [start|stop]
+# This script can be run in two modes.
+
+# When used with "source", from a build directory,
+# it enables toaster event logging and starts the bitbake resident server.
+# use as:  source toaster [start|stop] [noweb] [noui]
+
+# When it is called as a stand-alone script, it starts just the
+# web server, and the building shall be done through the web interface.
+# As script, it will not return to the command prompt. Stop with Ctrl-C.
 
 # Helper function to kill a background toaster development server
 
@@ -30,6 +36,8 @@ function webserverKillAll()
 		while kill -0 $(< ${pidfile}) 2>/dev/null; do
 			kill -SIGTERM -$(< ${pidfile}) 2>/dev/null
 			sleep 1;
+			# Kill processes if they are still running - may happen in interactive shells
+			ps fux | grep "python.*manage.py" | awk '{print $2}' | xargs kill
 		done;
 		rm  ${pidfile}
 		fi
@@ -38,6 +46,12 @@ function webserverKillAll()
 
 function webserverStartAll()
 {
+	# do not start if toastermain points to a valid process
+	if ! cat "${BUILDDIR}/.toastermain.pid" 2>/dev/null | xargs -I{} kill -0 {} ; then
+            retval=1
+            rm "${BUILDDIR}/.toastermain.pid"
+        fi
+
         retval=0
         python $BBBASEDIR/lib/toaster/manage.py syncdb || retval=1
         python $BBBASEDIR/lib/toaster/manage.py migrate orm || retval=2
@@ -49,7 +63,12 @@ function webserverStartAll()
             retval=0
             python $BBBASEDIR/lib/toaster/manage.py migrate orm || retval=1
         fi
+	if [ "x$TOASTER_MANAGED" == "x1" ]; then
+	        python $BBBASEDIR/lib/toaster/manage.py migrate bldcontrol || retval=1
+		python $BBBASEDIR/lib/toaster/manage.py checksettings  --traceback || retval=1
+	fi
         if [ $retval -eq 0 ]; then
+	    echo "Starting webserver"
             python $BBBASEDIR/lib/toaster/manage.py runserver 0.0.0.0:8000 </dev/null >${BUILDDIR}/toaster_web.log 2>&1 & echo $! >${BUILDDIR}/.toastermain.pid
             sleep 1
             if ! cat "${BUILDDIR}/.toastermain.pid" | xargs -I{} kill -0 {} ; then
@@ -80,7 +99,7 @@ function stop_system()
         kill $(< ${BUILDDIR}/.toasterui.pid ) 2>/dev/null
         rm ${BUILDDIR}/.toasterui.pid
     fi
-    BBSERVER=localhost:8200 bitbake -m
+    BBSERVER=0.0.0.0:8200 bitbake -m
     unset BBSERVER
     webserverKillAll
     # force stop any misbehaving bitbake server
@@ -103,30 +122,60 @@ function notify_chldexit() {
 }
 
 
-# We make sure we're running in the current shell and in a good environment
+BBBASEDIR=`dirname ${BASH_SOURCE}`/..
+RUNNING=0
 
 if [ -z "$ZSH_NAME" ] && [ `basename \"$0\"` = `basename \"$BASH_SOURCE\"` ]; then
-    echo "Error: This script needs to be sourced. Please run as 'source toaster [start|stop]'" 1>&2;
-    exit 1
+    # We are called as standalone. We refuse to run in a build environment - we need the interactive mode for that.
+    # Start just the web server, point the web browser to the interface, and start any Django services.
+
+    if [ -n "$BUILDDIR" ]; then
+        echo -e "Error: build/ directory detected. Toaster will not start in managed mode if a build environment is detected.\nUse a clean terminal to start Toaster." 1>&2;
+        exit 1;
+    fi
+
+    # Define a fake builddir where only the pid files are actually created. No real builds will take place here.
+    BUILDDIR=/tmp
+    RUNNING=1
+    function trap_ctrlc() {
+        echo "** Stopping system"
+        webserverKillAll
+        RUNNING=0
+    }
+    TOASTER_MANAGED=1
+    export TOASTER_MANAGED=1
+    if ! webserverStartAll; then
+        echo "Failed to start the web server, stopping" 1>&2;
+        exit 1;
+    fi
+    xdg-open http://0.0.0.0:8000/ >/dev/null 2>&1 &
+    trap trap_ctrlc SIGINT
+    echo "Running. Stop with Ctrl-C"
+    while [ $RUNNING -gt 0 ]; do
+	python $BBBASEDIR/lib/toaster/manage.py runbuilds
+	sleep 1
+    done
+    echo "**** Exit"
+    exit 0
 fi
 
+# We make sure we're running in the current shell and in a good environment
 if [ -z "$BUILDDIR" ] || [ -z `which bitbake` ]; then
     echo "Error: Build environment is not setup or bitbake is not in path." 1>&2;
     return 2
 fi
 
-BBBASEDIR=`dirname ${BASH_SOURCE}`/..
 
 
 # Verify prerequisites
 
-if ! echo "import django; print (1,5) == django.VERSION[0:2]" | python 2>/dev/null | grep True >/dev/null; then
-    echo -e "This program needs Django 1.5. Please install with\n\nsudo pip install django==1.5"
+if ! echo "import django; print (1,) == django.VERSION[0:1] and django.VERSION[1:2][0] in (5,6)" | python 2>/dev/null | grep True >/dev/null; then
+    echo -e "This program needs Django 1.5 or 1.6. Please install with\n\npip install django==1.6"
     return 2
 fi
 
 if ! echo "import south; print [0,8,4] ==  map(int,south.__version__.split(\".\"))" | python 2>/dev/null | grep True >/dev/null; then
-    echo -e "This program needs South 0.8.4. Please install with\n\nsudo pip install south==0.8.4"
+    echo -e "This program needs South 0.8.4. Please install with\n\npip install south==0.8.4"
     return 2
 fi
 
@@ -146,11 +195,15 @@ else
 fi
 
 NOTOASTERUI=0
+WEBSERVER=1
 for param in $*; do
     case $param in
     noui )
             NOTOASTERUI=1
     ;;
+    noweb )
+            WEBSERVER=0
+    ;;
     esac
 done
 
@@ -176,17 +229,17 @@ case $CMD in
     start )
         start_success=1
         addtoConfiguration "INHERIT+=\"toaster buildhistory\"" toaster.conf
-        if ! webserverStartAll; then
+        if [ $WEBSERVER -gt 0 ] && ! webserverStartAll; then
             echo "Failed ${CMD}."
             return 4
         fi
         unset BBSERVER
-        bitbake --postread conf/toaster.conf --server-only -t xmlrpc -B localhost:8200
+        bitbake --postread conf/toaster.conf --server-only -t xmlrpc -B 0.0.0.0:8200
         if [ $? -ne 0 ]; then
             start_success=0
             echo "Bitbake server start failed"
         else
-            export BBSERVER=localhost:8200
+            export BBSERVER=0.0.0.0:8200
             if [ $NOTOASTERUI == 0 ]; then        # we start the TOASTERUI only if not inhibited
                 bitbake --observe-only -u toasterui >${BUILDDIR}/toaster_ui.log 2>&1 & echo $! >${BUILDDIR}/.toasterui.pid
             fi

bitbake/contrib/vim/plugin/newbb.vim

@@ -53,7 +53,6 @@ fun! NewBBTemplate()
     put ='LICENSE = \"\"' 
     put ='SECTION = \"\"'
     put ='DEPENDS = \"\"'
-    put ='PR = \"r0\"'
     put =''
     put ='SRC_URI = \"\"'
 

bitbake/doc/Makefile

@@ -18,21 +18,21 @@
 #
 # Examples:
 #
-#     make DOC=user-manual
-#     make pdf DOC=user-manual
+#     make DOC=bitbake-user-manual
+#     make pdf DOC=bitbake-user-manual
 #
 # The first example generates the HTML and PDF versions of the User Manual.
 # The second example generates the HTML version only of the User Manual.
 #
 
-ifeq ($(DOC),user-manual)
-XSLTOPTS = --stringparam html.stylesheet user-manual-style.css \
+ifeq ($(DOC),bitbake-user-manual)
+XSLTOPTS = --stringparam html.stylesheet bitbake-user-manual-style.css \
            --stringparam  chapter.autolabel 1 \
            --stringparam  section.autolabel 1 \
            --stringparam  section.label.includes.component.label 1 \
            --xinclude
 ALLPREQ = html pdf tarball
-TARFILES = user-manual-style.css user-manual.html user-manual.pdf figures/bitbake-title.png
+TARFILES = bitbake-user-manual-style.css bitbake-user-manual.html bitbake-user-manual.pdf figures/bitbake-title.png
 MANUALS = $(DOC)/$(DOC).html $(DOC)/$(DOC).pdf
 FIGURES = figures
 STYLESHEET = $(DOC)/*.css
@@ -48,7 +48,7 @@ XSL_XHTML_URI = $(XSL_BASE_URI)/xhtml/docbook.xsl
 all: $(ALLPREQ)
 
 pdf:
-ifeq ($(DOC),user-manual)
+ifeq ($(DOC),bitbake-user-manual)
 	@echo " "
 	@echo "********** Building."$(DOC)
 	@echo " "
@@ -56,7 +56,7 @@ ifeq ($(DOC),user-manual)
 endif
 
 html:
-ifeq ($(DOC),user-manual)
+ifeq ($(DOC),bitbake-user-manual)
 #       See http://www.sagehill.net/docbookxsl/HtmlOutput.html
 	@echo " "
 	@echo "******** Building "$(DOC)

bitbake/doc/README

@@ -8,7 +8,7 @@ Manual Organization
 
 Folders exist for individual manuals as follows:
 
-* user-manual      - The BitBake User Manual 
+* bitbake-user-manual      - The BitBake User Manual 
 
 Each folder is self-contained regarding content and figures.
 
@@ -28,7 +28,7 @@ For example, the following command run from the documentation directory
 creates an HTML and a PDF version of the BitBake User Manual.
 The DOC variable specifies the manual you are making:
 
-     $ make DOC=user-manual
+     $ make DOC=bitbake-user-manual
 
 template
 ========

bitbake/doc/bitbake-user-manual/bitbake-user-manual-customization.xsl

@@ -3,9 +3,15 @@
 
   <xsl:import href="http://docbook.sourceforge.net/release/xsl/current/xhtml/docbook.xsl" />
 
+  <xsl:include href="../template/permalinks.xsl"/>
+  <xsl:include href="../template/section.title.xsl"/>
+  <xsl:include href="../template/component.title.xsl"/>
+  <xsl:include href="../template/division.title.xsl"/>
+  <xsl:include href="../template/formal.object.heading.xsl"/>
+  <xsl:include href="../template/gloss-permalinks.xsl"/>
+
   <xsl:param name="html.stylesheet" select="'user-manual-style.css'" />
   <xsl:param name="chapter.autolabel" select="1" />
-<!--  <xsl:param name="appendix.autolabel" select="A" /> -->
   <xsl:param name="section.autolabel" select="1" />
   <xsl:param name="section.label.includes.component.label" select="1" />
   <xsl:param name="appendix.autolabel">A</xsl:param>

bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.xml

@@ -1,13 +1,14 @@
 <!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
 "http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd">
 
-<chapter id="user-manual-execution">
+<chapter id="bitbake-user-manual-execution">
     <title>Execution</title>
 
     <para>
         The primary purpose for running BitBake is to produce some kind
-        of output such as an image, a kernel, or a software development
-        kit.
+        of output such as a single installable package, a kernel, a software
+        development kit, or even a full, board-specific bootable Linux image,
+        complete with bootloader, kernel, and root filesystem.
         Of course, you can execute the <filename>bitbake</filename>
         command with options that cause it to execute single tasks,
         compile single recipe files, capture or clear data, or simply
@@ -24,17 +25,34 @@
         </literallayout>
         For information on the BitBake command and its options,
         see
-        "<link linkend='user-manual-command'>The BitBake Command</link>"
+        "<link linkend='bitbake-user-manual-command'>The BitBake Command</link>"
         section.
-    </para>
+        <note>
+            <para>
+                Prior to executing BitBake, you should take advantage of available
+                parallel thread execution on your build host by setting the
+                <link linkend='var-BB_NUMBER_THREADS'><filename>BB_NUMBER_THREADS</filename></link>
+                variable in your project's <filename>local.conf</filename>
+                configuration file.
+            </para>
 
-    <note>
-        Prior to executing BitBake, you should take advantage of parallel
-        thread execution by setting the
-        <link linkend='var-BB_NUMBER_THREADS'><filename>BB_NUMBER_THREADS</filename></link>
-        variable in your <filename>local.conf</filename>
-        configuration file.
-    </note>
+            <para>
+                A common way to determine this value for your build host is to run:
+                <literallayout class='monospaced'>
+     $ grep processor /proc/cpuinfo
+                </literallayout>
+                and count the number of processors displayed. Note that the number of
+                processors will take into account hyper-threading, so that a quad-core
+                build host with hyper-threading will most likely show eight processors,
+                which is the value you would then assign to that variable.
+            </para>
+
+            <para>
+                A possibly simpler solution is that some Linux distributions
+                (e.g. Debian and Ubuntu) provide the <filename>ncpus</filename> command.
+            </para>
+        </note>
+    </para>
 
     <section id='parsing-the-base-configuration-metadata'>
         <title>Parsing the Base Configuration Metadata</title>
@@ -42,7 +60,7 @@
         <para>
             The first thing BitBake does is parse base configuration
             metadata.
-            Base configuration metadata consists of the
+            Base configuration metadata consists of your project's
             <filename>bblayers.conf</filename> file to determine what
             layers BitBake needs to recognize, all necessary
             <filename>layer.conf</filename> files (one from each layer),
@@ -71,10 +89,11 @@
             and
             <link linkend='var-BBFILES'><filename>BBFILES</filename></link>.
             <filename>BBPATH</filename> is used to search for
-            configuration and class files under
-            <filename>conf/</filename> and <filename>class/</filename>
+            configuration and class files under the
+            <filename>conf</filename> and <filename>classes</filename>
             directories, respectively.
-            <filename>BBFILES</filename> is used to find recipe files
+            <filename>BBFILES</filename> is used to locate both recipe
+            and recipe append files
             (<filename>.bb</filename> and <filename>.bbappend</filename>).
             If there is no <filename>bblayers.conf</filename> file,
             it is assumed the user has set the <filename>BBPATH</filename>
@@ -82,7 +101,7 @@
         </para>
 
         <para>
-            Next, the <filename>bitbake.conf</filename> file is searched
+            Next, the <filename>bitbake.conf</filename> file is located
             using the <filename>BBPATH</filename> variable that was
             just constructed.
             The <filename>bitbake.conf</filename> file may also include other
@@ -117,18 +136,18 @@
             optional <filename>conf/bblayers.conf</filename> configuration file.
             This file is expected to contain a
             <link linkend='var-BBLAYERS'><filename>BBLAYERS</filename></link>
-            variable that is a space delimited list of 'layer' directories.
+            variable that is a space-delimited list of 'layer' directories.
             Recall that if BitBake cannot find a <filename>bblayers.conf</filename>
-            file then it is assumed the user has set the <filename>BBPATH</filename>
-            and <filename>BBFILES</filename> directly in the environment.
+            file, then it is assumed the user has set the <filename>BBPATH</filename>
+            and <filename>BBFILES</filename> variables directly in the environment.
         </para>
 
         <para>
             For each directory (layer) in this list, a <filename>conf/layer.conf</filename>
-            file is searched for and parsed with the
+            file is located and parsed with the
             <link linkend='var-LAYERDIR'><filename>LAYERDIR</filename></link>
             variable being set to the directory where the layer was found.
-            The idea is these files automatically setup
+            The idea is these files automatically set up
             <link linkend='var-BBPATH'><filename>BBPATH</filename></link>
             and other variables correctly for a given build directory.
         </para>
@@ -143,7 +162,7 @@
 
         <para>
             Only variable definitions and include directives are allowed
-            in <filename>.conf</filename> files.
+            in BitBake <filename>.conf</filename> files.
             Some variables directly influence BitBake's behavior.
             These variables might have been set from the environment
             depending on the environment variables previously
@@ -166,7 +185,8 @@
             Other classes that are specified in the configuration using the
             <link linkend='var-INHERIT'><filename>INHERIT</filename></link>
             variable are also included.
-            BitBake searches for class files in a "classes" subdirectory under
+            BitBake searches for class files in a
+            <filename>classes</filename> subdirectory under
             the paths in <filename>BBPATH</filename> in the same way as
             configuration files.
         </para>
@@ -189,7 +209,7 @@
                 If a recipe uses a closing curly brace within the function and
                 the character has no leading spaces, BitBake produces a parsing
                 error.
-                If you use a pair of curly brace in a shell function, the
+                If you use a pair of curly braces in a shell function, the
                 closing curly brace must not be located at the start of the line
                 without leading spaces.
             </para>
@@ -261,14 +281,14 @@
             One common convention is to use the recipe filename to define
             pieces of metadata.
             For example, in <filename>bitbake.conf</filename> the recipe
-            name and version set
+            name and version are used to set the variables
             <link linkend='var-PN'><filename>PN</filename></link> and
             <link linkend='var-PV'><filename>PV</filename></link>:
             <literallayout class='monospaced'>
-     PV = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE'),d)[1] or '1.0'}"
      PN = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE'),d)[0] or 'defaultpkgname'}"
+     PV = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE'),d)[1] or '1.0'}"
             </literallayout>
-            In this example, a recipe called "something_1.2.3.bb" sets
+            In this example, a recipe called "something_1.2.3.bb" would set
             <filename>PN</filename> to "something" and
             <filename>PV</filename> to "1.2.3".
         </para>
@@ -331,38 +351,55 @@
     </section>
 
     <section id='bb-bitbake-providers'>
-        <title>Preferences and Providers</title>
+        <title>Providers</title>
 
         <para>
             Assuming BitBake has been instructed to execute a target
             and that all the recipe files have been parsed, BitBake
             starts to figure out how to build the target.
-            BitBake starts by looking through the
+            BitBake looks through the <filename>PROVIDES</filename> list
+            for each of the recipes.
+            A <filename>PROVIDES</filename> list is the list of names by which
+            the recipe can be known.
+            Each recipe's <filename>PROVIDES</filename> list is created
+            implicitly through the recipe's
+            <link linkend='var-PN'><filename>PN</filename></link> variable
+            and explicitly through the recipe's
             <link linkend='var-PROVIDES'><filename>PROVIDES</filename></link>
-            set in recipe files.
-            The default <filename>PROVIDES</filename> for a recipe is its name
-            (<link linkend='var-PN'><filename>PN</filename></link>),
-            however, a recipe can provide multiple things.
+            variable, which is optional.
         </para>
 
         <para>
-            As an example of adding an extra provider, suppose a recipe named
-            <filename>foo_1.0.bb</filename> contained the following:
+            When a recipe uses <filename>PROVIDES</filename>, that recipe's
+            functionality can be found under an alternative name or names other
+            than the implicit <filename>PN</filename> name.
+            As an example, suppose a recipe named <filename>keyboard_1.0.bb</filename>
+            contained the following:
             <literallayout class='monospaced'>
-     PROVIDES += "virtual/bar_1.0"
+     PROVIDES += "fullkeyboard"
             </literallayout>
-            The recipe now provides both "foo_1.0" and "virtual/bar_1.0".
-            The "virtual/" namespace is often used to denote cases where
-            multiple providers are expected with the user choosing between
-            them.
-            Kernels and toolchain components are common cases of this in
-            OpenEmbedded.
+            The <filename>PROVIDES</filename> list for this recipe becomes
+            "keyboard", which is implicit, and "fullkeyboard", which is explicit.
+            Consequently, the functionality found in
+            <filename>keyboard_1.0.bb</filename> can be found under two
+            different names.
+        </para>
+    </section>
+
+    <section id='bb-bitbake-preferences'>
+        <title>Preferences</title>
+
+        <para>
+            The <filename>PROVIDES</filename> list is only part of the solution
+            for figuring out a target's recipes.
+            Because targets might have multiple providers, BitBake needs
+            to prioritize providers by determining provider preferences.
         </para>
 
         <para>
-            Sometimes a target might have multiple providers.
-            A common example is "virtual/kernel", which is provided by each
-            kernel recipe.
+            A common example in which a target has multiple providers
+            is "virtual/kernel", which is on the
+            <filename>PROVIDES</filename> list for each kernel recipe.
             Each machine often selects the best kernel provider by using a
             line similar to the following in the machine configuration file:
             <literallayout class='monospaced'>
@@ -377,7 +414,7 @@
 
         <para>
             Understanding how providers are chosen is made complicated by the fact
-            that multiple versions might exist.
+            that multiple versions might exist for a given provider.
             BitBake defaults to the highest version of a provider.
             Version comparisons are made using the same method as Debian.
             You can use the
@@ -386,13 +423,19 @@
             You can influence the order by using the
             <link linkend='var-DEFAULT_PREFERENCE'><filename>DEFAULT_PREFERENCE</filename></link>
             variable.
+        </para>
+
+        <para>
             By default, files have a preference of "0".
-            Setting the <filename>DEFAULT_PREFERENCE</filename> to "-1" makes the
+            Setting <filename>DEFAULT_PREFERENCE</filename> to "-1" makes the
             recipe unlikely to be used unless it is explicitly referenced.
-            Setting the <filename>DEFAULT_PREFERENCE</filename> to "1" makes it likely the recipe is used.
-            <filename>PREFERRED_VERSION</filename> overrides any <filename>DEFAULT_PREFERENCE</filename> setting.
-            <filename>DEFAULT_PREFERENCE</filename> is often used to mark newer and more experimental recipe
-            versions until they have undergone sufficient testing to be considered stable.
+            Setting <filename>DEFAULT_PREFERENCE</filename> to "1" makes it
+            likely the recipe is used.
+            <filename>PREFERRED_VERSION</filename> overrides any
+            <filename>DEFAULT_PREFERENCE</filename> setting.
+            <filename>DEFAULT_PREFERENCE</filename> is often used to mark newer
+            and more experimental recipe versions until they have undergone
+            sufficient testing to be considered stable.
         </para>
 
         <para>
@@ -401,18 +444,16 @@
             version, unless otherwise specified.
             If the recipe in question has a
             <link linkend='var-DEFAULT_PREFERENCE'><filename>DEFAULT_PREFERENCE</filename></link>
-            set lower than
-            the other recipes (default is 0), then it will not be
-            selected.
+            set lower than the other recipes (default is 0), then
+            it will not be selected.
             This allows the person or persons maintaining
             the repository of recipe files to specify
             their preference for the default selected version.
-            In addition, the user can specify their preferred version.
+            Additionally, the user can specify their preferred version.
         </para>
 
         <para>
-            If the first recipe is named <filename>a_1.1.bb</filename>,
-            then the
+            If the first recipe is named <filename>a_1.1.bb</filename>, then the
             <link linkend='var-PN'><filename>PN</filename></link> variable
             will be set to “a”, and the
             <link linkend='var-PV'><filename>PV</filename></link>
@@ -420,19 +461,38 @@
         </para>
 
         <para>
-            If we then have a recipe named <filename>a_1.2.bb</filename>, BitBake
+            Thus, if a recipe named <filename>a_1.2.bb</filename> exists, BitBake
             will choose 1.2 by default.
-            However, if we define the following variable in a
-            <filename>.conf</filename> file that BitBake parses, we
-            can change that.
+            However, if you define the following variable in a
+            <filename>.conf</filename> file that BitBake parses, you
+            can change that preference:
             <literallayout class='monospaced'>
      PREFERRED_VERSION_a = "1.1"
             </literallayout>
         </para>
 
-        <para>
-            In summary, BitBake has created a list of providers, which is prioritized, for each target.
-        </para>
+        <note>
+            <para>
+                It is common for a recipe to provide two versions -- a stable,
+                numbered (and preferred) version, and a version that is
+                automatically checked out from a source code repository that
+                is considered more "bleeding edge" but can be selected only
+                explicitly.
+            </para>
+
+            <para>
+                For example, in the OpenEmbedded codebase, there is a standard,
+                versioned recipe file for BusyBox,
+                <filename>busybox_1.22.1.bb</filename>,
+                but there is also a Git-based version,
+                <filename>busybox_git.bb</filename>, which explicitly contains the line
+                <literallayout class='monospaced'>
+    DEFAULT_PREFERENCE = "-1"
+                </literallayout>
+                to ensure that the numbered, stable version is always preferred
+                unless the developer selects otherwise.
+            </para>
+        </note>
     </section>
 
     <section id='bb-bitbake-dependencies'>
@@ -495,7 +555,7 @@
             As each task completes, a timestamp is written to the directory specified by the
             <link linkend='var-STAMP'><filename>STAMP</filename></link> variable.
             On subsequent runs, BitBake looks in the build directory within
-            <filename>tmp/stamps</filename>and does not rerun
+            <filename>tmp/stamps</filename> and does not rerun
             tasks that are already completed unless a timestamp is found to be invalid.
             Currently, invalid timestamps are only considered on a per
             recipe file basis.
@@ -535,7 +595,7 @@
         <title>Executing Tasks</title>
 
         <para>
-            Tasks can either be a shell task or a Python task.
+            Tasks can be either a shell task or a Python task.
             For shell tasks, BitBake writes a shell script to
             <filename>${</filename><link linkend='var-T'><filename>T</filename></link><filename>}/run.do_taskname.pid</filename>
             and then executes the script.
@@ -755,9 +815,9 @@
             to determine the stamps and delta where these two
             stamp trees diverge.
             <note>
-                It is likely that future versions of BitBake with
+                It is likely that future versions of BitBake will
                 provide other signature handlers triggered through
-                additional "-S" paramters.
+                additional "-S" parameters.
             </note>
         </para>
 

bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml

@@ -8,14 +8,14 @@
         BitBake's fetch module is a standalone piece of library code
         that deals with the intricacies of downloading source code
         and files from remote systems.
-        Fetching source code is one of the corner stones of building software.
+        Fetching source code is one of the cornerstones of building software.
         As such, this module forms an important part of BitBake.
     </para>
 
     <para>
         The current fetch module is called "fetch2" and refers to the
         fact that it is the second major version of the API.
-        The original version is obsolete and removed from the codebase.
+        The original version is obsolete and has been removed from the codebase.
         Thus, in all cases, "fetch" refers to "fetch2" in this
         manual.
     </para>
@@ -60,17 +60,19 @@
             <note>
                 For convenience, the naming in these examples matches
                 the variables used by OpenEmbedded.
+                If you want to see the above code in action, examine
+                the OpenEmbedded class file <filename>base.bbclass</filename>.
             </note>
             The <filename>SRC_URI</filename> and <filename>WORKDIR</filename>
-            variables are not coded into the fetcher.
-            They variables can (and are) called with different variable names.
+            variables are not hardcoded into the fetcher, since those fetcher
+            methods can be (and are) called with different variable names.
             In OpenEmbedded for example, the shared state (sstate) code uses
             the fetch module to fetch the sstate files.
         </para>
 
         <para>
             When the <filename>download()</filename> method is called,
-            BitBake tries to fulfill the URLs by looking for source files
+            BitBake tries to resolve the URLs by looking for source files
             in a specific search order:
             <itemizedlist>
                 <listitem><para><emphasis>Pre-mirror Sites:</emphasis>
@@ -84,7 +86,7 @@
                     <filename>SRC_URI</filename>).
                     </para></listitem>
                 <listitem><para><emphasis>Mirror Sites:</emphasis>
-                    If fetch failures occur, BitBake next uses mirror location as
+                    If fetch failures occur, BitBake next uses mirror locations as
                     defined by the
                     <link linkend='var-MIRRORS'><filename>MIRRORS</filename></link>
                     variable.
@@ -149,7 +151,7 @@
         <para>
             File integrity is of key importance for reproducing builds.
             For non-local archive downloads, the fetcher code can verify
-            sha256 and md5 checksums to ensure the archives have been
+            SHA-256 and MD5 checksums to ensure the archives have been
             downloaded correctly.
             You can specify these checksums by using the
             <filename>SRC_URI</filename> variable with the appropriate
@@ -260,8 +262,8 @@
             <para>
                 This submodule handles URLs that begin with
                 <filename>file://</filename>.
-                The filename you specify with in the URL can
-                either be an absolute or relative path to a file.
+                The filename you specify within the URL can be
+                either an absolute or relative path to a file.
                 If the filename is relative, the contents of the
                 <link linkend='var-FILESPATH'><filename>FILESPATH</filename></link>
                 variable is used in the same way
@@ -286,15 +288,45 @@
             </para>
 
             <para>
-                Here are some example URLs:
+                Here are a couple of example URLs, the first relative and
+                the second absolute:
                 <literallayout class='monospaced'>
      SRC_URI = "file://relativefile.patch"
-     SRC_URI = "file://relativefile.patch;this=ignored"
      SRC_URI = "file:///Users/ich/very_important_software"
                 </literallayout>
             </para>
         </section>
 
+        <section id='http-ftp-fetcher'>
+            <title>HTTP/FTP wget fetcher (<filename>http://</filename>, <filename>ftp://</filename>, <filename>https://</filename>)</title>
+
+            <para>
+                This fetcher obtains files from web and FTP servers.
+                Internally, the fetcher uses the wget utility.
+            </para>
+
+            <para>
+                The executable and parameters used are specified by the
+                <filename>FETCHCMD_wget</filename> variable, which defaults
+                to sensible values.
+                The fetcher supports a parameter "downloadfilename" that
+                allows the name of the downloaded file to be specified.
+                Specifying the name of the downloaded file is useful
+                for avoiding collisions in
+                <link linkend='var-DL_DIR'><filename>DL_DIR</filename></link>
+                when dealing with multiple files that have the same name.
+            </para>
+
+            <para>
+                Some example URLs are as follows:
+                <literallayout class='monospaced'>
+     SRC_URI = "http://oe.handhelds.org/not_there.aac"
+     SRC_URI = "ftp://oe.handhelds.org/not_there_as_well.aac"
+     SRC_URI = "ftp://you@oe.handhelds.org/home/you/secret.plan"
+                </literallayout>
+            </para>
+        </section>
+
         <section id='cvs-fetcher'>
             <title>CVS fetcher (<filename>(cvs://</filename>)</title>
 
@@ -334,7 +366,7 @@
                 The supported parameters are as follows:
                 <itemizedlist>
                     <listitem><para><emphasis>"method":</emphasis>
-                        The protocol over which to communicate with the cvs server.
+                        The protocol over which to communicate with the CVS server.
                         By default, this protocol is "pserver".
                         If "method" is set to "ext", BitBake examines the
                         "rsh" parameter and sets <filename>CVS_RSH</filename>.
@@ -394,36 +426,6 @@
             </para>
         </section>
 
-        <section id='http-ftp-fetcher'>
-            <title>HTTP/FTP wget fetcher (<filename>http://</filename>, <filename>ftp://</filename>, <filename>https://</filename>)</title>
-
-            <para>
-                This fetcher obtains files from web and FTP servers.
-                Internally, the fetcher uses the wget utility.
-            </para>
-
-            <para>
-                The executable and parameters used are specified by the
-                <filename>FETCHCMD_wget</filename> variable, which defaults
-                to a sensible values.
-                The fetcher supports a parameter "downloadfilename" that
-                allows the name of the downloaded file to be specified.
-                Specifying the name of the downloaded file is useful
-                for avoiding collisions in
-                <link linkend='var-DL_DIR'><filename>DL_DIR</filename></link>
-                when dealing with multiple files that have the same name.
-            </para>
-
-            <para>
-                Some example URLs are as follows:
-                <literallayout class='monospaced'>
-     SRC_URI = "http://oe.handhelds.org/not_there.aac"
-     SRC_URI = "ftp://oe.handhelds.org/not_there_as_well.aac"
-     SRC_URI = "ftp://you@oe.handheld.sorg/home/you/secret.plan"
-                </literallayout>
-            </para>
-        </section>
-
         <section id='svn-fetcher'>
             <title>Subversion (SVN) Fetcher (<filename>svn://</filename>)</title>
 
@@ -466,6 +468,13 @@
                         compile-time when set to "keep".
                         By default, these directories are removed.
                         </para></listitem>
+                    <listitem><para><emphasis>"transportuser":</emphasis>
+                        When required, sets the username for the transport.
+                        By default, this parameter is empty.
+                        The transport username is different than the username
+                        used in the main URL, which is passed to the subversion
+                        command.
+                        </para></listitem>
                 </itemizedlist>
                 Following are two examples using svn:
                 <literallayout class='monospaced'>
@@ -476,14 +485,14 @@
         </section>
 
         <section id='git-fetcher'>
-            <title>GIT Fetcher (<filename>git://</filename>)</title>
+            <title>Git Fetcher (<filename>git://</filename>)</title>
 
             <para>
                 This fetcher submodule fetches code from the Git
                 source control system.
                 The fetcher works by creating a bare clone of the
                 remote into <filename>GITDIR</filename>, which is
-                usually <filename>DL_DIR/git</filename>.
+                usually <filename>DL_DIR/git2</filename>.
                 This bare clone is then cloned into the work directory during the
                 unpack stage when a specific tree is checked out.
                 This is done using alternates and by reference to
@@ -551,7 +560,7 @@
                         network.
                         For that reason, tags are often not used.
                         As far as Git is concerned, the "tag" parameter behaves
-                        effectively the same as the "revision" parameter.
+                        effectively the same as the "rev" parameter.
                         </para></listitem>
                     <listitem><para><emphasis>"subpath":</emphasis>
                         Limits the checkout to a specific subpath of the tree.
@@ -570,6 +579,116 @@
             </para>
         </section>
 
+        <section id='gitsm-fetcher'>
+            <title>Git Submodule Fetcher (<filename>gitsm://</filename>)</title>
+
+            <para>
+                This fetcher submodule inherits from the
+                <link linkend='git-fetcher'>Git fetcher</link> and extends
+                that fetcher's behavior by fetching a repository's submodules.
+                <link linkend='var-SRC_URI'><filename>SRC_URI</filename></link>
+                is passed to the Git fetcher as described in the
+                "<link linkend='git-fetcher'>Git Fetcher (<filename>git://</filename>)</link>"
+                section.
+                <note>
+                    <title>Notes and Warnings</title>
+                    <para>
+                        You must clean a recipe when switching between
+                        '<filename>git://</filename>' and
+                        '<filename>gitsm://</filename>' URLs.
+                    </para>
+
+                    <para>
+                        The Git Submodules fetcher is not a complete fetcher
+                        implementation.
+                        The fetcher has known issues where it does not use the
+                        normal source mirroring infrastructure properly.
+                    </para>
+                </note>
+            </para>
+        </section>
+
+        <section id='clearcase-fetcher'>
+            <title>ClearCase Fetcher (<filename>ccrc://</filename>)</title>
+
+            <para>
+                This fetcher submodule fetches code from a
+                <ulink url='http://en.wikipedia.org/wiki/Rational_ClearCase'>ClearCase</ulink>
+                repository.
+            </para>
+
+            <para>
+                To use this fetcher, make sure your recipe has proper
+                <link linkend='var-SRC_URI'><filename>SRC_URI</filename></link>,
+                <link linkend='var-SRCREV'><filename>SRCREV</filename></link>, and
+                <link linkend='var-PV'><filename>PV</filename></link> settings.
+                Here is an example:
+                <literallayout class='monospaced'>
+     SRC_URI = "ccrc://cc.example.org/ccrc;vob=/example_vob;module=/example_module"
+     SRCREV = "EXAMPLE_CLEARCASE_TAG"
+     PV = "${@d.getVar("SRCREV").replace("/", "+")}"
+                </literallayout>
+                The fetcher uses the <filename>rcleartool</filename> or
+                <filename>cleartool</filename> remote client, depending on
+                which one is available.
+            </para>
+
+            <para>
+                Following are options for the <filename>SRC_URI</filename>
+                statement:
+                <itemizedlist>
+                    <listitem><para><emphasis><filename>vob</filename></emphasis>:
+                        The name, which must include the
+                        prepending "/" character, of the ClearCase VOB.
+                        This option is required.
+                        </para></listitem>
+                    <listitem><para><emphasis><filename>module</filename></emphasis>:
+                        The module, which must include the
+                        prepending "/" character, in the selected VOB
+                        The <filename>module</filename> and <filename>vob</filename>
+                        options are combined to create the following load rule in
+                        the view config spec:
+                        <literallayout class='monospaced'>
+     load &lt;vob&gt;&lt;module&gt;
+                        </literallayout>
+                        </para></listitem>
+                    <listitem><para><emphasis><filename>proto</filename></emphasis>:
+                        The protocol, which can be either <filename>http</filename> or
+                        <filename>https</filename>.
+                        </para></listitem>
+                </itemizedlist>
+            </para>
+
+            <para>
+                By default, the fetcher creates a configuration specification.
+                If you want this specification written to an area other than the default,
+                use the <filename>CCASE_CUSTOM_CONFIG_SPEC</filename> variable
+                in your recipe to define where the specification is written.
+                <note>
+                    the <filename>SRCREV</filename> loses its functionality if you
+                    specify this variable.
+                    However, <filename>SRCREV</filename> is still used to label the
+                    archive after a fetch even though it does not define what is
+                    fetched.
+                </note>
+            </para>
+
+            <para>
+                Here are a couple of other behaviors worth mentioning:
+                <itemizedlist>
+                    <listitem><para>
+                        When using <filename>cleartool</filename>, the login of
+                        <filename>cleartool</filename> is handled by the system.
+                        The login require no special steps.
+                        </para></listitem>
+                    <listitem><para>
+                        In order to use <filename>rcleartool</filename> with authenticated
+                        users, an "rcleartool login" is necessary before using the fetcher.
+                        </para></listitem>
+                </itemizedlist>
+            </para>
+        </section>
+
         <section id='other-fetchers'>
             <title>Other Fetchers</title>
 
@@ -582,9 +701,6 @@
                     <listitem><para>
                         Perforce (<filename>p4://</filename>)
                         </para></listitem>
-                    <listitem><para>
-                        Git Submodules (<filename>gitsm://</filename>)
-                        </para></listitem>
                     <listitem><para>
                         Trees using Git Annex (<filename>gitannex://</filename>)
                         </para></listitem>

bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.xml

@@ -0,0 +1,506 @@
+<!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
+"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd">
+
+<appendix id='hello-world-example'>
+    <title>Hello World Example</title>
+
+    <section id='bitbake-hello-world'>
+        <title>BitBake Hello World</title>
+
+        <para>
+            The simplest example commonly used to demonstrate any new
+            programming language or tool is the
+            "<ulink url="http://en.wikipedia.org/wiki/Hello_world_program">Hello World</ulink>"
+            example.
+            This appendix demonstrates, in tutorial form, Hello
+            World within the context of BitBake.
+            The tutorial describes how to create a new project
+            and the applicable metadata files necessary to allow
+            BitBake to build it.
+        </para>
+    </section>
+
+    <section id='example-obtaining-bitbake'>
+        <title>Obtaining BitBake</title>
+
+        <para>
+            See the
+            "<link linkend='obtaining-bitbake'>Obtaining BitBake</link>"
+            section for information on how to obtain BitBake.
+            Once you have the source code on your machine, the BitBake directory
+            appears as follows:
+            <literallayout class='monospaced'>
+     $ ls -al
+     total 100
+     drwxrwxr-x. 9 wmat wmat  4096 Jan 31 13:44 .
+     drwxrwxr-x. 3 wmat wmat  4096 Feb  4 10:45 ..
+     -rw-rw-r--. 1 wmat wmat   365 Nov 26 04:55 AUTHORS
+     drwxrwxr-x. 2 wmat wmat  4096 Nov 26 04:55 bin
+     drwxrwxr-x. 4 wmat wmat  4096 Jan 31 13:44 build
+     -rw-rw-r--. 1 wmat wmat 16501 Nov 26 04:55 ChangeLog
+     drwxrwxr-x. 2 wmat wmat  4096 Nov 26 04:55 classes
+     drwxrwxr-x. 2 wmat wmat  4096 Nov 26 04:55 conf
+     drwxrwxr-x. 3 wmat wmat  4096 Nov 26 04:55 contrib
+     -rw-rw-r--. 1 wmat wmat 17987 Nov 26 04:55 COPYING
+     drwxrwxr-x. 3 wmat wmat  4096 Nov 26 04:55 doc
+     -rw-rw-r--. 1 wmat wmat    69 Nov 26 04:55 .gitignore
+     -rw-rw-r--. 1 wmat wmat   849 Nov 26 04:55 HEADER
+     drwxrwxr-x. 5 wmat wmat  4096 Jan 31 13:44 lib
+     -rw-rw-r--. 1 wmat wmat   195 Nov 26 04:55 MANIFEST.in
+     -rwxrwxr-x. 1 wmat wmat  3195 Jan 31 11:57 setup.py
+     -rw-rw-r--. 1 wmat wmat  2887 Nov 26 04:55 TODO
+            </literallayout>
+        </para>
+
+        <para>
+            At this point, you should have BitBake cloned to
+            a directory that matches the previous listing except for
+            dates and user names.
+        </para>
+    </section>
+
+    <section id='setting-up-the-bitbake-environment'>
+        <title>Setting Up the BitBake Environment</title>
+
+        <para>
+            First, you need to be sure that you can run BitBake.
+            Set your working directory to where your local BitBake
+            files are and run the following command:
+            <literallayout class='monospaced'>
+     $ ./bin/bitbake --version
+     BitBake Build Tool Core version 1.23.0, bitbake version 1.23.0
+            </literallayout>
+            The console output tells you what version you are running.
+        </para>
+
+        <para>
+            The recommended method to run BitBake is from a directory of your
+            choice.
+            To be able to run BitBake from any directory, you need to add the
+            executable binary to your binary to your shell's environment
+            <filename>PATH</filename> variable.
+            First, look at your current <filename>PATH</filename> variable
+            by entering the following:
+            <literallayout class='monospaced'>
+     $ echo $PATH
+            </literallayout>
+            Next, add the directory location for the BitBake binary to the
+            <filename>PATH</filename>.
+            Here is an example that adds the
+            <filename>/home/scott-lenovo/bitbake/bin</filename> directory
+            to the front of the <filename>PATH</filename> variable:
+            <literallayout class='monospaced'>
+     $ export PATH=/home/scott-lenovo/bitbake/bin:$PATH
+            </literallayout>
+            You should now be able to enter the <filename>bitbake</filename>
+            command from the command line while working from any directory.
+        </para>
+    </section>
+
+    <section id='the-hello-world-example'>
+        <title>The Hello World Example</title>
+
+        <para>
+            The overall goal of this exercise is to build a
+            complete "Hello World" example utilizing task and layer
+            concepts.
+            Because this is how modern projects such as OpenEmbedded and
+            the Yocto Project utilize BitBake, the example
+            provides an excellent starting point for understanding
+            BitBake.
+        </para>
+
+        <para>
+            To help you understand how to use BitBake to build targets,
+            the example starts with nothing but the <filename>bitbake</filename>
+            command, which causes BitBake to fail and report problems.
+            The example progresses by adding pieces to the build to
+            eventually conclude with a working, minimal "Hello World"
+            example.
+        </para>
+
+        <para>
+            While every attempt is made to explain what is happening during
+            the example, the descriptions cannot cover everything.
+            You can find further information throughout this manual.
+            Also, you can actively participate in the
+            <ulink url='http://lists.openembedded.org/mailman/listinfo/bitbake-devel'></ulink>
+            discussion mailing list about the BitBake build tool.
+        </para>
+
+        <note>
+            This example was inspired by and drew heavily from these sources:
+            <itemizedlist>
+                <listitem><para>
+                    <ulink url="http://www.mail-archive.com/yocto@yoctoproject.org/msg09379.html">Mailing List post - The BitBake equivalent of "Hello, World!"</ulink>
+                    </para></listitem>
+                <listitem><para>
+                    <ulink url="http://hambedded.org/blog/2012/11/24/from-bitbake-hello-world-to-an-image/">Hambedded Linux blog post - From Bitbake Hello World to an Image</ulink>
+                    </para></listitem>
+            </itemizedlist>
+        </note>
+
+        <para>
+            As stated earlier, the goal of this example
+            is to eventually compile "Hello World".
+            However, it is unknown what BitBake needs and what you have
+            to provide in order to achieve that goal.
+            Recall that BitBake utilizes three types of metadata files:
+            <link linkend='configuration-files'>Configuration Files</link>,
+            <link linkend='classes'>Classes</link>, and
+            <link linkend='recipes'>Recipes</link>.
+            But where do they go?
+            How does BitBake find them?
+            BitBake's error messaging helps you answer these types of questions
+            and helps you better understand exactly what is going on.
+        </para>
+
+        <para>
+            Following is the complete "Hello World" example.
+        </para>
+
+        <orderedlist>
+            <listitem><para><emphasis>Create a Project Directory:</emphasis>
+                First, set up a directory for the "Hello World" project.
+                Here is how you can do so in your home directory:
+                <literallayout class='monospaced'>
+     $ mkdir ~/hello
+     $ cd ~/hello
+                </literallayout>
+                This is the directory that BitBake will use to do all of
+                its work.
+                You can use this directory to keep all the metafiles needed
+                by BitBake.
+                Having a project directory is a good way to isolate your
+                project.
+                </para></listitem>
+            <listitem><para><emphasis>Run Bitbake:</emphasis>
+                At this point, you have nothing but a project directory.
+                Run the <filename>bitbake</filename> command and see what
+                it does:
+                <literallayout class='monospaced'>
+     $ bitbake
+     The BBPATH variable is not set and bitbake did not
+     find a conf/bblayers.conf file in the expected location.
+     Maybe you accidentally invoked bitbake from the wrong directory?
+     DEBUG: Removed the following variables from the environment:
+     GNOME_DESKTOP_SESSION_ID, XDG_CURRENT_DESKTOP,
+     GNOME_KEYRING_CONTROL, DISPLAY, SSH_AGENT_PID, LANG, no_proxy,
+     XDG_SESSION_PATH, XAUTHORITY, SESSION_MANAGER, SHLVL,
+     MANDATORY_PATH, COMPIZ_CONFIG_PROFILE, WINDOWID, EDITOR,
+     GPG_AGENT_INFO, SSH_AUTH_SOCK, GDMSESSION, GNOME_KEYRING_PID,
+     XDG_SEAT_PATH, XDG_CONFIG_DIRS, LESSOPEN, DBUS_SESSION_BUS_ADDRESS,
+     _, XDG_SESSION_COOKIE, DESKTOP_SESSION, LESSCLOSE, DEFAULTS_PATH,
+     UBUNTU_MENUPROXY, OLDPWD, XDG_DATA_DIRS, COLORTERM, LS_COLORS
+                </literallayout>
+                The majority of this output is specific to environment variables
+                that are not directly relevant to BitBake.
+                However, the very first message regarding the
+                <filename>BBPATH</filename> variable and the
+                <filename>conf/bblayers.conf</filename> file
+                is relevant.</para>
+                <para>
+                When you run BitBake, it begins looking for metadata files.
+                The
+                <link linkend='var-BBPATH'><filename>BBPATH</filename></link>
+                variable is what tells BitBake where to look for those files.
+                <filename>BBPATH</filename> is not set and you need to set it.
+                Without <filename>BBPATH</filename>, Bitbake cannot
+                find any configuration files (<filename>.conf</filename>)
+                or recipe files (<filename>.bb</filename>) at all.
+                BitBake also cannot find the <filename>bitbake.conf</filename>
+                file.
+                </para></listitem>
+            <listitem><para><emphasis>Setting <filename>BBPATH</filename>:</emphasis>
+                For this example, you can set <filename>BBPATH</filename>
+                in the same manner that you set <filename>PATH</filename>
+                earlier in the appendix.
+                You should realize, though, that it is much more flexible to set the
+                <filename>BBPATH</filename> variable up in a configuration
+                file for each project.</para>
+                <para>From your shell, enter the following commands to set and
+                export the <filename>BBPATH</filename> variable:
+                <literallayout class='monospaced'>
+     $ BBPATH="&lt;projectdirectory&gt;"
+     $ export BBPATH
+                </literallayout>
+                Use your actual project directory in the command.
+                BitBake uses that directory to find the metadata it needs for
+                your project.
+                <note>
+                    When specifying your project directory, do not use the
+                    tilde ("~") character as BitBake does not expand that character
+                    as the shell would.
+                </note>
+                </para></listitem>
+            <listitem><para><emphasis>Run Bitbake:</emphasis>
+                Now that you have <filename>BBPATH</filename> defined, run
+                the <filename>bitbake</filename> command again:
+                <literallayout class='monospaced'>
+     $ bitbake
+     ERROR: Traceback (most recent call last):
+       File "/home/scott-lenovo/bitbake/lib/bb/cookerdata.py", line 163, in wrapped
+         return func(fn, *args)
+       File "/home/scott-lenovo/bitbake/lib/bb/cookerdata.py", line 173, in parse_config_file
+         return bb.parse.handle(fn, data, include)
+       File "/home/scott-lenovo/bitbake/lib/bb/parse/__init__.py", line 99, in handle
+         return h['handle'](fn, data, include)
+       File "/home/scott-lenovo/bitbake/lib/bb/parse/parse_py/ConfHandler.py", line 120, in handle
+         abs_fn = resolve_file(fn, data)
+       File "/home/scott-lenovo/bitbake/lib/bb/parse/__init__.py", line 117, in resolve_file
+         raise IOError("file %s not found in %s" % (fn, bbpath))
+     IOError: file conf/bitbake.conf not found in /home/scott-lenovo/hello
+
+     ERROR: Unable to parse conf/bitbake.conf: file conf/bitbake.conf not found in /home/scott-lenovo/hello
+                </literallayout>
+                This sample output shows that BitBake could not find the
+                <filename>conf/bitbake.conf</filename> file in the project
+                directory.
+                This file is the first thing BitBake must find in order
+                to build a target.
+                And, since the project directory for this example is
+                empty, you need to provide a <filename>conf/bitbake.conf</filename>
+                file.
+                </para></listitem>
+            <listitem><para><emphasis>Creating <filename>conf/bitbake.conf</filename>:</emphasis>
+                The <filename>conf/bitbake.conf</filename> includes a number of
+                configuration variables BitBake uses for metadata and recipe
+                files.
+                For this example, you need to create the file in your project directory
+                and define some key BitBake variables.
+                For more information on the <filename>bitbake.conf</filename>,
+                see
+                <ulink url='http://hambedded.org/blog/2012/11/24/from-bitbake-hello-world-to-an-image/#an-overview-of-bitbakeconf'></ulink>
+                </para>
+                <para>Use the following commands to create the <filename>conf</filename>
+                directory in the project directory:
+                <literallayout class='monospaced'>
+     $ mkdir conf
+                </literallayout>
+                From within the <filename>conf</filename> directory, use
+                some editor to create the <filename>bitbake.conf</filename>
+                so that it contains the following:
+                <literallayout class='monospaced'>
+     TMPDIR  = "${<link linkend='var-TOPDIR'>TOPDIR</link>}/tmp"
+     <link linkend='var-CACHE'>CACHE</link>   = "${TMPDIR}/cache"
+     <link linkend='var-STAMP'>STAMP</link>   = "${TMPDIR}/stamps"
+     <link linkend='var-T'>T</link>       = "${TMPDIR}/work"
+     <link linkend='var-B'>B</link>       = "${TMPDIR}"
+                </literallayout>
+                The <filename>TMPDIR</filename> variable establishes a directory
+                that BitBake uses for build output and intermediate files (other
+                than the cached information used by the
+                <link linkend='setscene'>Setscene</link> process.
+                Here, the <filename>TMPDIR</filename> directory is set to
+                <filename>hello/tmp</filename>.
+                <note><title>Tip</title>
+                    You can always safely delete the <filename>tmp</filename>
+                    directory in order to rebuild a BitBake target.
+                    The build process creates the directory for you
+                    when you run BitBake.
+                </note></para>
+                <para>For information about each of the other variables defined in this
+                example, click on the links to take you to the definitions in
+                the glossary.
+                </para></listitem>
+            <listitem><para><emphasis>Run Bitbake:</emphasis>
+                After making sure that the <filename>conf/bitbake.conf</filename>
+                file exists, you can run the <filename>bitbake</filename>
+                command again:
+                <literallayout class='monospaced'>
+$ bitbake
+ERROR: Traceback (most recent call last):
+  File "/home/scott-lenovo/bitbake/lib/bb/cookerdata.py", line 163, in wrapped
+    return func(fn, *args)
+  File "/home/scott-lenovo/bitbake/lib/bb/cookerdata.py", line 177, in _inherit
+    bb.parse.BBHandler.inherit(bbclass, "configuration INHERITs", 0, data)
+  File "/home/scott-lenovo/bitbake/lib/bb/parse/parse_py/BBHandler.py", line 92, in inherit
+    include(fn, file, lineno, d, "inherit")
+  File "/home/scott-lenovo/bitbake/lib/bb/parse/parse_py/ConfHandler.py", line 100, in include
+    raise ParseError("Could not %(error_out)s file %(fn)s" % vars(), oldfn, lineno)
+ParseError: ParseError in configuration INHERITs: Could not inherit file classes/base.bbclass
+
+ERROR: Unable to parse base: ParseError in configuration INHERITs: Could not inherit file classes/base.bbclass
+                </literallayout>
+                In the sample output, BitBake could not find the
+                <filename>classes/base.bbclass</filename> file.
+                You need to create that file next.
+                </para></listitem>
+            <listitem><para><emphasis>Creating <filename>classes/base.bbclass</filename>:</emphasis>
+                BitBake uses class files to provide common code and functionality.
+                The minimally required class for BitBake is the
+                <filename>classes/base.bbclass</filename> file.
+                The <filename>base</filename> class is implicitly inherited by
+                every recipe.
+                BitBake looks for the class in the <filename>classes</filename>
+                directory of the project (i.e <filename>hello/classes</filename>
+                in this example).
+                </para>
+                <para>Create the <filename>classes</filename> directory as follows:
+                <literallayout class='monospaced'>
+     $ cd $HOME/hello
+     $ mkdir classes
+                </literallayout>
+                Move to the <filename>classes</filename> directory and then
+                create the <filename>base.bbclass</filename> file by inserting
+                this single line:
+                <literallayout class='monospaced'>
+     addtask build
+                </literallayout>
+                The minimal task that BitBake runs is the
+                <filename>do_build</filename> task.
+                This is all the example needs in order to build the project.
+                Of course, the <filename>base.bbclass</filename> can have much
+                more depending on which build environments BitBake is
+                supporting.
+                For more information on the <filename>base.bbclass</filename> file,
+                you can look at
+                <ulink url='http://hambedded.org/blog/2012/11/24/from-bitbake-hello-world-to-an-image/#tasks'></ulink>.
+                </para></listitem>
+            <listitem><para><emphasis>Run Bitbake:</emphasis>
+                After making sure that the <filename>classes/base.bbclass</filename>
+                file exists, you can run the <filename>bitbake</filename>
+                command again:
+                <literallayout class='monospaced'>
+     $ bitbake
+     Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.
+                </literallayout>
+                BitBake is finally reporting no errors.
+                However, you can see that it really does not have anything
+                to do.
+                You need to create a recipe that gives BitBake something to do.
+                </para></listitem>
+            <listitem><para><emphasis>Creating a Layer:</emphasis>
+                While it is not really necessary for such a small example,
+                it is good practice to create a layer in which to keep your
+                code separate from the general metadata used by BitBake.
+                Thus, this example creates and uses a layer called "mylayer".
+                <note>
+                    You can find additional information on adding a layer at
+                    <ulink url='http://hambedded.org/blog/2012/11/24/from-bitbake-hello-world-to-an-image/#adding-an-example-layer'></ulink>.
+                </note>
+                </para>
+                <para>Minimally, you need a recipe file and a layer configuration
+                file in your layer.
+                The configuration file needs to be in the <filename>conf</filename>
+                directory inside the layer.
+                Use these commands to set up the layer and the <filename>conf</filename>
+                directory:
+                <literallayout class='monospaced'>
+     $ cd $HOME
+     $ mkdir mylayer
+     $ cd mylayer
+     $ mkdir conf
+                </literallayout>
+                Move to the <filename>conf</filename> directory and create a
+                <filename>layer.conf</filename> file that has the following:
+                <literallayout class='monospaced'>
+     BBPATH .= ":${<link linkend='var-LAYERDIR'>LAYERDIR</link>}"
+
+     <link linkend='var-BBFILES'>BBFILES</link> += "${LAYERDIR}/*.bb"
+
+     <link linkend='var-BBFILE_COLLECTIONS'>BBFILE_COLLECTIONS</link> += "mylayer"
+     <link linkend='var-BBFILE_PATTERN'>BBFILE_PATTERN_mylayer</link> := "^${LAYERDIR}/"
+                </literallayout>
+                For information on these variables, click the links
+                to go to the definitions in the glossary.</para>
+                <para>You need to create the recipe file next.
+                Inside your layer at the top-level, use an editor and create
+                a recipe file named <filename>printhello.bb</filename> that
+                has the following:
+                <literallayout class='monospaced'>
+     <link linkend='var-DESCRIPTION'>DESCRIPTION</link> = "Prints Hello World"
+     <link linkend='var-PN'>PN</link> = 'printhello'
+     <link linkend='var-PV'>PV</link> = '1'
+
+     python do_build() {
+        bb.plain("********************");
+        bb.plain("*                  *");
+        bb.plain("*  Hello, World!   *");
+        bb.plain("*                  *");
+        bb.plain("********************");
+     }
+                </literallayout>
+                The recipe file simply provides a description of the
+                recipe, the name, version, and the <filename>do_build</filename>
+                task, which prints out "Hello World" to the console.
+                For more information on these variables, follow the links
+                to the glossary.
+                </para></listitem>
+            <listitem><para><emphasis>Run Bitbake With a Target:</emphasis>
+                Now that a BitBake target exists, run the command and provide
+                that target:
+                <literallayout class='monospaced'>
+     $ cd $HOME/hello
+     $ bitbake printhello
+     ERROR: no recipe files to build, check your BBPATH and BBFILES?
+
+     Summary: There was 1 ERROR message shown, returning a non-zero exit code.
+                </literallayout>
+                We have created the layer with the recipe and the layer
+                configuration file but it still seems that BitBake cannot
+                find the recipe.
+                BitBake needs a <filename>conf/bblayers.conf</filename> that
+                lists the layers for the project.
+                Without this file, BitBake cannot find the recipe.
+                </para></listitem>
+            <listitem><para><emphasis>Creating <filename>conf/bblayers.conf</filename>:</emphasis>
+                BitBake uses the <filename>conf/bblayers.conf</filename> file
+                to locate layers needed for the project.
+                This file must reside in the <filename>conf</filename> directory
+                of the project (i.e. <filename>hello/conf</filename> for this
+                example).</para>
+                <para>Set your working directory to the <filename>hello/conf</filename>
+                directory and then create the <filename>bblayers.conf</filename>
+                file so that it contains the following:
+                <literallayout class='monospaced'>
+     BBLAYERS ?= " \
+       /home/&lt;you&gt;/mylayer \
+       "
+                </literallayout>
+                You need to provide your own information for
+                <filename>you</filename> in the file.
+                </para></listitem>
+            <listitem><para><emphasis>Run Bitbake With a Target:</emphasis>
+                Now that you have supplied the <filename>bblayers.conf</filename>
+                file, run the <filename>bitbake</filename> command and provide
+                the target:
+                <literallayout class='monospaced'>
+     $ bitbake printhello
+     Parsing recipes: 100% |##################################################################################|
+     Time: 00:00:00
+     Parsing of 1 .bb files complete (0 cached, 1 parsed). 1 targets, 0 skipped, 0 masked, 0 errors.
+     NOTE: Resolving any missing task queue dependencies
+     NOTE: Preparing runqueue
+     NOTE: Executing RunQueue Tasks
+     ********************
+     *                  *
+     *  Hello, World!   *
+     *                  *
+     ********************
+     NOTE: Tasks Summary: Attempted 1 tasks of which 0 didn't need to be rerun and all succeeded.
+                </literallayout>
+                BitBake finds the <filename>printhello</filename> recipe and
+                successfully runs the task.
+                <note>
+                    After the first execution, re-running
+                    <filename>bitbake printhello</filename> again will not
+                    result in a BitBake run that prints the same console
+                    output.
+                    The reason for this is that the first time the
+                    <filename>printhello.bb</filename> recipe's
+                    <filename>do_build</filename> task executes
+                    successfully, BitBake writes a stamp file for the task.
+                    Thus, the next time you attempt to run the task
+                    using that same <filename>bitbake</filename> command,
+                    BitBake notices the stamp and therefore determines
+                    that the task does not need to be re-run.
+                    If you delete the <filename>tmp</filename> directory
+                    or run <filename>bitbake -c clean printhello</filename>
+                    and then re-run the build, the "Hello, World!" message will
+                    be printed again.
+                </note>
+                </para></listitem>
+        </orderedlist>
+    </section>
+</appendix>

bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.xml

@@ -1,16 +1,16 @@
 <!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
     "http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd">
 
-<chapter id="user-manual-intro">
+<chapter id="bitbake-user-manual-intro">
     <title>Overview</title>
 
     <para>
         Welcome to the BitBake User Manual.
         This manual provides information on the BitBake tool.
         The information attempts to be as independent as possible regarding
-        systems that use BitBake, such as the Yocto Project and
-        OpenEmbedded.
-        In some cases, scenarios or examples that within the context of
+        systems that use BitBake, such as OpenEmbedded and the
+        Yocto Project.
+        In some cases, scenarios or examples within the context of
         a build system are used in the manual to help with understanding.
         For these cases, the manual clearly states the context.
     </para>
@@ -35,28 +35,31 @@
                 <listitem><para>
                     BitBake executes tasks according to provided
                     metadata that builds up the tasks.
-                    Metadata is stored in recipe (<filename>.bb</filename>),
-                    configuration (<filename>.conf</filename>), and class
-                    (<filename>.bbclass</filename>) files and provides
+                    Metadata is stored in recipe (<filename>.bb</filename>)
+                    and related recipe "append" (<filename>.bbappend</filename>)
+                    files, configuration (<filename>.conf</filename>) and
+                    underlying include (<filename>.inc</filename>) files, and
+                    in class (<filename>.bbclass</filename>) files.
+                    The metadata provides
                     BitBake with instructions on what tasks to run and
                     the dependencies between those tasks.
                     </para></listitem>
                 <listitem><para>
                     BitBake includes a fetcher library for obtaining source
-                    code from various places such as source control
-                    systems or websites.
+                    code from various places such as local files, source control
+                    systems, or websites.
                     </para></listitem>
                 <listitem><para>
                     The instructions for each unit to be built (e.g. a piece
-                    of software) are known as recipe files and
+                    of software) are known as "recipe" files and
                     contain all the information about the unit
                     (dependencies, source file locations, checksums, description
                     and so on).
                     </para></listitem>
                 <listitem><para>
                     BitBake includes a client/server abstraction and can
-                    be used from a command line or used as a service over XMLRPC and
-                    has several different user interfaces.
+                    be used from a command line or used as a service over
+                    XML-RPC and has several different user interfaces.
                     </para></listitem>
             </itemizedlist>
         </para>
@@ -69,7 +72,7 @@
             BitBake was originally a part of the OpenEmbedded project.
             It was inspired by the Portage package management system
             used by the Gentoo Linux distribution.
-            On December 7, 2004, OpenEmbedded project team member,
+            On December 7, 2004, OpenEmbedded project team member
             Chris Larson split the project into two distinct pieces:
             <itemizedlist>
                 <listitem><para>BitBake, a generic task executor</para></listitem>
@@ -79,8 +82,11 @@
             Today, BitBake is the primary basis of the
             <ulink url="http://www.openembedded.org/">OpenEmbedded</ulink>
             project, which is being used to build and maintain Linux
-            distributions such as the Angstrom Distribution and which is used
-            as the build tool for Linux projects such as the Yocto Project.
+            distributions such as the
+            <ulink url='http://www.angstrom-distribution.org/'>Angstrom Distribution</ulink>,
+            and which is also being used as the build tool for Linux projects
+            such as the
+            <ulink url='http://www.yoctoproject.org'>Yocto Project</ulink>.
         </para>
 
         <para>
@@ -88,7 +94,7 @@
             an aspiring embedded Linux distribution.
             All of the build systems used by traditional desktop Linux
             distributions lacked important functionality, and none of the
-            ad-hoc Buildroot-based systems, prevalent in the
+            ad hoc Buildroot-based systems, prevalent in the
             embedded space, were scalable or maintainable.
         </para>
 
@@ -138,7 +144,7 @@
                     projects for their builds.
                     </para></listitem>
                 <listitem><para>
-                    Provide an inheritance mechanism that share
+                    Provide an inheritance mechanism to share
                     common metadata between many packages.
                     </para></listitem>
             </itemizedlist>
@@ -151,7 +157,7 @@
                     </para></listitem>
                 <listitem><para>
                     Split metadata into layers and allow layers
-                    to override each other.
+                    to enhance or override other layers.
                     </para></listitem>
                 <listitem><para>
                     Allow representation of a given set of input variables
@@ -178,14 +184,14 @@
             what tasks are required to run, and executes those tasks.
             Similar to GNU Make, BitBake controls how software is
             built.
-            GNU Make achieves its control through "makefiles".
+            GNU Make achieves its control through "makefiles", while
             BitBake uses "recipes".
         </para>
 
         <para>
             BitBake extends the capabilities of a simple
-            tool like GNU Make by allowing for much more complex tasks
-            to be completed, such as assembling entire embedded Linux
+            tool like GNU Make by allowing for the definition of much more
+            complex tasks, such as assembling entire embedded Linux
             distributions.
         </para>
 
@@ -203,14 +209,20 @@
                 <filename>.bb</filename>, are the most basic metadata files.
                 These recipe files provide BitBake with the following:
                 <itemizedlist>
-                    <listitem><para>Descriptive information about the package</para></listitem>
+                    <listitem><para>Descriptive information about the
+                        package (author, homepage, license, and so on)</para></listitem>
                     <listitem><para>The version of the recipe</para></listitem>
-                    <listitem><para>Existing Dependencies</para></listitem>
-                    <listitem><para>Where the source code resides</para></listitem>
-                    <listitem><para>Whether the source code requires any patches</para></listitem>
-                    <listitem><para>How to compile the source code</para></listitem>
+                    <listitem><para>Existing dependencies (both build
+                        and runtime dependencies)</para></listitem>
+                    <listitem><para>Where the source code resides and
+                        how to fetch it</para></listitem>
+                    <listitem><para>Whether the source code requires
+                        any patches, where to find them, and how to apply
+                        them</para></listitem>
+                    <listitem><para>How to configure and compile the
+                        source code</para></listitem>
                     <listitem><para>Where on the target machine to install the
-                        package being compiled</para></listitem>
+                        package or packages created</para></listitem>
                 </itemizedlist>
             </para>
 
@@ -222,7 +234,11 @@
                     The term "package" is also commonly used to describe recipes.
                     However, since the same word is used to describe packaged
                     output from a project, it is best to maintain a single
-                    descriptive term, "recipes".
+                    descriptive term - "recipes".
+                    Put another way, a single "recipe" file is quite capable
+                    of generating a number of related but separately installable
+                    "packages".
+                    In fact, that ability is fairly common.
                 </note>
             </para>
         </section>
@@ -257,7 +273,7 @@
                 called <filename>base.bbclass</filename>.
                 You can find this file in the
                 <filename>classes</filename> directory.
-                The <filename>base.bbclass</filename> is special since it
+                The <filename>base.bbclass</filename> class files is special since it
                 is always included automatically for all recipes
                 and classes.
                 This class contains definitions for standard basic tasks such
@@ -284,7 +300,8 @@
                 To illustrate how you can use layers to keep things modular,
                 consider customizations you might make to support a specific target machine.
                 These types of customizations typically reside in a special layer,
-                rather than a general layer, called a Board Specific Package (BSP) Layer.
+                rather than a general layer, called a Board Support Package (BSP)
+                Layer.
                 Furthermore, the machine customizations should be isolated from
                 recipes and metadata that support a new GUI environment, for
                 example.
@@ -304,9 +321,8 @@
 
             <para>
                 Append files, which are files that have the
-                <filename>.bbappend</filename> file extension, add or
-                extend build information to an existing
-                recipe file.
+                <filename>.bbappend</filename> file extension, extend or
+                override information in an existing recipe file.
             </para>
 
             <para>
@@ -319,8 +335,9 @@
             </para>
 
             <para>
-                Information in append files overrides the information in the
-                similarly-named recipe file.
+                Information in append files extends or
+                overrides the information in the underlying,
+                similarly-named recipe files.
             </para>
 
             <para>
@@ -345,6 +362,12 @@
                 However, if you named the append file
                 <filename>busybox_1.%.bbappend</filename>, then you would have a match.
             </para>
+
+            <para>
+                In the most general case, you could name the append file something as
+                simple as <filename>busybox_%.bbappend</filename> to be entirely
+                version independent.
+            </para>
         </section>
     </section>
 
@@ -413,11 +436,25 @@
                     you have a directory entitled
                     <filename>bitbake-1.17.0</filename>.
                     </para></listitem>
+                <listitem><para><emphasis>Using the BitBake that Comes With Your
+                    Build Checkout:</emphasis>
+                    A final possibility for getting a copy of BitBake is that it
+                    already comes with your checkout of a larger Bitbake-based build
+                    system, such as Poky or Yocto Project.
+                    Rather than manually checking out individual layers and
+                    gluing them together yourself, you can check
+                    out an entire build system.
+                    The checkout will already include a version of BitBake that
+                    has been thoroughly tested for compatibility with the other
+                    components.
+                    For information on how to check out a particular BitBake-based
+                    build system, consult that build system's supporting documentation.
+                    </para></listitem>
             </itemizedlist>
         </para>
     </section>
 
-    <section id="user-manual-command">
+    <section id="bitbake-user-manual-command">
         <title>The BitBake Command</title>
 
         <para>
@@ -471,14 +508,16 @@
        -D, --debug           Increase the debug level. You can specify this more
                              than once.
        -n, --dry-run         Don't execute, just go through the motions.
-       -S DUMP_SIGNATURES, --dump-signatures=DUMP_SIGNATURES
+       -S SIGNATURE_HANDLER, --dump-signatures=SIGNATURE_HANDLER
                              Dump out the signature construction information, with
-                             no task execution. Parameters are passed to the
-                             signature handling code, use 'none' if no specific
-                             handler is required.
+                             no task execution. The SIGNATURE_HANDLER parameter is
+                             passed to the handler. Two common values are none and
+                             printdiff but the handler may define more/less. none
+                             means only dump the signature, printdiff means compare
+                             the dumped signature with the cached one.
        -p, --parse-only      Quit after parsing the BB recipes.
        -s, --show-versions   Show current and preferred versions of all recipes.
-       -e, --environment     Show the global or per-package environment complete
+       -e, --environment     Show the global or per-recipe environment complete
                              with information about where variables were
                              set/changed.
        -g, --graphviz        Save dependency tree information for the specified
@@ -493,6 +532,8 @@
        -u UI, --ui=UI        The user interface to use (e.g. knotty, hob, depexp).
        -t SERVERTYPE, --servertype=SERVERTYPE
                              Choose which server to use, process or xmlrpc.
+       --token=XMLRPCTOKEN   Specify the connection token to be used when
+                             connecting to a remote server.
        --revisions-changed   Set the exit code depending on whether upstream
                              floating revisions have changed or not.
        --server-only         Run bitbake without a UI, only starting a server
@@ -559,14 +600,14 @@
                     when one wants to manage multiple <filename>.bb</filename>
                     files.
                     Clearly there needs to be a way to tell BitBake what
-                    files are available, and of those, which you
+                    files are available and, of those, which you
                     want to execute.
                     There also needs to be a way for each recipe
                     to express its dependencies, both for build-time and
                     runtime.
                     There must be a way for you to express recipe preferences
                     when multiple recipes provide the same functionality, or when
-                    there are multiple versions of a  recipe.
+                    there are multiple versions of a recipe.
                 </para>
 
                 <para>

bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml

@@ -1,7 +1,7 @@
 <!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
 "http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd">
 
-<chapter id="user-manual-metadata">
+<chapter id="bitbake-user-manual-metadata">
     <title>Syntax and Operators</title>
 
     <para>
@@ -159,6 +159,10 @@
                 using the "+=" and "=+" operators.
                 These operators insert a space between the current
                 value and prepended or appended value.
+            </para>
+
+            <para>
+                These operators take immediate effect during parsing.
                 Here are some examples:
                 <literallayout class='monospaced'>
      B = "bval"
@@ -178,6 +182,10 @@
             <para>
                 If you want to append or prepend values without an
                 inserted space, use the ".=" and "=." operators.
+            </para>
+
+            <para>
+                These operators take immediate effect during parsing.
                 Here are some examples:
                 <literallayout class='monospaced'>
      B = "bval"
@@ -198,6 +206,13 @@
                 You can also append and prepend a variable's value
                 using an override style syntax.
                 When you use this syntax, no spaces are inserted.
+            </para>
+
+            <para>
+                These operators differ from the ":=", ".=", "=.", "+=", and "=+"
+                operators in that their effects are deferred
+                until after parsing completes rather than being immediately
+                applied.
                 Here are some examples:
                 <literallayout class='monospaced'>
      B = "bval"
@@ -217,13 +232,6 @@
                     override syntax.
                 </note>
             </para>
-
-            <para>
-                The operators "_append" and "_prepend" differ from
-                the operators ".=" and "=." in that they are deferred
-                until after parsing completes rather than being immediately
-                applied.
-            </para>
         </section>
 
         <section id='removing-override-style-syntax'>
@@ -283,7 +291,18 @@
                 The variable <filename>FOO</filename> has two flags:
                 <filename>a</filename> and <filename>b</filename>.
                 The flags are immediately set to "abc" and "123", respectively.
-                The <filename>a</filename> flag becomes "abc456".
+                The <filename>a</filename> flag becomes "abc 456".
+            </para>
+
+            <para>
+                No need exists to pre-define variable flags.
+                You can simply start using them.
+                One extremely common application
+                is to attach some brief documentation to a BitBake variable as
+                follows:
+                <literallayout class='monospaced'>
+     CACHE[doc] = "The directory holding the cache of the metadata."
+                </literallayout>
             </para>
         </section>
 
@@ -298,7 +317,42 @@
      DATE = "${@time.strftime('%Y%m%d',time.gmtime())}"
                 </literallayout>
                 This example results in the <filename>DATE</filename>
-                variable becoming the current date.
+                variable being set to the current date.
+            </para>
+
+            <para>
+                Probably the most common use of this feature is to extract
+                the value of variables from BitBake's internal data dictionary,
+                <filename>d</filename>.
+                The following lines select the values of a package name
+                and its version number, respectively:
+                <literallayout class='monospaced'>
+     PN = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE'),d)[0] or 'defaultpkgname'}"
+     PV = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE'),d)[1] or '1.0'}"
+                </literallayout>
+            </para>
+        </section>
+
+        <section id='providing-pathnames'>
+            <title>Providing Pathnames</title>
+
+            <para>
+                When specifying pathnames for use with BitBake,
+                do not use the tilde ("~") character as a shortcut
+                for your home directory.
+                Doing so might cause BitBake to not recognize the
+                path since BitBake does not expand this character in
+                the same way a shell would.
+            </para>
+
+            <para>
+                Instead, provide a fuller path as the following
+                example illustrates:
+                <literallayout class='monospaced'>
+     BBLAYERS ?= " \
+       /home/scott-lenovo/LayerA \
+       "
+                </literallayout>
             </para>
         </section>
     </section>
@@ -349,6 +403,25 @@
                         You select the os-specific version of the <filename>TEST</filename>
                         variable by appending the "os" override to the variable
                         (i.e.<filename>TEST_os</filename>).
+                        </para>
+
+                        <para>
+                            To better understand this, consider a practical example
+                            that assumes an OpenEmbedded metadata-based Linux
+                            kernel recipe file.
+                            The following lines from the recipe file first set
+                            the kernel branch variable <filename>KBRANCH</filename>
+                            to a default value, then conditionally override that
+                            value based on the architecture of the build:
+                        <literallayout class='monospaced'>
+     KBRANCH = "standard/base"
+     KBRANCH_qemuarm  = "standard/arm-versatile-926ejs"
+     KBRANCH_qemumips = "standard/mti-malta32"
+     KBRANCH_qemuppc  = "standard/qemuppc"
+     KBRANCH_qemux86  = "standard/common-pc/base"
+     KBRANCH_qemux86-64  = "standard/common-pc-64/base"
+     KBRANCH_qemumips64 = "standard/mti-malta64"
+                        </literallayout>
                         </para></listitem>
                     <listitem><para><emphasis>Appending and Prepending:</emphasis>
                         BitBake also supports append and prepend operations to
@@ -362,6 +435,19 @@
                         </literallayout>
                         In this example, <filename>DEPENDS</filename> becomes
                         "glibc ncurses libmad".
+                        </para>
+
+                        <para>
+                            Again, using an OpenEmbedded metadata-based
+                            kernel recipe file as an example, the
+                            following lines will conditionally append to the
+                            <filename>KERNEL_FEATURES</filename> variable based
+                            on the architecture:
+                        <literallayout class='monospaced'>
+     KERNEL_FEATURES_append = " ${KERNEL_EXTRA_FEATURES}"
+     KERNEL_FEATURES_append_qemux86=" cfg/sound.scc cfg/paravirt_kvm.scc"
+     KERNEL_FEATURES_append_qemux86-64=" cfg/sound.scc cfg/paravirt_kvm.scc"
+                        </literallayout>
                         </para></listitem>
                 </itemizedlist>
             </para>
@@ -683,7 +769,7 @@
         <para>
             As with most languages, functions are the building blocks that
             are used to build up operations into tasks.
-            BitBake supports three types of functions:
+            BitBake supports these types of functions:
             <itemizedlist>
                 <listitem><para><emphasis>Shell Functions:</emphasis>
                     Functions written in shell script and executed either
@@ -697,6 +783,10 @@
                 <listitem><para><emphasis>Python Functions:</emphasis>
                     Functions written in Python and executed by Python.
                     </para></listitem>
+                <listitem><para><emphasis>Anonymous Python Functions:</emphasis>
+                    Python functions executed automatically during
+                    parsing.
+                    </para></listitem>
             </itemizedlist>
             Regardless of the type of function, you can only
             define them in class (<filename>.bbclass</filename>)
@@ -793,36 +883,71 @@
             </para>
         </section>
 
-        <section id='automatically-mapping-functions-within-the-context-of-a-class'>
-            <title>Automatically Mapping Functions Within the Context of a Class</title>
+        <section id='anonymous-python-functions'>
+            <title>Anonymous Python Functions</title>
+
+            <para>
+                Sometimes it is useful to run some code during
+                parsing to set variables or to perform other operations
+                programmatically.
+                To do this, you can define an anonymous Python function.
+                Here is an example that conditionally sets a
+                variable based on the value of another variable:
+                <literallayout class='monospaced'>
+     python __anonymous () {
+         if d.getVar('SOMEVAR', True) == 'value':
+             d.setVar('ANOTHERVAR', 'value2')
+     }
+                </literallayout>
+                The "__anonymous" function name is optional, so the
+                following example is functionally equivalent to the above:
+                <literallayout class='monospaced'>
+     python () {
+         if d.getVar('SOMEVAR', True) == 'value':
+             d.setVar('ANOTHERVAR', 'value2')
+     }
+                </literallayout>
+                Because unlike other Python functions anonymous
+                Python functions are executed during parsing, the
+                "d" variable within an anonymous Python function represents
+                the datastore for the entire recipe.
+                Consequently, you can set variable values here and
+                those values can be picked up by other functions.
+            </para>
+        </section>
+
+        <section id='flexible-inheritance-for-class-functions'>
+            <title>Flexible Inheritance for Class Functions</title>
 
             <para>
                 Through coding techniques and the use of
                 <filename>EXPORT_FUNCTIONS</filename>, BitBake supports
-                automatic mapping for functions within the context of
-                a class.
+                exporting a function from a class such that the
+                class function appears as the default implementation
+                of the function, but can still be called if a recipe
+                inheriting the class needs to define its own version of
+                the function.
             </para>
 
             <para>
-                To understand the benefits of this feature, consider the basic scenario
-                where a class defines a function and your recipe inherits the class.
-                In this basic scenario, your recipe has access to the function in the
-                class by way of inheritance and can freely call and use the function
-                as defined in the class.
-                However, if you need to have a modified version of that function
-                in your recipe you are limited to using either your modified version
-                of the function or using "prepend_" or "_append" operators to add
-                code to be executed before or after the original function in the
-                class.
-                Your recipe cannot use both versions of the fucntion.
+                To understand the benefits of this feature, consider
+                the basic scenario where a class defines a task function
+                and your recipe inherits the class.
+                In this basic scenario, your recipe inherits the task
+                function as defined in the class.
+                If desired, your recipe can add to the start and end of the
+                function by using the "_prepend" or "_append" operations
+                respectively, or it can redefine the function completely.
+                However, if it redefines the function, there is
+                no means  for it to call the class version of the function.
+                <filename>EXPORT_FUNCTIONS</filename> provides a mechanism
+                that enables the recipe's version of the function to call
+                the original version of the function.
             </para>
 
             <para>
-                Function mapping allows you to access both your custom function
-                function that is defined in the recipe and the original function that
-                is defined in the class.
-                You have this access all from within your recipe.
-                To accomplish this, you need some things in place:
+                To make use of this technique, you need the following
+                things in place:
                 <itemizedlist>
                     <listitem><para>
                         The class needs to define the function as follows:
@@ -853,12 +978,24 @@
                     <listitem><para>
                         You need to call the function appropriately from within your
                         recipe.
-                        Continuing with the same example,
-                        your recipe would call the <filename>do_foo</filename> function
-                        from the recipe by referring to it as
-                        <filename>bar_do_foo</filename>.
-                        To call your modified version of the function as defined in your
-                        recipe, call it as <filename>do_foo</filename>.
+                        Continuing with the same example, if your recipe
+                        needs to call the class version of the function,
+                        it should call <filename>bar_do_foo</filename>.
+                        Assuming <filename>do_foo</filename> was a shell function
+                        and <filename>EXPORT_FUNCTIONS</filename> was used as above,
+                        the recipe's function could conditionally call the
+                        class version of the function as follows:
+                        <literallayout class='monospaced'>
+     do_foo() {
+             if [ somecondition ] ; then
+                     bar_do_foo
+             else
+                     # Do something else
+             fi
+     }
+                        </literallayout>
+                        To call your modified version of the function as defined
+                        in your recipe, call it as <filename>do_foo</filename>.
                         </para></listitem>
                 </itemizedlist>
                 With these conditions met, your single recipe
@@ -1066,12 +1203,6 @@
                     Tells BitBake to not generate a stamp file for a task,
                     which implies the task should always be executed.
                     </para></listitem>
-                <listitem><para><emphasis>fakeroot:</emphasis>
-                    Causes a task to be run in a fakeroot environment,
-                    obtained by adding the variables in
-                    <link linkend='var-FAKEROOTENV'><filename>FAKEROOTENV</filename></link>
-                    to the environment.
-                    </para></listitem>
                 <listitem><para><emphasis>umask:</emphasis>
                     The umask to run the task under.
                     </para></listitem>
@@ -1392,9 +1523,9 @@
                 complete before that task can be executed.
                 Here is an example:
                 <literallayout class='monospaced'>
-     do_configure[deptask] = "do_populate_staging"
+     do_configure[deptask] = "do_populate_sysroot"
                 </literallayout>
-                In this example, the <filename>do_populate_staging</filename>
+                In this example, the <filename>do_populate_sysroot</filename>
                 task of each item in <filename>DEPENDS</filename> must complete before
                 <filename>do_configure</filename> can execute.
             </para>
@@ -1469,9 +1600,9 @@
                 the data in <filename>DEPENDS</filename>.
                 Here is an example:
                 <literallayout class='monospaced'>
-     do_patch[depends] = "quilt-native:do_populate_staging"
+     do_patch[depends] = "quilt-native:do_populate_sysroot"
                 </literallayout>
-                In this example, the <filename>do_populate_staging</filename>
+                In this example, the <filename>do_populate_sysroot</filename>
                 task of the target <filename>quilt-native</filename>
                 must have completed before the
                 <filename>do_patch</filename> task can execute.

bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.xml

@@ -1080,20 +1080,21 @@
                     (<filename>.conf</filename>) files.
                     This variable is analogous to the
                     <filename>PATH</filename> variable.
-                    <note>
-                        If you run BitBake from a directory outside of the
-                        build directory,
-                        you must be sure to set
-                        <filename>BBPATH</filename> to point to the
-                        build directory.
-                        Set the variable as you would any environment variable
-                        and then run BitBake:
-                        <literallayout class='monospaced'>
-     $ BBPATH = "&lt;build_directory&gt;"
+                </para>
+
+                <para>
+                    If you run BitBake from a directory outside of the
+                    build directory,
+                    you must be sure to set
+                    <filename>BBPATH</filename> to point to the
+                    build directory.
+                    Set the variable as you would any environment variable
+                    and then run BitBake:
+                    <literallayout class='monospaced'>
+     $ BBPATH="&lt;build_directory&gt;"
      $ export BBPATH
      $ bitbake &lt;target&gt;
-                        </literallayout>
-                    </note>
+                    </literallayout>
                 </para>
             </glossdef>
         </glossentry>
@@ -1559,8 +1560,16 @@
                     BitBake uses <filename>OVERRIDES</filename> to control
                     what variables are overridden after BitBake parses
                     recipes and configuration files.
-                    You can find more information on how overrides are handled
-                    in the
+                </para>
+
+                <para>
+                    Following is a simple example that uses an overrides
+                    list based on machine architectures:
+                    <literallayout class='monospaced'>
+     OVERRIDES = "arm:x86:mips:powerpc"
+                    </literallayout>
+                    You can find information on how to use
+                    <filename>OVERRIDES</filename> in the
                     "<link linkend='conditional-syntax-overrides'>Conditional Syntax (Overrides)</link>"
                     section.
                 </para>
@@ -1743,16 +1752,28 @@
         <glossentry id='var-PROVIDES'><glossterm>PROVIDES</glossterm>
             <glossdef>
                 <para>
-                    A list of aliases that a recipe also provides.
-                    These aliases are useful for satisfying dependencies of
-                    other recipes during the build (as specified by
-                    <filename><link linkend='var-DEPENDS'>DEPENDS</link></filename>).
-                    <note>
-                        A recipe's own
-                        <filename><link linkend='var-PN'>PN</link></filename>
-                        is implicitly already in its
-                        <filename>PROVIDES</filename> list.
-                    </note>
+                    A list of aliases by which a particular recipe can be
+                    known.
+                    By default, a recipe's own
+                    <filename><link linkend='var-PN'>PN</link></filename>
+                    is implicitly already in its <filename>PROVIDES</filename>
+                    list.
+                    If a recipe uses <filename>PROVIDES</filename>, the
+                    additional aliases are synonyms for the recipe and can
+                    be useful satisfying dependencies of other recipes during
+                    the build as specified by
+                    <filename><link linkend='var-DEPENDS'>DEPENDS</link></filename>.
+                </para>
+
+                <para>
+                    Consider the following example
+                    <filename>PROVIDES</filename> statement from a recipe
+                    file <filename>libav_0.8.11.bb</filename>:
+                    <literallayout class='monospaced'>
+     PROVIDES += "libpostproc"
+                    </literallayout>
+                    The <filename>PROVIDES</filename> statement results in
+                    the "libav" recipe also being known as "libpostproc".
                 </para>
             </glossdef>
         </glossentry>

bitbake/doc/bitbake-user-manual/bitbake-user-manual-style.css

@@ -313,6 +313,13 @@ a:hover {
   /*font-weight: bold;*/
 }
 
+/* This style defines how the permalink character
+   appears by itself and when hovered over with
+   the mouse. */
+
+[alt='Permalink'] { color: #eee; }
+[alt='Permalink']:hover { color: black; }
+
 
 div.informalfigure,
 div.informalexample,
@@ -793,7 +800,6 @@ div.sect2 .titlepage .title {
 
 h1.title {
   background-color: transparent;
-  background-image: url("figures/yocto-project-bw.png");
   background-repeat: no-repeat;
   height: 256px;
   text-indent: -9000px;

bitbake/doc/bitbake-user-manual/bitbake-user-manual.xml

@@ -73,16 +73,16 @@
         </legalnotice>
     </bookinfo>
 
-    <xi:include href="user-manual-intro.xml"/>
+    <xi:include href="bitbake-user-manual-intro.xml"/>
 
-    <xi:include href="user-manual-execution.xml"/>
+    <xi:include href="bitbake-user-manual-execution.xml"/>
 
-    <xi:include href="user-manual-metadata.xml"/>
+    <xi:include href="bitbake-user-manual-metadata.xml"/>
 
-    <xi:include href="user-manual-fetching.xml"/>
+    <xi:include href="bitbake-user-manual-fetching.xml"/>
 
-    <xi:include href="user-manual-ref-variables.xml"/>
+    <xi:include href="bitbake-user-manual-ref-variables.xml"/>
 
-    <xi:include href="user-manual-hello.xml"/>
+    <xi:include href="bitbake-user-manual-hello.xml"/>
 
 </book>

bitbake/doc/bitbake.1

@@ -89,7 +89,7 @@ quit after parsing the BB files (developers only)
 show current and preferred versions of all packages
 .TP
 .B \-e, \-\-environment
-show the global or per-package environment (this is what used to be bbread)
+show the global or per-recipe environment (this is what used to be bbread)
 .TP
 .B \-g, \-\-graphviz
 emit the dependency trees of the specified packages in the dot syntax

bitbake/doc/template/component.title.xsl

@@ -0,0 +1,39 @@
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:d="http://docbook.org/ns/docbook"
+  xmlns="http://www.w3.org/1999/xhtml"
+  exclude-result-prefixes="d">
+  
+  <xsl:template name="component.title">
+    <xsl:param name="node" select="."/>
+    
+    <xsl:variable name="level">
+      <xsl:choose>
+        <xsl:when test="ancestor::d:section">
+          <xsl:value-of select="count(ancestor::d:section)+1"/>
+        </xsl:when>
+        <xsl:when test="ancestor::d:sect5">6</xsl:when>
+        <xsl:when test="ancestor::d:sect4">5</xsl:when>
+        <xsl:when test="ancestor::d:sect3">4</xsl:when>
+        <xsl:when test="ancestor::d:sect2">3</xsl:when>
+        <xsl:when test="ancestor::d:sect1">2</xsl:when>
+        <xsl:otherwise>1</xsl:otherwise>
+      </xsl:choose>
+    </xsl:variable>
+    <xsl:element name="h{$level+1}" namespace="http://www.w3.org/1999/xhtml">
+      <xsl:attribute name="class">title</xsl:attribute>
+      <xsl:if test="$generate.id.attributes = 0">
+        <xsl:call-template name="anchor">
+          <xsl:with-param name="node" select="$node"/>
+          <xsl:with-param name="conditional" select="0"/>
+        </xsl:call-template>
+      </xsl:if>
+      <xsl:apply-templates select="$node" mode="object.title.markup">
+        <xsl:with-param name="allow-anchors" select="1"/>
+      </xsl:apply-templates>
+      <xsl:call-template name="permalink">
+        <xsl:with-param name="node" select="$node"/>
+      </xsl:call-template>
+    </xsl:element>
+  </xsl:template>
+</xsl:stylesheet>

bitbake/doc/template/division.title.xsl

@@ -0,0 +1,25 @@
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:d="http://docbook.org/ns/docbook"
+  xmlns="http://www.w3.org/1999/xhtml"
+  exclude-result-prefixes="d">
+  
+  <xsl:template name="division.title">
+    <xsl:param name="node" select="."/>
+    
+    <h1>
+      <xsl:attribute name="class">title</xsl:attribute>
+      <xsl:call-template name="anchor">
+        <xsl:with-param name="node" select="$node"/>
+        <xsl:with-param name="conditional" select="0"/>
+      </xsl:call-template>
+      <xsl:apply-templates select="$node" mode="object.title.markup">
+        <xsl:with-param name="allow-anchors" select="1"/>
+      </xsl:apply-templates>
+      <xsl:call-template name="permalink">
+        <xsl:with-param name="node" select="$node"/>
+      </xsl:call-template>
+    </h1>
+  </xsl:template>
+</xsl:stylesheet>
+

bitbake/doc/template/formal.object.heading.xsl

@@ -0,0 +1,21 @@
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:d="http://docbook.org/ns/docbook"
+  xmlns="http://www.w3.org/1999/xhtml"
+  exclude-result-prefixes="d">
+  
+  <xsl:template name="formal.object.heading">
+    <xsl:param name="object" select="."/>
+    <xsl:param name="title">
+      <xsl:apply-templates select="$object" mode="object.title.markup">
+        <xsl:with-param name="allow-anchors" select="1"/>
+      </xsl:apply-templates>
+    </xsl:param>
+    <p class="title">
+      <b><xsl:copy-of select="$title"/></b>
+      <xsl:call-template name="permalink">
+        <xsl:with-param name="node" select="$object"/>
+      </xsl:call-template>
+    </p>
+  </xsl:template>
+</xsl:stylesheet>

bitbake/doc/template/gloss-permalinks.xsl

@@ -0,0 +1,14 @@
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:d="http://docbook.org/ns/docbook"
+  xmlns="http://www.w3.org/1999/xhtml">
+
+  <xsl:template match="glossentry/glossterm">
+    <xsl:apply-imports/>
+    <xsl:if test="$generate.permalink != 0">
+      <xsl:call-template name="permalink">
+        <xsl:with-param name="node" select=".."/>
+      </xsl:call-template>
+    </xsl:if>
+  </xsl:template>
+</xsl:stylesheet>

bitbake/doc/template/permalinks.xsl

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsl:stylesheet version="1.0"
+  xmlns="http://www.w3.org/1999/xhtml"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
+
+  <xsl:param name="generate.permalink" select="1"/>
+  <xsl:param name="permalink.text">¶</xsl:param>
+
+  <xsl:template name="permalink">
+    <xsl:param name="node"/>
+
+    <xsl:if test="$generate.permalink != '0'">
+      <span class="permalink">
+        <a alt="Permalink" title="Permalink">
+          <xsl:attribute name="href">
+            <xsl:call-template name="href.target">
+              <xsl:with-param name="object"  select="$node"/>
+            </xsl:call-template>
+          </xsl:attribute>
+          <xsl:copy-of select="$permalink.text"/>
+        </a>
+      </span>
+    </xsl:if>
+  </xsl:template>
+</xsl:stylesheet>

bitbake/doc/template/section.title.xsl

@@ -0,0 +1,55 @@
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:d="http://docbook.org/ns/docbook"
+  xmlns="http://www.w3.org/1999/xhtml" exclude-result-prefixes="d">
+
+  <xsl:template name="section.title">
+    <xsl:variable name="section"
+      select="(ancestor::section |
+               ancestor::simplesect|
+               ancestor::sect1|
+               ancestor::sect2|
+               ancestor::sect3|
+               ancestor::sect4|
+               ancestor::sect5)[last()]"/>
+
+    <xsl:variable name="renderas">
+      <xsl:choose>
+        <xsl:when test="$section/@renderas = 'sect1'">1</xsl:when>
+        <xsl:when test="$section/@renderas = 'sect2'">2</xsl:when>
+        <xsl:when test="$section/@renderas = 'sect3'">3</xsl:when>
+        <xsl:when test="$section/@renderas = 'sect4'">4</xsl:when>
+        <xsl:when test="$section/@renderas = 'sect5'">5</xsl:when>
+        <xsl:otherwise><xsl:value-of select="''"/></xsl:otherwise>
+      </xsl:choose>
+    </xsl:variable>
+
+    <xsl:variable name="level">
+      <xsl:choose>
+        <xsl:when test="$renderas != ''">
+          <xsl:value-of select="$renderas"/>
+        </xsl:when>
+        <xsl:otherwise>
+          <xsl:call-template name="section.level">
+            <xsl:with-param name="node" select="$section"/>
+          </xsl:call-template>
+        </xsl:otherwise>
+      </xsl:choose>
+    </xsl:variable>
+
+    <xsl:call-template name="section.heading">
+      <xsl:with-param name="section" select="$section"/>
+      <xsl:with-param name="level" select="$level"/>
+      <xsl:with-param name="title">
+        <xsl:apply-templates select="$section" mode="object.title.markup">
+          <xsl:with-param name="allow-anchors" select="1"/>
+        </xsl:apply-templates>
+        <xsl:if test="$level &gt; 0">
+          <xsl:call-template name="permalink">
+            <xsl:with-param name="node" select="$section"/>
+          </xsl:call-template>
+        </xsl:if>
+      </xsl:with-param>
+    </xsl:call-template>
+  </xsl:template>
+</xsl:stylesheet>

bitbake/doc/template/titlepage.templates.xml

@@ -144,7 +144,7 @@
 
 # If you leave this block of code in then the text title in the
 # <title>BitBake User Manual</title> statement of the
-# user-manual.xml file is rendered on the title page below the
+# bitbake-user-manual.xml file is rendered on the title page below the
 # image.  Commenting it out gets it out of there yet allows it
 # to be retained in the tab text for the HTML version of the
 # manual.
@@ -176,7 +176,7 @@
 <!--
 # If you leave this block of code in then the text title in the
 # <title>BitBake User Manual</title> statement of the
-# user-manual.xml file is rendered on the title page below the
+# bitbake-user-manual.xml file is rendered on the title page below the
 # image.  Commenting it out gets it out of there yet allows it
 # to be retained in the tab text for the HTML version of the
 # manual.

bitbake/doc/user-manual/user-manual-hello.xml

@@ -1,334 +0,0 @@
-<!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
-"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd">
-
-<appendix id='hello-world-example'>
-    <title>Hello World Example</title>
-
-    <section id='bitbake-hello-world'>
-        <title>BitBake Hello World</title>
-
-        <para>
-            The simplest example commonly used to demonstrate any new
-            programming language or tool is the
-            <ulink url="http://en.wikipedia.org/wiki/Hello_world_program">Hello World</ulink>
-            example.
-            This appendix demonstrates, in tutorial form, Hello
-            World within the context of BitBake.
-            The tutorial describes how to create a new Project
-            and the applicable metadata files necessary to allow
-            BitBake to build it.
-        </para>
-    </section>
-
-    <section id='example-obtaining-bitbake'>
-        <title>Obtaining BitBake</title>
-
-        <para>
-            See the
-            "<link linkend='obtaining-bitbake'>Obtaining BitBake</link>"
-            section for information on how to obtain BitBake.
-            Once you have the source code on your machine, the BitBake directory
-            appears as follows:
-            <literallayout class='monospaced'>
-     $ ls -al
-     total 100
-     drwxrwxr-x. 9 wmat wmat  4096 Jan 31 13:44 .
-     drwxrwxr-x. 3 wmat wmat  4096 Feb  4 10:45 ..
-     -rw-rw-r--. 1 wmat wmat   365 Nov 26 04:55 AUTHORS
-     drwxrwxr-x. 2 wmat wmat  4096 Nov 26 04:55 bin
-     drwxrwxr-x. 4 wmat wmat  4096 Jan 31 13:44 build
-     -rw-rw-r--. 1 wmat wmat 16501 Nov 26 04:55 ChangeLog
-     drwxrwxr-x. 2 wmat wmat  4096 Nov 26 04:55 classes
-     drwxrwxr-x. 2 wmat wmat  4096 Nov 26 04:55 conf
-     drwxrwxr-x. 3 wmat wmat  4096 Nov 26 04:55 contrib
-     -rw-rw-r--. 1 wmat wmat 17987 Nov 26 04:55 COPYING
-     drwxrwxr-x. 3 wmat wmat  4096 Nov 26 04:55 doc
-     -rw-rw-r--. 1 wmat wmat    69 Nov 26 04:55 .gitignore
-     -rw-rw-r--. 1 wmat wmat   849 Nov 26 04:55 HEADER
-     drwxrwxr-x. 5 wmat wmat  4096 Jan 31 13:44 lib
-     -rw-rw-r--. 1 wmat wmat   195 Nov 26 04:55 MANIFEST.in
-     -rwxrwxr-x. 1 wmat wmat  3195 Jan 31 11:57 setup.py
-     -rw-rw-r--. 1 wmat wmat  2887 Nov 26 04:55 TODO
-            </literallayout>
-        </para>
-
-        <para>
-            At this point, you should have BitBake cloned to
-            a directory that matches the previous listing except for
-            dates and user names.
-        </para>
-    </section>
-
-    <section id='setting-up-the-bitbake-environment'>
-        <title>Setting Up the BitBake Environment</title>
-
-        <para>
-            The recommended method to run BitBake is from a directory of your
-            choice.
-            The directory can be within your home directory or in
-            <filename>/usr/local</filename>,
-            depending on your preference.
-        </para>
-
-        <para>
-            First, run BitBake to make sure it's working.
-            From the BitBake source code directory, issue the following command:
-            <literallayout class='monospaced'>
-     $ ./bin/bitbake --version
-     BitBake Build Tool Core version 1.19.0, bitbake version
-     1.19.0
-            </literallayout>
-            You are now ready to use BitBake.
-        </para>
-
-        <para>
-            A final step to make development easier is to add the executable
-            binary to your environment <filename>PATH</filename>.
-            First, look at your current <filename>PATH</filename> variable
-            by entering the following:
-            <literallayout class='monospaced'>
-     $ echo $PATH
-            </literallayout>
-            Next, add the directory location for the BitBake binary to the
-            <filename>PATH</filename> using this form:
-            <literallayout class='monospaced'>
-     $ export PATH=&lt;path-to-bitbake-executable&gt;:$PATH
-            </literallayout>
-            This will add the directory to the beginning of your
-            <filename>PATH</filename> environment variable.
-            You should now be able to enter the <filename>bitbake</filename>
-            command at the command line to run BitBake.
-        </para>
-
-        <para>
-            For a more permanent solution assuming you are running the BASH
-            shell, edit <filename>~/.bashrc</filename> and add the following to the end
-            of that file:
-            <literallayout class='monospaced'>
-     PATH=&lt;path-to-bitbake-executable&gt;:$PATH
-            </literallayout>
-        </para>
-
-        <para>
-            If you're a Vim user, you will find useful
-            Vim configuration contributions in the
-            <filename>contrib/vim</filename> directory.
-            Copy the files from that directory to your
-            <filename>/home/yourusername/.vim</filename>
-            directory.
-            If that directory does not exist, create it, and then
-            restart Vim.
-        </para>
-    </section>
-
-    <section id='the-hello-world-example'>
-        <title>The Hello World Example</title>
-
-        <para>
-            The following example leaps directly into how BitBake
-            works.
-            While every attempt is made to explain what is happening,
-            not everything can be covered.
-            You can find further information in the
-            "<link linkend='user-manual-metadata'>Syntax and Operators</link>"
-            chapter.
-        </para>
-
-        <para>
-            The overall goal of this exercise is to build a
-            complete "Hello World" example utilizing task and layer
-            concepts.
-            This is how modern projects such as OpenEmbedded and
-            the Yocto Project utilize BitBake, therefore it
-            provides an excellent starting point for understanding
-            BitBake.
-        </para>
-
-        <para>
-            It should be noted that this chapter was inspired by
-            and draws heavily from several sources:
-            <itemizedlist>
-                <listitem><para>
-                    <ulink href="http://www.mail-archive.com/yocto@yoctoproject.org/msg09379.html">Mailing List post - The BitBake equivalent of "Hello, World!"</ulink>
-                    </para></listitem>
-                <listitem><para>
-                    <ulink href="http://hambedded.org/blog/2012/11/24/from-bitbake-hello-world-to-an-image/">Hambedded Linux blog post - From Bitbake Hello World to an Image</ulink>
-                    </para></listitem>
-            </itemizedlist>
-        </para>
-
-        <section id='a-reverse-walk-through'>
-            <title>A Reverse Walk-Through</title>
-
-            <para>
-                A good way to understand anything is to walk through the steps
-                that take you to where you want to be and observe first
-                principles.
-                BitBake allows us to do this through the
-                <filename>-D</filename> or <filename>Debug</filename>
-                command-line parameter.
-            </para>
-
-            <para>
-                The goal is to eventually compile a "Hello World" example.
-                However, it is unknown what is needed to achieve that goal.
-                Recall that BitBake utilizes three types of metadata files:
-                <link linkend='configuration-files'>Configuration Files</link>,
-                <link linkend='classes'>Classes</link>, and
-                <link linkend='recipes'>Recipes</link>.
-                But where do they go?
-                How does BitBake find them?
-                BitBake's error messaging helps you answer these types of questions
-                and helps you better understand exactly what is going on.
-            </para>
-
-            <para>
-                First, set up a directory for the "Hello World" project.
-                Here is how you can do so in your home directory:
-                <literallayout class='monospaced'>
-     $ mkdir ~/dev/hello &amp;&amp; cd ~/dev/hello
-                </literallayout>
-                Within this new, empty directory, run BitBake with
-                debugging output and see what happens:
-                <literallayout class='monospaced'>
-     $ bitbake -DDD
-     The BBPATH variable is not set
-     DEBUG: Removed the following variables from the environment:
-     GNOME_DESKTOP_SESSION_ID, LESSOPEN, WINDOWID,
-     GNOME_KEYRING_CONTROL, DISPLAY, SSH_AGENT_PID, LANG,
-     XDG_SESSION_PATH, XAUTHORITY, LANGUAGE, SESSION_MANAGER,
-     SHLVL, MANDATORY_PATH, COMPIZ_CONFIG_PROFILE, TEXTDOMAIN,
-     GPG_AGENT_INFO, SSH_AUTH_SOCK, XDG_RUNTIME_DIR,
-     COMPIZ_BIN_PATH, GDMSESSION, DEFAULTS_PATH, TEXTDOMAINDIR,
-     XDG_SEAT_PATH, XDG_CONFIG_DIRS, XDG_CURRENT_DESKTOP,
-     DBUS_SESSION_BUS_ADDRESS, _, XDG_SESSION_COOKIE,
-     DESKTOP_SESSION, LESSCLOSE, GNOME_KEYRING_PID,
-     UBUNTU_MENUPROXY, OLDPWD, GTK_MODULES, XDG_DATA_DIRS,
-     COLORTERM, LS_COLORS
-                </literallayout>
-                The majority of this output is specific to environment variables
-                that are not directly relevant to BitBake.
-                However, the very first message
-                "<filename>The BBPATH variable is not set</filename>"
-                is relevant and you need to rectify it by setting
-                <link linkend='var-BBPATH'><filename>BBPATH</filename></link>.
-            </para>
-
-            <para>
-                When you run BitBake, it begins looking for metadata files.
-                The <filename>BBPATH</filename> variable is what tells
-                BitBake where to look.
-                You could set <filename>BBPATH</filename> in the same manner
-                that you set <filename>PATH</filename> as shown earlier.
-                However, it is much more flexible to set the
-                <link linkend='var-BBPATH'><filename>BBPATH</filename></link>
-                variable for each project.
-            </para>
-
-            <para>
-                Without <filename>BBPATH</filename>, Bitbake cannot
-                find any configuration files (<filename>.conf</filename>)
-                or recipe files (<filename>.bb</filename>) at all.
-                BitBake also cannot find the <filename>bitbake.conf</filename>
-                file.
-            </para>
-
-            <para>
-                It is standard practice to organize the project's directory tree
-                to include both a <filename>conf/</filename> and
-                <filename>classes/</filename> directory.
-                You need to add those directories to your project:
-                <literallayout class='monospaced'>
-     $ mkdir conf classes
-                </literallayout>
-                Once those directories are in place, you can copy the
-                sample configuration files provided in the
-                BitBake source tree to their appropriate directories.
-                First, change to the BitBake source tree directory and
-                then copy the directories:
-                <literallayout class='monospaced'>
-     cp conf/bitbake.conf ~/dev/hello/conf/
-     cp classes/base.bbclass ~/dev/hello/classes/
-                </literallayout>
-                At this point your project directory structure should look like
-                the following:
-               <literallayout class='monospaced'>
-     ~/dev/hello$ tree
-     .
-     |-- classes
-     |   +-- base.bbclass
-     +-- conf
-     +-- bitbake.conf
-                </literallayout>
-            </para>
-
-            <para>
-                Once you have copied these files into your project, you
-                can now get back to resolving the <filename>BBPATH</filename>
-                issue.
-            </para>
-
-            <para>
-                The first configuration file that BitBake looks for is always
-                <filename>bblayers.conf</filename>.
-                With this knowledge, you know that to resolve your
-                <filename>BBPATH</filename> error you can add a
-                <filename>conf/bblayers.conf</filename> file to the
-                project source tree and populate it with the
-                <filename>BBPATH</filename> variable declaration.
-            </para>
-
-            <para>
-                From your project source tree:
-                <literallayout class='monospaced'>
-     $ vim conf/bblayers.conf
-                </literallayout>
-                Now add the following to the empty
-                <filename>bblayers.conf</filename> file:
-                <literallayout class='monospaced'>
-     BBPATH := "${TOPDIR}"
-                </literallayout>
-            </para>
-
-            <para>
-                Now, from the root of your project directory, run BitBake
-                again and see what happens:
-                <literallayout class='monospaced'>
-     $ bitbake -DDD
-     Nothing to do.  Use 'bitbake world' to build everything, or run
-     'bitbake --help' for usage information.
-     DEBUG: Removed the following variables from the environment:
-     GNOME_DESKTOP_SESSION_ID, LESSOPEN, WINDOWID,
-     GNOME_KEYRING_CONTROL, DISPLAY, SSH_AGENT_PID, LANG,
-     XDG_SESSION_PATH, XAUTHORITY, LANGUAGE, SESSION_MANAGER,
-     SHLVL, MANDATORY_PATH, COMPIZ_CONFIG_PROFILE, TEXTDOMAIN,
-     GPG_AGENT_INFO, SSH_AUTH_SOCK, XDG_RUNTIME_DIR,
-     COMPIZ_BIN_PATH, GDMSESSION, DEFAULTS_PATH, TEXTDOMAINDIR,
-     XDG_SEAT_PATH, XDG_CONFIG_DIRS, XDG_CURRENT_DESKTOP,
-     DBUS_SESSION_BUS_ADDRESS, _, XDG_SESSION_COOKIE,
-     DESKTOP_SESSION, LESSCLOSE, GNOME_KEYRING_PID, UBUNTU_MENUPROXY,
-     OLDPWD, GTK_MODULES, XDG_DATA_DIRS, COLORTERM, LS_COLORS
-     DEBUG: Found bblayers.conf (/home/wmat/dev/hello/conf/
-     bblayers.conf)
-     DEBUG: LOAD /home/wmat/dev/hello/conf/bblayers.conf
-     DEBUG: LOAD /home/wmat/dev/hello/conf/bitbake.conf
-     DEBUG: BB configuration INHERITs:0: inheriting /home/wmat/dev/
-     hello/classes/base.bbclass
-     DEBUG: BB /home/wmat/dev/hello/classes/base.bbclass: handle
-     (data, include)
-     DEBUG: LOAD /home/wmat/dev/hello/classes/base.bbclass
-     DEBUG: Clearing SRCREV cache due to cache policy of: clear
-     DEBUG: Using cache in '/home/wmat/dev/hello/tmp/cache/
-     local_file_checksum_cache.dat'
-     DEBUG: Using cache in '/home/wmat/dev/hello/tmp/cache/
-     bb_codeparser.dat'
-                </literallayout>
-                <note>
-                    From this point forward in the example, the environment
-                    variable removal messages are ignored and omitted.
-                    Examine the relevant DEBUG messages:
-                </note>
-            </para>
-        </section>
-    </section>
-</appendix>

bitbake/lib/bb/__init__.py

@@ -21,7 +21,7 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-__version__ = "1.22.0"
+__version__ = "1.24.0"
 
 import sys
 if sys.version_info < (2, 7, 3):
@@ -99,12 +99,11 @@ def error(*args):
 
 def fatal(*args):
     logger.critical(''.join(args))
-    sys.exit(1)
-
+    raise BBHandledException()
 
 def deprecated(func, name=None, advice=""):
     """This is a decorator which can be used to mark functions
-    as deprecated. It will result in a warning being emmitted
+    as deprecated. It will result in a warning being emitted
     when the function is used."""
     import warnings
 

bitbake/lib/bb/build.py

@@ -23,7 +23,7 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 #
-#Based on functions from the base bb module, Copyright 2003 Holger Schurig
+# Based on functions from the base bb module, Copyright 2003 Holger Schurig
 
 import os
 import sys
@@ -42,9 +42,8 @@ logger = logging.getLogger('BitBake.Build')
 
 NULL = open(os.devnull, 'r+')
 
-
-# When we execute a python function we'd like certain things
-# in all namespaces, hence we add them to __builtins__
+# When we execute a Python function, we'd like certain things
+# in all namespaces, hence we add them to __builtins__.
 # If we do not do this and use the exec globals, they will
 # not be available to subfunctions.
 __builtins__['bb'] = bb
@@ -143,7 +142,7 @@ class LogTee(object):
         self.outfile.flush()
 
 def exec_func(func, d, dirs = None):
-    """Execute an BB 'function'"""
+    """Execute a BB 'function'"""
 
     body = d.getVar(func)
     if not body:
@@ -228,7 +227,7 @@ def exec_func_python(func, d, runfile, cwd=None):
     code = _functionfmt.format(function=func, body=d.getVar(func, True))
     bb.utils.mkdirhier(os.path.dirname(runfile))
     with open(runfile, 'w') as script:
-        script.write(code)
+        bb.data.emit_func_python(func, script, d)
 
     if cwd:
         try:
@@ -242,10 +241,9 @@ def exec_func_python(func, d, runfile, cwd=None):
     try:
         comp = utils.better_compile(code, func, bbfile)
         utils.better_exec(comp, {"d": d}, code, bbfile)
+    except (bb.parse.SkipRecipe, bb.build.FuncFailed):
+        raise
     except:
-        if sys.exc_info()[0] in (bb.parse.SkipPackage, bb.build.FuncFailed):
-            raise
-
         raise FuncFailed(func, None)
     finally:
         bb.debug(2, "Python function %s finished" % func)
@@ -418,7 +416,7 @@ def _exec_task(fn, task, d, quieterr):
     os.dup2(logfile.fileno(), oso[1])
     os.dup2(logfile.fileno(), ose[1])
 
-    # Ensure python logging goes to the logfile
+    # Ensure Python logging goes to the logfile
     handler = logging.StreamHandler(logfile)
     handler.setFormatter(logformatter)
     # Always enable full debug output into task logfiles
@@ -507,7 +505,7 @@ def exec_task(fn, task, d, profile = False):
             event.fire(failedevent, d)
         return 1
 
-def stamp_internal(taskname, d, file_name):
+def stamp_internal(taskname, d, file_name, baseonly=False):
     """
     Internal stamp helper function
     Makes sure the stamp directory exists
@@ -528,6 +526,9 @@ def stamp_internal(taskname, d, file_name):
         file_name = d.getVar('BB_FILENAME', True)
         extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info', True) or ""
 
+    if baseonly:
+        return stamp
+
     if not stamp:
         return
 
@@ -592,8 +593,9 @@ def make_stamp(task, d, file_name = None):
     # If we're in task context, write out a signature file for each task
     # as it completes
     if not task.endswith("_setscene") and task != "do_setscene" and not file_name:
+        stampbase = stamp_internal(task, d, None, True)
         file_name = d.getVar('BB_FILENAME', True)
-        bb.parse.siggen.dump_sigtask(file_name, task, d.getVar('STAMP', True), True)
+        bb.parse.siggen.dump_sigtask(file_name, task, stampbase, True)
 
 def del_stamp(task, d, file_name = None):
     """

bitbake/lib/bb/cache.py

@@ -225,14 +225,16 @@ class CoreRecipeInfo(RecipeInfoCommon):
         for package in self.packages_dynamic:
             cachedata.packages_dynamic[package].append(fn)
 
-        # Build hash of runtime depends and rececommends
+        # Build hash of runtime depends and recommends
         for package in self.packages + [self.pn]:
             cachedata.rundeps[fn][package] = list(self.rdepends) + self.rdepends_pkg[package]
             cachedata.runrecs[fn][package] = list(self.rrecommends) + self.rrecommends_pkg[package]
 
         # Collect files we may need for possible world-dep
         # calculations
-        if not self.not_world:
+        if self.not_world:
+            logger.debug(1, "EXCLUDE FROM WORLD: %s", fn)
+        else:
             cachedata.possible_world.append(fn)
 
         # create a collection of all targets for sanity checking
@@ -259,7 +261,7 @@ class Cache(object):
 
     def __init__(self, data, data_hash, caches_array):
         # Pass caches_array information into Cache Constructor
-        # It will be used in later for deciding whether we 
+        # It will be used later for deciding whether we 
         # need extra cache file dump/load support 
         self.caches_array = caches_array
         self.cachedir = data.getVar("CACHE", True)
@@ -692,7 +694,7 @@ def init(cooker):
 
     * Its mtime
     * The mtimes of all its dependencies
-    * Whether it caused a parse.SkipPackage exception
+    * Whether it caused a parse.SkipRecipe exception
 
     Files causing parsing errors are evicted from the cache.
 
@@ -762,16 +764,6 @@ class MultiProcessCache(object):
 
         self.cachedata = data
 
-    def internSet(self, items):
-        new = set()
-        for i in items:
-            new.add(intern(i))
-        return new
-
-    def compress_keys(self, data):
-        # Override in subclasses if desired
-        return
-
     def create_cachedata(self):
         data = [{}]
         return data
@@ -812,15 +804,7 @@ class MultiProcessCache(object):
 
         glf = bb.utils.lockfile(self.cachefile + ".lock")
 
-        try:
-            with open(self.cachefile, "rb") as f:
-                p = pickle.Unpickler(f)
-                data, version = p.load()
-        except (IOError, EOFError):
-            data, version = None, None
-
-        if version != self.__class__.CACHE_VERSION:
-            data = self.create_cachedata()
+        data = self.cachedata
 
         for f in [y for y in os.listdir(os.path.dirname(self.cachefile)) if y.startswith(os.path.basename(self.cachefile) + '-')]:
             f = os.path.join(os.path.dirname(self.cachefile), f)
@@ -829,16 +813,16 @@ class MultiProcessCache(object):
                     p = pickle.Unpickler(fd)
                     extradata, version = p.load()
             except (IOError, EOFError):
-                extradata, version = self.create_cachedata(), None
+                os.unlink(f)
+                continue
 
             if version != self.__class__.CACHE_VERSION:
+                os.unlink(f)
                 continue
 
             self.merge_data(extradata, data)
             os.unlink(f)
 
-        self.compress_keys(data)
-
         with open(self.cachefile, "wb") as f:
             p = pickle.Pickler(f, -1)
             p.dump([data, self.__class__.CACHE_VERSION])

bitbake/lib/bb/codeparser.py

@@ -33,9 +33,82 @@ def check_indent(codestr):
     return codestr
 
 
+# Basically pickle, in python 2.7.3 at least, does badly with data duplication 
+# upon pickling and unpickling. Combine this with duplicate objects and things
+# are a mess.
+#
+# When the sets are originally created, python calls intern() on the set keys
+# which significantly improves memory usage. Sadly the pickle/unpickle process
+# doesn't call intern() on the keys and results in the same strings being duplicated
+# in memory. This also means pickle will save the same string multiple times in
+# the cache file.
+#
+# By having shell and python cacheline objects with setstate/getstate, we force
+# the object creation through our own routine where we can call intern (via internSet).
+#
+# We also use hashable frozensets and ensure we use references to these so that
+# duplicates can be removed, both in memory and in the resulting pickled data.
+#
+# By playing these games, the size of the cache file shrinks dramatically
+# meaning faster load times and the reloaded cache files also consume much less
+# memory. Smaller cache files, faster load times and lower memory usage is good.
+#
+# A custom getstate/setstate using tuples is actually worth 15% cachesize by
+# avoiding duplication of the attribute names!
+
+class SetCache(object):
+    def __init__(self):
+        self.setcache = {}
+
+    def internSet(self, items):
+        
+        new = []
+        for i in items:
+            new.append(intern(i))
+        s = frozenset(new)
+        if hash(s) in self.setcache:
+            return self.setcache[hash(s)]
+        self.setcache[hash(s)] = s
+        return s
+
+codecache = SetCache()
+
+class pythonCacheLine(object):
+    def __init__(self, refs, execs, contains):
+        self.refs = codecache.internSet(refs)
+        self.execs = codecache.internSet(execs)
+        self.contains = {}
+        for c in contains:
+            self.contains[c] = codecache.internSet(contains[c])
+
+    def __getstate__(self):
+        return (self.refs, self.execs, self.contains)
+
+    def __setstate__(self, state):
+        (refs, execs, contains) = state
+        self.__init__(refs, execs, contains)
+    def __hash__(self):
+        l = (hash(self.refs), hash(self.execs))
+        for c in sorted(self.contains.keys()):
+            l = l + (c, hash(self.contains[c]))
+        return hash(l)
+
+class shellCacheLine(object):
+    def __init__(self, execs):
+        self.execs = codecache.internSet(execs)
+
+    def __getstate__(self):
+        return (self.execs)
+
+    def __setstate__(self, state):
+        (execs) = state
+        self.__init__(execs)
+    def __hash__(self):
+        return hash(self.execs)
+
 class CodeParserCache(MultiProcessCache):
     cache_file_name = "bb_codeparser.dat"
-    CACHE_VERSION = 4
+    CACHE_VERSION = 7
 
     def __init__(self):
         MultiProcessCache.__init__(self)
@@ -44,6 +117,27 @@ class CodeParserCache(MultiProcessCache):
         self.pythoncacheextras = self.cachedata_extras[0]
         self.shellcacheextras = self.cachedata_extras[1]
 
+        # To avoid duplication in the codeparser cache, keep
+        # a lookup of hashes of objects we already have
+        self.pythoncachelines = {}
+        self.shellcachelines = {}
+
+    def newPythonCacheLine(self, refs, execs, contains):
+        cacheline = pythonCacheLine(refs, execs, contains)
+        h = hash(cacheline)
+        if h in self.pythoncachelines:
+            return self.pythoncachelines[h]
+        self.pythoncachelines[h] = cacheline
+        return cacheline
+
+    def newShellCacheLine(self, execs):
+        cacheline = shellCacheLine(execs)
+        h = hash(cacheline)
+        if h in self.shellcachelines:
+            return self.shellcachelines[h]
+        self.shellcachelines[h] = cacheline
+        return cacheline
+
     def init_cache(self, d):
         MultiProcessCache.init_cache(self, d)
 
@@ -51,25 +145,6 @@ class CodeParserCache(MultiProcessCache):
         self.pythoncache = self.cachedata[0]
         self.shellcache = self.cachedata[1]
 
-    def compress_keys(self, data):
-        # When the dicts are originally created, python calls intern() on the set keys
-        # which significantly improves memory usage. Sadly the pickle/unpickle process
-        # doesn't call intern() on the keys and results in the same strings being duplicated
-        # in memory. This also means pickle will save the same string multiple times in
-        # the cache file. By interning the data here, the cache file shrinks dramatically
-        # meaning faster load times and the reloaded cache files also consume much less
-        # memory. This is worth any performance hit from this loops and the use of the
-        # intern() data storage.
-        # Python 3.x may behave better in this area
-        for h in data[0]:
-            data[0][h]["refs"] = self.internSet(data[0][h]["refs"])
-            data[0][h]["execs"] = self.internSet(data[0][h]["execs"])
-            for k in data[0][h]["contains"]:
-                data[0][h]["contains"][k] = self.internSet(data[0][h]["contains"][k])
-        for h in data[1]:
-            data[1][h]["execs"] = self.internSet(data[1][h]["execs"])
-        return
-
     def create_cachedata(self):
         data = [{}, {}]
         return data
@@ -102,8 +177,8 @@ class BufferedLogger(Logger):
         self.buffer = []
 
 class PythonParser():
-    getvars = ("d.getVar", "bb.data.getVar", "data.getVar", "d.appendVar", "d.prependVar")
-    containsfuncs = ("bb.utils.contains", "base_contains", "oe.utils.contains")
+    getvars = (".getVar", ".appendVar", ".prependVar")
+    containsfuncs = ("bb.utils.contains", "base_contains", "oe.utils.contains", "bb.utils.contains_any")
     execfuncs = ("bb.build.exec_func", "bb.build.exec_task")
 
     def warn(self, func, arg):
@@ -122,7 +197,7 @@ class PythonParser():
 
     def visit_Call(self, node):
         name = self.called_node_name(node.func)
-        if name in self.getvars or name in self.containsfuncs:
+        if name and name.endswith(self.getvars) or name in self.containsfuncs:
             if isinstance(node.args[0], ast.Str):
                 varname = node.args[0].s
                 if name in self.containsfuncs and isinstance(node.args[1], ast.Str):
@@ -168,15 +243,19 @@ class PythonParser():
         h = hash(str(node))
 
         if h in codeparsercache.pythoncache:
-            self.references = codeparsercache.pythoncache[h]["refs"]
-            self.execs = codeparsercache.pythoncache[h]["execs"]
-            self.contains = codeparsercache.pythoncache[h]["contains"]
+            self.references = set(codeparsercache.pythoncache[h].refs)
+            self.execs = set(codeparsercache.pythoncache[h].execs)
+            self.contains = {}
+            for i in codeparsercache.pythoncache[h].contains:
+                self.contains[i] = set(codeparsercache.pythoncache[h].contains[i])
             return
 
         if h in codeparsercache.pythoncacheextras:
-            self.references = codeparsercache.pythoncacheextras[h]["refs"]
-            self.execs = codeparsercache.pythoncacheextras[h]["execs"]
-            self.contains = codeparsercache.pythoncacheextras[h]["contains"]
+            self.references = set(codeparsercache.pythoncacheextras[h].refs)
+            self.execs = set(codeparsercache.pythoncacheextras[h].execs)
+            self.contains = {}
+            for i in codeparsercache.pythoncacheextras[h].contains:
+                self.contains[i] = set(codeparsercache.pythoncacheextras[h].contains[i])
             return
 
         code = compile(check_indent(str(node)), "<string>", "exec",
@@ -188,10 +267,7 @@ class PythonParser():
 
         self.execs.update(self.var_execs)
 
-        codeparsercache.pythoncacheextras[h] = {}
-        codeparsercache.pythoncacheextras[h]["refs"] = self.references
-        codeparsercache.pythoncacheextras[h]["execs"] = self.execs
-        codeparsercache.pythoncacheextras[h]["contains"] = self.contains
+        codeparsercache.pythoncacheextras[h] = codeparsercache.newPythonCacheLine(self.references, self.execs, self.contains)
 
 class ShellParser():
     def __init__(self, name, log):
@@ -210,13 +286,21 @@ class ShellParser():
         h = hash(str(value))
 
         if h in codeparsercache.shellcache:
-            self.execs = codeparsercache.shellcache[h]["execs"]
+            self.execs = set(codeparsercache.shellcache[h].execs)
             return self.execs
 
         if h in codeparsercache.shellcacheextras:
-            self.execs = codeparsercache.shellcacheextras[h]["execs"]
+            self.execs = set(codeparsercache.shellcacheextras[h].execs)
             return self.execs
 
+        self._parse_shell(value)
+        self.execs = set(cmd for cmd in self.allexecs if cmd not in self.funcdefs)
+
+        codeparsercache.shellcacheextras[h] = codeparsercache.newShellCacheLine(self.execs)
+
+        return self.execs
+
+    def _parse_shell(self, value):
         try:
             tokens, _ = pyshyacc.parse(value, eof=True, debug=False)
         except pyshlex.NeedMore:
@@ -224,12 +308,6 @@ class ShellParser():
 
         for token in tokens:
             self.process_tokens(token)
-        self.execs = set(cmd for cmd in self.allexecs if cmd not in self.funcdefs)
-
-        codeparsercache.shellcacheextras[h] = {}
-        codeparsercache.shellcacheextras[h]["execs"] = self.execs
-
-        return self.execs
 
     def process_tokens(self, tokens):
         """Process a supplied portion of the syntax tree as returned by
@@ -303,7 +381,7 @@ class ShellParser():
 
                 if part[0] in ('`', '$('):
                     command = pyshlex.wordtree_as_string(part[1:-1])
-                    self.parse_shell(command)
+                    self._parse_shell(command)
 
                     if word[0] in ("cmd_name", "cmd_word"):
                         if word in words:
@@ -322,7 +400,7 @@ class ShellParser():
                     self.log.debug(1, self.unhandled_template % cmd)
                 elif cmd == "eval":
                     command = " ".join(word for _, word in words[1:])
-                    self.parse_shell(command)
+                    self._parse_shell(command)
                 else:
                     self.allexecs.add(cmd)
                 break

bitbake/lib/bb/command.py

@@ -86,7 +86,10 @@ class Command:
 
     def runAsyncCommand(self):
         try:
-            if self.cooker.state == bb.cooker.state.error:
+            if self.cooker.state in (bb.cooker.state.error, bb.cooker.state.shutdown, bb.cooker.state.forceshutdown):
+                # updateCache will trigger a shutdown of the parser
+                # and then raise BBHandledException triggering an exit
+                self.cooker.updateCache()
                 return False
             if self.currentAsyncCommand is not None:
                 (command, options) = self.currentAsyncCommand
@@ -268,6 +271,10 @@ class CommandsSync:
     # we always take and leave the cooker in state.initial
     setFeatures.readonly = True
 
+    def updateConfig(self, command, params):
+        options = params[0]
+        command.cooker.updateConfigOpts(options)
+
 class CommandsAsync:
     """
     A class of asynchronous commands

bitbake/lib/bb/cooker.py

@@ -371,6 +371,10 @@ class BBCooker:
 
         self.handleCollections( self.data.getVar("BBFILE_COLLECTIONS", True) )
 
+    def updateConfigOpts(self,options):
+        for o in options:
+            setattr(self.configuration, o, options[o])
+
     def runCommands(self, server, data, abort):
         """
         Run any queued asynchronous command
@@ -402,7 +406,7 @@ class BBCooker:
 
     def showEnvironment(self, buildfile = None, pkgs_to_build = []):
         """
-        Show the outer or per-package environment
+        Show the outer or per-recipe environment
         """
         fn = None
         envdata = None
@@ -818,7 +822,6 @@ class BBCooker:
         or to find all machine configuration files one could call:
         findFilesMatchingInDir(self, 'conf/machines', 'conf')
         """
-        import re
 
         matches = []
         p = re.compile(re.escape(filepattern))
@@ -1228,7 +1231,6 @@ class BBCooker:
         '''
         Create a new image with a "require"/"inherit" base_image statement
         '''
-        import re
         if timestamp:
             image_name = os.path.splitext(image)[0]
             timestr = time.strftime("-%Y%m%d-%H%M%S")
@@ -1437,7 +1439,7 @@ class CookerCollectFiles(object):
             for ignored in ('SCCS', 'CVS', '.svn'):
                 if ignored in dirs:
                     dirs.remove(ignored)
-            found += [os.path.join(dir, f) for f in files if (f.endswith('.bb') or f.endswith('.bbappend'))]
+            found += [os.path.join(dir, f) for f in files if (f.endswith(['.bb', '.bbappend']))]
 
         return found
 

bitbake/lib/bb/cookerdata.py

@@ -69,6 +69,17 @@ class ConfigParameters(object):
             if bbpkgs:
                 self.options.pkgs_to_build.extend(bbpkgs.split())
 
+    def updateToServer(self, server):
+        options = {}
+        for o in ["abort", "tryaltconfigs", "force", "invalidate_stamp", 
+                  "verbose", "debug", "dry_run", "dump_signatures", 
+                  "debug_domains", "extra_assume_provided", "profile"]:
+            options[o] = getattr(self.options, o)
+
+        ret, error = server.runCommand(["updateConfig", options])
+        if error:
+                raise Exception("Unable to update the server configuration with local parameters: %s" % error)
+
     def parseActions(self):
         # Parse any commandline into actions
         action = {'action':None, 'msg':None}
@@ -227,10 +238,13 @@ class CookerDataBuilder(object):
         try:
             self.parseConfigurationFiles(self.prefiles, self.postfiles)
         except SyntaxError:
-            sys.exit(1)
+            raise bb.BBHandledException
+        except bb.data_smart.ExpansionError as e:
+            logger.error(str(e))
+            raise bb.BBHandledException
         except Exception:
             logger.exception("Error parsing configuration files")
-            sys.exit(1)
+            raise bb.BBHandledException
 
     def _findLayerConf(self, data):
         return findConfigFile("bblayers.conf", data)

bitbake/lib/bb/daemonize.py

@@ -1,5 +1,5 @@
 """
-Python Deamonizing helper
+Python Daemonizing helper
 
 Configurable daemon behaviors:
 
@@ -12,8 +12,11 @@ A failed call to fork() now raises an exception.
 
 References:
     1) Advanced Programming in the Unix Environment: W. Richard Stevens
-    2) Unix Programming Frequently Asked Questions:
-            http://www.erlenstar.demon.co.uk/unix/faq_toc.html
+	http://www.apuebook.com/apue3e.html
+    2) The Linux Programming Interface: Michael Kerrisk
+	http://man7.org/tlpi/index.html
+    3) Unix Programming Frequently Asked Questions:
+	http://www.faqs.org/faqs/unix-faq/programmer/faq/
 
 Modified to allow a function to be daemonized and return for 
 bitbake use by Richard Purdie
@@ -25,7 +28,7 @@ __version__ = "0.2"
 
 # Standard Python modules.
 import os                    # Miscellaneous OS interfaces.
-import sys                  # System-specific parameters and functions.
+import sys                   # System-specific parameters and functions.
 
 # Default daemon parameters.
 # File mode creation mask of the daemon.
@@ -128,7 +131,7 @@ def createDaemon(function, logfile):
     # of methods to accomplish this task.  Three are listed below.
     #
     # Try the system configuration variable, SC_OPEN_MAX, to obtain the maximum
-    # number of open file descriptors to close.  If it doesn't exists, use
+    # number of open file descriptors to close.  If it doesn't exist, use
     # the default value (configurable).
     #
     # try:
@@ -146,7 +149,7 @@ def createDaemon(function, logfile):
     # OR
     #
     # Use the getrlimit method to retrieve the maximum file descriptor number
-    # that can be opened by this process.  If there is not limit on the
+    # that can be opened by this process.  If there is no limit on the
     # resource, use the default value.
     #
     import resource             # Resource usage information.

bitbake/lib/bb/data.py

@@ -6,7 +6,7 @@ BitBake 'Data' implementations
 Functions for interacting with the data structure used by the
 BitBake build tools.
 
-The expandData and update_data are the most expensive
+The expandKeys and update_data are the most expensive
 operations. At night the cookie monster came by and
 suggested 'give me cookies on setting the variables and
 things will work out'. Taking this suggestion into account
@@ -15,7 +15,7 @@ Analyse von Algorithmen' lecture and the cookie
 monster seems to be right. We will track setVar more carefully
 to have faster update_data and expandKeys operations.
 
-This is a treade-off between speed and memory again but
+This is a trade-off between speed and memory again but
 the speed is more critical here.
 """
 
@@ -35,7 +35,7 @@ the speed is more critical here.
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 #
-#Based on functions from the base bb module, Copyright 2003 Holger Schurig
+# Based on functions from the base bb module, Copyright 2003 Holger Schurig
 
 import sys, os, re
 if sys.argv[0][-5:] == "pydoc":
@@ -281,6 +281,41 @@ def emit_func(func, o=sys.__stdout__, d = init()):
                newdeps |= set((d.getVarFlag(dep, "vardeps", True) or "").split())
         newdeps -= seen
 
+_functionfmt = """
+def {function}(d):
+{body}"""
+
+def emit_func_python(func, o=sys.__stdout__, d = init()):
+    """Emits all items in the data store in a format such that it can be sourced by a shell."""
+
+    def write_func(func, o, call = False):
+        body = d.getVar(func, True)
+        if not body.startswith("def"):
+            body = _functionfmt.format(function=func, body=body)
+
+        o.write(body.strip() + "\n\n")
+        if call:
+            o.write(func + "(d)" + "\n\n")
+
+    write_func(func, o, True)
+    pp = bb.codeparser.PythonParser(func, logger)
+    pp.parse_python(d.getVar(func, True))
+    newdeps = pp.execs
+    newdeps |= set((d.getVarFlag(func, "vardeps", True) or "").split())
+    seen = set()
+    while newdeps:
+        deps = newdeps
+        seen |= deps
+        newdeps = set()
+        for dep in deps:
+            if d.getVarFlag(dep, "func") and d.getVarFlag(dep, "python"):
+               write_func(dep, o)
+               pp = bb.codeparser.PythonParser(dep, logger)
+               pp.parse_python(d.getVar(dep, True))
+               newdeps |= pp.execs
+               newdeps |= set((d.getVarFlag(dep, "vardeps", True) or "").split())
+        newdeps -= seen
+
 def update_data(d):
     """Performs final steps upon the datastore, including application of overrides"""
     d.finalize(parent = True)

bitbake/lib/bb/data_smart.py

@@ -263,7 +263,7 @@ class VariableHistory(object):
                     flag = ''
                 o.write("#   %s %s:%s%s\n#     %s\"%s\"\n" % (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n#     ', event['detail'])))
             if len(history) > 1:
-                o.write("# computed:\n")
+                o.write("# pre-expansion value:\n")
                 o.write('#   "%s"\n' % (commentVal))
         else:
             o.write("#\n# $%s\n#   [no history recorded]\n#\n" % var)
@@ -334,7 +334,7 @@ class DataSmart(MutableMapping):
                     break
             except ExpansionError:
                 raise
-            except bb.parse.SkipPackage:
+            except bb.parse.SkipRecipe:
                 raise
             except Exception as exc:
                 raise ExpansionError(varname, s, exc)
@@ -513,10 +513,15 @@ class DataSmart(MutableMapping):
     def _setvar_update_overrides(self, var):
         # aka pay the cookie monster
         override = var[var.rfind('_')+1:]
-        if len(override) > 0:
+        shortvar = var[:var.rfind('_')]
+        while override:
             if override not in self._seen_overrides:
                 self._seen_overrides[override] = set()
             self._seen_overrides[override].add( var )
+            override = None
+            if "_" in shortvar:
+                override = var[shortvar.rfind('_')+1:]
+                shortvar = var[:shortvar.rfind('_')]
 
     def getVar(self, var, expand=False, noweakdefault=False):
         return self.getVarFlag(var, "_content", expand, noweakdefault)
@@ -610,8 +615,9 @@ class DataSmart(MutableMapping):
             else:
                 cachename = var + "[" + flag + "]"
             value = self.expand(value, cachename)
-        if value is not None and flag == "_content" and local_var is not None and "_removeactive" in local_var:
-            filtered = filter(lambda v: v not in local_var["_removeactive"],
+        if value and flag == "_content" and local_var is not None and "_removeactive" in local_var:
+            removes = [self.expand(r) for r in local_var["_removeactive"]]
+            filtered = filter(lambda v: v not in removes,
                               value.split(" "))
             value = " ".join(filtered)
             if expand:

bitbake/lib/bb/event.py

@@ -72,7 +72,7 @@ def execute_handler(name, handler, event, d):
     event.data = d
     try:
         ret = handler(event)
-    except bb.parse.SkipPackage:
+    except (bb.parse.SkipRecipe, bb.BBHandledException):
         raise
     except Exception:
         etype, value, tb = sys.exc_info()
@@ -94,10 +94,7 @@ def fire_class_handlers(event, d):
     evt_hmap = _event_handler_map.get(eid, {})
     for name, handler in _handlers.iteritems():
         if name in _catchall_handlers or name in evt_hmap:
-            try:
-                execute_handler(name, handler, event, d)
-            except Exception:
-                continue
+            execute_handler(name, handler, event, d)
 
 ui_queue = []
 @atexit.register
@@ -597,11 +594,11 @@ class MetadataEvent(Event):
     def __init__(self, eventtype, eventdata):
         Event.__init__(self)
         self.type = eventtype
-        self.data = eventdata
+        self._localdata = eventdata
 
 class SanityCheck(Event):
     """
-    Event to runs sanity checks, either raise errors or generate events as return status.
+    Event to run sanity checks, either raise errors or generate events as return status.
     """
     def __init__(self, generateevents = True):
         Event.__init__(self)
@@ -609,7 +606,7 @@ class SanityCheck(Event):
 
 class SanityCheckPassed(Event):
     """
-    Event to indicate sanity check is passed
+    Event to indicate sanity check has passed
     """
 
 class SanityCheckFailed(Event):

bitbake/lib/bb/fetch2/__init__.py

@@ -56,8 +56,11 @@ class BBFetchException(Exception):
 
 class MalformedUrl(BBFetchException):
     """Exception raised when encountering an invalid url"""
-    def __init__(self, url):
-         msg = "The URL: '%s' is invalid and cannot be interpreted" % url
+    def __init__(self, url, message=''):
+         if message:
+             msg = message
+         else:
+             msg = "The URL: '%s' is invalid and cannot be interpreted" % url
          self.url = url
          BBFetchException.__init__(self, msg)
          self.args = (url,)
@@ -371,8 +374,11 @@ def decodeurl(url):
     p = {}
     if parm:
         for s in parm.split(';'):
-            s1, s2 = s.split('=')
-            p[s1] = s2
+            if s:
+                if not '=' in s:
+                    raise MalformedUrl(url, "The URL: '%s' is invalid: parameter %s does not specify a value (missing '=')" % (url, s))
+                s1, s2 = s.split('=')
+                p[s1] = s2
 
     return type, host, urllib.unquote(path), user, pswd, p
 
@@ -537,8 +543,8 @@ def verify_checksum(ud, d):
 
     if ud.method.recommends_checksum(ud):
         # If strict checking enabled and neither sum defined, raise error
-        strict = d.getVar("BB_STRICT_CHECKSUM", True) or None
-        if strict and not (ud.md5_expected or ud.sha256_expected):
+        strict = d.getVar("BB_STRICT_CHECKSUM", True) or "0"
+        if (strict == "1") and not (ud.md5_expected or ud.sha256_expected):
             logger.error('No checksum specified for %s, please add at least one to the recipe:\n'
                              'SRC_URI[%s] = "%s"\nSRC_URI[%s] = "%s"' %
                              (ud.localpath, ud.md5_name, md5data,
@@ -975,7 +981,6 @@ def get_file_checksums(filelist, pn):
 
     checksums = []
     for pth in filelist.split():
-        checksum = None
         if '*' in pth:
             # Handle globs
             for f in glob.glob(pth):
@@ -985,15 +990,12 @@ def get_file_checksums(filelist, pn):
                     checksum = checksum_file(f)
                     if checksum:
                         checksums.append((f, checksum))
-            continue
         elif os.path.isdir(pth):
             checksums.extend(checksum_dir(pth))
-            continue
         else:
             checksum = checksum_file(pth)
-
-        if checksum:
-            checksums.append((pth, checksum))
+            if checksum:
+                checksums.append((pth, checksum))
 
     checksums.sort(key=operator.itemgetter(1))
     return checksums
@@ -1255,17 +1257,22 @@ class FetchMethod(object):
                         destdir = destdir.strip('/')
                     if destdir != "." and not os.access("%s/%s" % (rootdir, destdir), os.F_OK):
                         os.makedirs("%s/%s" % (rootdir, destdir))
-                    cmd = 'cp -pPR %s %s/%s/' % (file, rootdir, destdir)
+                    cmd = 'cp -fpPR %s %s/%s/' % (file, rootdir, destdir)
                     #cmd = 'tar -cf - -C "%d" -ps . | tar -xf - -C "%s/%s/"' % (file, rootdir, destdir)
                 else:
                     # The "destdir" handling was specifically done for FILESPATH
                     # items.  So, only do so for file:// entries.
                     if urldata.type == "file" and urldata.path.find("/") != -1:
                        destdir = urldata.path.rsplit("/", 1)[0]
+                       if urldata.parm.get('subdir') != None:
+                          destdir = urldata.parm.get('subdir') + "/" + destdir
                     else:
-                       destdir = "."
+                       if urldata.parm.get('subdir') != None:
+                          destdir = urldata.parm.get('subdir')
+                       else:
+                          destdir = "."
                     bb.utils.mkdirhier("%s/%s" % (rootdir, destdir))
-                    cmd = 'cp %s %s/%s/' % (file, rootdir, destdir)
+                    cmd = 'cp -f %s %s/%s/' % (file, rootdir, destdir)
 
         if not cmd:
             return
@@ -1558,6 +1565,7 @@ from . import bzr
 from . import hg
 from . import osc
 from . import repo
+from . import clearcase
 
 methods.append(local.Local())
 methods.append(wget.Wget())
@@ -1573,3 +1581,4 @@ methods.append(bzr.Bzr())
 methods.append(hg.Hg())
 methods.append(osc.Osc())
 methods.append(repo.Repo())
+methods.append(clearcase.ClearCase())

bitbake/lib/bb/fetch2/clearcase.py

@@ -0,0 +1,263 @@
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+"""
+BitBake 'Fetch' clearcase implementation
+
+The clearcase fetcher is used to retrieve files from a ClearCase repository.
+
+Usage in the recipe:
+
+    SRC_URI = "ccrc://cc.example.org/ccrc;vob=/example_vob;module=/example_module"
+    SRCREV = "EXAMPLE_CLEARCASE_TAG"
+    PV = "${@d.getVar("SRCREV").replace("/", "+")}"
+
+The fetcher uses the rcleartool or cleartool remote client, depending on which one is available.
+
+Supported SRC_URI options are:
+
+- vob
+    (required) The name of the clearcase VOB (with prepending "/")
+
+- module
+    The module in the selected VOB (with prepending "/")
+
+    The module and vob parameters are combined to create
+    the following load rule in the view config spec:
+                load <vob><module>
+
+- proto
+    http or https
+
+Related variables:
+
+    CCASE_CUSTOM_CONFIG_SPEC
+            Write a config spec to this variable in your recipe to use it instead
+            of the default config spec generated by this fetcher.
+            Please note that the SRCREV loses its functionality if you specify
+            this variable. SRCREV is still used to label the archive after a fetch,
+            but it doesn't define what's fetched.
+
+User credentials:
+    cleartool:
+            The login of cleartool is handled by the system. No special steps needed.
+
+    rcleartool:
+            In order to use rcleartool with authenticated users an `rcleartool login` is
+            necessary before using the fetcher.
+"""
+# Copyright (C) 2014 Siemens AG
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+
+import os
+import sys
+import shutil
+import bb
+from   bb import data
+from   bb.fetch2 import FetchMethod
+from   bb.fetch2 import FetchError
+from   bb.fetch2 import runfetchcmd
+from   bb.fetch2 import logger
+from   distutils import spawn
+
+class ClearCase(FetchMethod):
+    """Class to fetch urls via 'clearcase'"""
+    def init(self, d):
+        pass
+
+    def supports(self, ud, d):
+        """
+        Check to see if a given url can be fetched with Clearcase.
+        """
+        return ud.type in ['ccrc']
+
+    def debug(self, msg):
+        logger.debug(1, "ClearCase: %s", msg)
+
+    def urldata_init(self, ud, d):
+        """
+        init ClearCase specific variable within url data
+        """
+        ud.proto = "https"
+        if 'protocol' in ud.parm:
+            ud.proto = ud.parm['protocol']
+        if not ud.proto in ('http', 'https'):
+            raise fetch2.ParameterError("Invalid protocol type", ud.url)
+
+        ud.vob = ''
+        if 'vob' in ud.parm:
+            ud.vob = ud.parm['vob']
+        else:
+            msg = ud.url+": vob must be defined so the fetcher knows what to get."
+            raise MissingParameterError('vob', msg)
+
+        if 'module' in ud.parm:
+            ud.module = ud.parm['module']
+        else:
+            ud.module = ""
+
+        ud.basecmd = d.getVar("FETCHCMD_ccrc", True) or spawn.find_executable("cleartool") or spawn.find_executable("rcleartool")
+
+        if data.getVar("SRCREV", d, True) == "INVALID":
+          raise FetchError("Set a valid SRCREV for the clearcase fetcher in your recipe, e.g. SRCREV = \"/main/LATEST\" or any other label of your choice.")
+
+        ud.label = d.getVar("SRCREV")
+        ud.customspec = d.getVar("CCASE_CUSTOM_CONFIG_SPEC", True)
+
+        ud.server     = "%s://%s%s" % (ud.proto, ud.host, ud.path)
+
+        ud.identifier = "clearcase-%s%s-%s" % ( ud.vob.replace("/", ""),
+                                                ud.module.replace("/", "."),
+                                                ud.label.replace("/", "."))
+
+        ud.viewname         = "%s-view%s" % (ud.identifier, d.getVar("DATETIME", d, True))
+        ud.csname           = "%s-config-spec" % (ud.identifier)
+        ud.ccasedir         = os.path.join(data.getVar("DL_DIR", d, True), ud.type)
+        ud.viewdir          = os.path.join(ud.ccasedir, ud.viewname)
+        ud.configspecfile   = os.path.join(ud.ccasedir, ud.csname)
+        ud.localfile        = "%s.tar.gz" % (ud.identifier)
+
+        self.debug("host            = %s" % ud.host)
+        self.debug("path            = %s" % ud.path)
+        self.debug("server          = %s" % ud.server)
+        self.debug("proto           = %s" % ud.proto)
+        self.debug("type            = %s" % ud.type)
+        self.debug("vob             = %s" % ud.vob)
+        self.debug("module          = %s" % ud.module)
+        self.debug("basecmd         = %s" % ud.basecmd)
+        self.debug("label           = %s" % ud.label)
+        self.debug("ccasedir        = %s" % ud.ccasedir)
+        self.debug("viewdir         = %s" % ud.viewdir)
+        self.debug("viewname        = %s" % ud.viewname)
+        self.debug("configspecfile  = %s" % ud.configspecfile)
+        self.debug("localfile       = %s" % ud.localfile)
+
+        ud.localfile = os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
+
+    def _build_ccase_command(self, ud, command):
+        """
+        Build up a commandline based on ud
+        command is: mkview, setcs, rmview
+        """
+        options = []
+
+        if "rcleartool" in ud.basecmd:
+            options.append("-server %s" % ud.server)
+
+        basecmd = "%s %s" % (ud.basecmd, command)
+
+        if command is 'mkview':
+            if not "rcleartool" in ud.basecmd:
+                # Cleartool needs a -snapshot view
+                options.append("-snapshot")
+            options.append("-tag %s" % ud.viewname)
+            options.append(ud.viewdir)
+
+        elif command is 'rmview':
+            options.append("-force")
+            options.append("%s" % ud.viewdir)
+
+        elif command is 'setcs':
+            options.append("-overwrite")
+            options.append(ud.configspecfile)
+
+        else:
+            raise FetchError("Invalid ccase command %s" % command)
+
+        ccasecmd = "%s %s" % (basecmd, " ".join(options))
+        self.debug("ccasecmd = %s" % ccasecmd)
+        return ccasecmd
+
+    def _write_configspec(self, ud, d):
+        """
+        Create config spec file (ud.configspecfile) for ccase view
+        """
+        config_spec = ""
+        custom_config_spec = d.getVar("CCASE_CUSTOM_CONFIG_SPEC", d)
+        if custom_config_spec is not None:
+            for line in custom_config_spec.split("\\n"):
+                config_spec += line+"\n"
+            bb.warn("A custom config spec has been set, SRCREV is only relevant for the tarball name.")
+        else:
+            config_spec += "element * CHECKEDOUT\n"
+            config_spec += "element * %s\n" % ud.label
+            config_spec += "load %s%s\n" % (ud.vob, ud.module)
+
+        logger.info("Using config spec: \n%s" % config_spec)
+
+        with open(ud.configspecfile, 'w') as f:
+            f.write(config_spec)
+
+    def _remove_view(self, ud, d):
+        if os.path.exists(ud.viewdir):
+            os.chdir(ud.ccasedir)
+            cmd = self._build_ccase_command(ud, 'rmview');
+            logger.info("cleaning up [VOB=%s label=%s view=%s]", ud.vob, ud.label, ud.viewname)
+            bb.fetch2.check_network_access(d, cmd, ud.url)
+            output = runfetchcmd(cmd, d)
+            logger.info("rmview output: %s", output)
+
+    def need_update(self, ud, d):
+        if ("LATEST" in ud.label) or (ud.customspec and "LATEST" in ud.customspec):
+            ud.identifier += "-%s" % d.getVar("DATETIME",d, True)
+            return True
+        if os.path.exists(ud.localpath):
+            return False
+        return True
+
+    def supports_srcrev(self):
+        return True
+
+    def sortable_revision(self, ud, d, name):
+        return False, ud.identifier
+
+    def download(self, ud, d):
+        """Fetch url"""
+
+        # Make a fresh view
+        bb.utils.mkdirhier(ud.ccasedir)
+        self._write_configspec(ud, d)
+        cmd = self._build_ccase_command(ud, 'mkview')
+        logger.info("creating view [VOB=%s label=%s view=%s]", ud.vob, ud.label, ud.viewname)
+        bb.fetch2.check_network_access(d, cmd, ud.url)
+        try:
+            runfetchcmd(cmd, d)
+        except FetchError as e:
+            if "CRCLI2008E" in e.msg:
+                raise FetchError("%s\n%s\n" % (e.msg, "Call `rcleartool login` in your console to authenticate to the clearcase server before running bitbake."))
+            else:
+                raise e
+
+        # Set configspec: Setting the configspec effectively fetches the files as defined in the configspec
+        os.chdir(ud.viewdir)
+        cmd = self._build_ccase_command(ud, 'setcs');
+        logger.info("fetching data [VOB=%s label=%s view=%s]", ud.vob, ud.label, ud.viewname)
+        bb.fetch2.check_network_access(d, cmd, ud.url)
+        output = runfetchcmd(cmd, d)
+        logger.info("%s", output)
+
+        # Copy the configspec to the viewdir so we have it in our source tarball later
+        shutil.copyfile(ud.configspecfile, os.path.join(ud.viewdir, ud.csname))
+
+        # Clean clearcase meta-data before tar
+
+        runfetchcmd('tar -czf "%s" .' % (ud.localpath), d, cleanup = [ud.localpath])
+
+        # Clean up so we can create a new view next time
+        self.clean(ud, d);
+
+    def clean(self, ud, d):
+        self._remove_view(ud, d)
+        bb.utils.remove(ud.configspecfile)

bitbake/lib/bb/fetch2/git.py

@@ -123,7 +123,7 @@ class Git(FetchMethod):
             ud.branches[name] = branch
             ud.unresolvedrev[name] = branch
 
-        ud.basecmd = data.getVar("FETCHCMD_git", d, True) or "git"
+        ud.basecmd = data.getVar("FETCHCMD_git", d, True) or "git -c core.fsyncobjectfiles=0"
 
         ud.write_tarballs = ((data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True) or "0") != "0") or ud.rebaseable
 
@@ -275,7 +275,7 @@ class Git(FetchMethod):
             os.symlink(ud.clonedir, indirectiondir)
             clonedir = indirectiondir
 
-        runfetchcmd("git clone %s %s/ %s" % (cloneflags, clonedir, destdir), d)
+        runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, cloneflags, clonedir, destdir), d)
         if not ud.nocheckout:
             os.chdir(destdir)
             if subdir != "":

bitbake/lib/bb/fetch2/gitsm.py

@@ -2,6 +2,16 @@
 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 """
 BitBake 'Fetch' git submodules implementation
+
+Inherits from and extends the Git fetcher to retrieve submodules of a git repository
+after cloning.
+
+SRC_URI = "gitsm://<see Git fetcher for syntax>"
+
+See the Git fetcher, git://, for usage documentation.
+
+NOTE: Switching a SRC_URI from "git://" to "gitsm://" requires a clean of your recipe.
+
 """
 
 # Copyright (C) 2013 Richard Purdie

bitbake/lib/bb/fetch2/hg.py

@@ -110,7 +110,10 @@ class Hg(FetchMethod):
             options.append("-r %s" % ud.revision)
 
         if command == "fetch":
-            cmd = "%s clone %s %s://%s/%s %s" % (basecmd, " ".join(options), proto, hgroot, ud.module, ud.module)
+            if ud.user and ud.pswd:
+                cmd = "%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" clone %s %s://%s/%s %s" % (basecmd, ud.user, ud.pswd, proto, " ".join(options), proto, hgroot, ud.module, ud.module)
+            else:
+                cmd = "%s clone %s %s://%s/%s %s" % (basecmd, " ".join(options), proto, hgroot, ud.module, ud.module)	      
         elif command == "pull":
             # do not pass options list; limiting pull to rev causes the local
             # repo not to contain it and immediately following "update" command
@@ -120,7 +123,10 @@ class Hg(FetchMethod):
             else:
                 cmd = "%s pull" % (basecmd)
         elif command == "update":
-            cmd = "%s update -C %s" % (basecmd, " ".join(options))
+            if ud.user and ud.pswd:
+                cmd = "%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" update -C %s" % (basecmd, ud.user, ud.pswd, proto, " ".join(options))
+            else:
+                cmd = "%s update -C %s" % (basecmd, " ".join(options))
         else:
             raise FetchError("Invalid hg command %s" % command, ud.url)
 

bitbake/lib/bb/fetch2/local.py

@@ -56,7 +56,7 @@ class Local(FetchMethod):
         if path[0] != "/":
             filespath = data.getVar('FILESPATH', d, True)
             if filespath:
-                logger.debug(2, "Searching for %s in paths:    \n%s" % (path, "\n    ".join(filespath.split(":"))))
+                logger.debug(2, "Searching for %s in paths:\n    %s" % (path, "\n    ".join(filespath.split(":"))))
                 newpath = bb.utils.which(filespath, path)
             if not newpath:
                 filesdir = data.getVar('FILESDIR', d, True)

bitbake/lib/bb/fetch2/perforce.py

@@ -48,7 +48,7 @@ class Perforce(FetchMethod):
             (user, pswd, host, port) = path.split('@')[0].split(":")
             path = path.split('@')[1]
         else:
-            (host, port) = data.getVar('P4PORT', d).split(':')
+            (host, port) = d.getVar('P4PORT').split(':')
             user = ""
             pswd = ""
 
@@ -81,7 +81,7 @@ class Perforce(FetchMethod):
         if host:
             p4opt += " -p %s" % (host)
 
-        p4date = data.getVar("P4DATE", d, True)
+        p4date = d.getVar("P4DATE", True)
         if "revision" in parm:
             depot += "#%s" % (parm["revision"])
         elif "label" in parm:
@@ -89,7 +89,7 @@ class Perforce(FetchMethod):
         elif p4date:
             depot += "@%s" % (p4date)
 
-        p4cmd = data.getVar('FETCHCMD_p4', d, True)
+        p4cmd = d.getVar('FETCHCMD_p4', True) or "p4"
         logger.debug(1, "Running %s%s changes -m 1 %s", p4cmd, p4opt, depot)
         p4file, errors = bb.process.run("%s%s changes -m 1 %s" % (p4cmd, p4opt, depot))
         cset = p4file.strip()
@@ -103,22 +103,15 @@ class Perforce(FetchMethod):
     def urldata_init(self, ud, d):
         (host, path, user, pswd, parm) = Perforce.doparse(ud.url, d)
 
-        # If a label is specified, we use that as our filename
-
+        base_path = path.replace('/...', '')
+        base_path = self._strip_leading_slashes(base_path)
+        
         if "label" in parm:
-            ud.localfile = "%s.tar.gz" % (parm["label"])
-            return
+            version = parm["label"]
+        else:
+            version = Perforce.getcset(d, path, host, user, pswd, parm)
 
-        base = path
-        which = path.find('/...')
-        if which != -1:
-            base = path[:which-1]
-
-        base = self._strip_leading_slashes(base)
-
-        cset = Perforce.getcset(d, path, host, user, pswd, parm)
-
-        ud.localfile = data.expand('%s+%s+%s.tar.gz' % (host, base.replace('/', '.'), cset), d)
+        ud.localfile = data.expand('%s+%s+%s.tar.gz' % (host, base_path.replace('/', '.'), version), d)
 
     def download(self, ud, d):
         """
@@ -145,7 +138,7 @@ class Perforce(FetchMethod):
         if host:
             p4opt += " -p %s" % (host)
 
-        p4cmd = data.getVar('FETCHCMD_p4', d, True)
+        p4cmd = d.getVar('FETCHCMD_p4', True) or "p4"
 
         # create temp directory
         logger.debug(2, "Fetch: creating temporary directory")

bitbake/lib/bb/fetch2/svn.py

@@ -101,7 +101,8 @@ class Svn(FetchMethod):
                 suffix = "@%s" % (ud.revision)
 
             if command == "fetch":
-                svncmd = "%s co %s %s://%s/%s%s %s" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module, suffix, ud.module)
+                transportuser = ud.parm.get("transportuser", "")
+                svncmd = "%s co %s %s://%s%s/%s%s %s" % (ud.basecmd, " ".join(options), proto, transportuser, svnroot, ud.module, suffix, ud.module)
             elif command == "update":
                 svncmd = "%s update %s" % (ud.basecmd, " ".join(options))
             else:

bitbake/lib/bb/monitordisk.py

@@ -52,10 +52,10 @@ def getMountedDev(path):
     parentDev = os.stat(path).st_dev
     currentDev = parentDev
     # When the current directory's device is different from the
-    # parrent's, then the current directory is a mount point
+    # parent's, then the current directory is a mount point
     while parentDev == currentDev:
         mountPoint = path
-        # Use dirname to get the parrent's directory
+        # Use dirname to get the parent's directory
         path = os.path.dirname(path)
         # Reach the "/"
         if path == mountPoint:
@@ -77,7 +77,7 @@ def getDiskData(BBDirs, configuration):
     """Prepare disk data for disk space monitor"""
 
     # Save the device IDs, need the ID to be unique (the dictionary's key is
-    # unique), so that when more than one directories are located in the same
+    # unique), so that when more than one directory is located on the same
     # device, we just monitor it once
     devDict = {}
     for pathSpaceInode in BBDirs.split():
@@ -187,11 +187,11 @@ class diskMonitor:
                 if self.spaceInterval and self.inodeInterval:
                     self.enableMonitor = True
                     # These are for saving the previous disk free space and inode, we
-                    # use them to avoid print too many warning messages
+                    # use them to avoid printing too many warning messages
                     self.preFreeS = {}
                     self.preFreeI = {}
-                    # This is for STOPTASKS and ABORT, to avoid print the message repeatly
-                    # during waiting the tasks to finish
+                    # This is for STOPTASKS and ABORT, to avoid printing the message
+                    # repeatedly while waiting for the tasks to finish
                     self.checked = {}
                     for k in self.devDict:
                         self.preFreeS[k] = 0
@@ -239,11 +239,9 @@ class diskMonitor:
                 freeInode = st.f_favail
 
                 if minInode and freeInode < minInode:
-                    # Some fs formats' (e.g., btrfs) statvfs.f_files (inodes) is
-                    # zero, this is a feature of the fs, we disable the inode
-                    # checking for such a fs.
+                    # Some filesystems use dynamic inodes so can't run out
+                    # (e.g. btrfs). This is reported by the inode count being 0.
                     if st.f_files == 0:
-                        logger.info("Inode check for %s is unavaliable, will remove it from disk monitor" % path)
                         self.devDict[k][2] = None
                         continue
                     # Always show warning, the self.checked would always be False if the action is WARN

bitbake/lib/bb/namedtuple_with_abc.py

@@ -202,8 +202,8 @@ if __name__ == '__main__':
     print(rec5._replace(k=222)._my_custom_method())  # MyMixIn's
     print(rec5._replace(k=222).count(2))  # MyMixIn's
 
-    # None that behavior: the standard namedtuple methods cannot be
-    # overriden by a foreign mix-in -- even if the mix-in is declared
+    # Note that behavior: the standard namedtuple methods cannot be
+    # overridden by a foreign mix-in -- even if the mix-in is declared
     # as the leftmost base class (but, obviously, you can override them
     # in the defined class or its subclasses):
 

bitbake/lib/bb/parse/__init__.py

@@ -49,8 +49,11 @@ class ParseError(Exception):
         else:
             return "ParseError in %s: %s" % (self.filename, self.msg)
 
-class SkipPackage(Exception):
-    """Exception raised to skip this package"""
+class SkipRecipe(Exception):
+    """Exception raised to skip this recipe"""
+
+class SkipPackage(SkipRecipe):
+    """Exception raised to skip this recipe (use SkipRecipe in new code)"""
 
 __mtime_cache = {}
 def cached_mtime(f):
@@ -121,7 +124,6 @@ def resolve_file(fn, d):
     if not os.path.isfile(fn):
         raise IOError("file %s not found" % fn)
 
-    logger.debug(2, "LOAD %s", fn)
     return fn
 
 # Used by OpenEmbedded metadata

bitbake/lib/bb/parse/ast.py

@@ -337,8 +337,10 @@ def finalize(fn, d, variant = None):
 
     bb.event.fire(bb.event.RecipeParsed(fn), d)
 
-def _create_variants(datastores, names, function):
+def _create_variants(datastores, names, function, onlyfinalise):
     def create_variant(name, orig_d, arg = None):
+        if onlyfinalise and name not in onlyfinalise:
+            return
         new_d = bb.data.createCopy(orig_d)
         function(arg or name, new_d)
         datastores[name] = new_d
@@ -375,7 +377,7 @@ def _expand_versions(versions):
 def multi_finalize(fn, d):
     appends = (d.getVar("__BBAPPEND", True) or "").split()
     for append in appends:
-        logger.debug(2, "Appending .bbappend file %s to %s", append, fn)
+        logger.debug(1, "Appending .bbappend file %s to %s", append, fn)
         bb.parse.BBHandler.handle(append, d, True)
 
     onlyfinalise = d.getVar("__ONLYFINALISE", False)
@@ -384,7 +386,7 @@ def multi_finalize(fn, d):
     d = bb.data.createCopy(safe_d)
     try:
         finalize(fn, d)
-    except bb.parse.SkipPackage as e:
+    except bb.parse.SkipRecipe as e:
         d.setVar("__SKIPPED", e.args[0])
     datastores = {"": safe_d}
 
@@ -427,10 +429,10 @@ def multi_finalize(fn, d):
             verfunc(pv, d, safe_d)
             try:
                 finalize(fn, d)
-            except bb.parse.SkipPackage as e:
+            except bb.parse.SkipRecipe as e:
                 d.setVar("__SKIPPED", e.args[0])
 
-        _create_variants(datastores, versions, verfunc)
+        _create_variants(datastores, versions, verfunc, onlyfinalise)
 
     extended = d.getVar("BBCLASSEXTEND", True) or ""
     if extended:
@@ -460,14 +462,14 @@ def multi_finalize(fn, d):
             bb.parse.BBHandler.inherit(extendedmap[name], fn, 0, d)
 
         safe_d.setVar("BBCLASSEXTEND", extended)
-        _create_variants(datastores, extendedmap.keys(), extendfunc)
+        _create_variants(datastores, extendedmap.keys(), extendfunc, onlyfinalise)
 
     for variant, variant_d in datastores.iteritems():
         if variant:
             try:
                 if not onlyfinalise or variant in onlyfinalise:
                     finalize(fn, variant_d, variant)
-            except bb.parse.SkipPackage as e:
+            except bb.parse.SkipRecipe as e:
                 variant_d.setVar("__SKIPPED", e.args[0])
 
     if len(datastores) > 1:

bitbake/lib/bb/parse/parse_py/BBHandler.py

@@ -86,7 +86,7 @@ def inherit(files, fn, lineno, d):
                 file = abs_fn
 
         if not file in __inherit_cache:
-            logger.log(logging.DEBUG -1, "BB %s:%d: inheriting %s", fn, lineno, file)
+            logger.debug(1, "Inheriting %s (from %s:%d)" % (file, fn, lineno))
             __inherit_cache.append( file )
             d.setVar('__inherit_cache', __inherit_cache)
             include(fn, file, lineno, d, "inherit")
@@ -124,12 +124,6 @@ def handle(fn, d, include):
     __classname__ = ""
     __residue__ = []
 
-
-    if include == 0:
-        logger.debug(2, "BB %s: handle(data)", fn)
-    else:
-        logger.debug(2, "BB %s: handle(data, include)", fn)
-
     base_name = os.path.basename(fn)
     (root, ext) = os.path.splitext(base_name)
     init(d)
@@ -160,7 +154,7 @@ def handle(fn, d, include):
 
     try:
         statements.eval(d)
-    except bb.parse.SkipPackage:
+    except bb.parse.SkipRecipe:
         bb.data.setVar("__SKIPPED", True, d)
         if include == 0:
             return { "" : d }

bitbake/lib/bb/persist_data.py

@@ -199,7 +199,9 @@ class PersistData(object):
         del self.data[domain][key]
 
 def connect(database):
-    return sqlite3.connect(database, timeout=5, isolation_level=None)
+    connection = sqlite3.connect(database, timeout=5, isolation_level=None)
+    connection.execute("pragma synchronous = off;")
+    return connection
 
 def persist(domain, d):
     """Convenience factory for SQLTable objects based upon metadata"""

bitbake/lib/bb/pysh/sherrors.py

@@ -38,4 +38,4 @@ class ExitSignal(ShellError):
     
 class ReturnSignal(ShellError):
     """Exit signal."""
-    pass
+    pass

bitbake/lib/bb/runqueue.py

@@ -97,7 +97,7 @@ class RunQueueScheduler(object):
     def __init__(self, runqueue, rqdata):
         """
         The default scheduler just returns the first buildable task (the
-        priority map is sorted by task numer)
+        priority map is sorted by task number)
         """
         self.rq = runqueue
         self.rqdata = rqdata
@@ -139,7 +139,7 @@ class RunQueueScheduler(object):
         bestprio = None
         for taskid in self.buildable:
             prio = self.rev_prio_map[taskid]
-            if not bestprio or bestprio > prio:
+            if bestprio is None or bestprio > prio:
                 stamp = self.stamps[taskid]
                 if stamp in self.rq.build_stamps.itervalues():
                     continue
@@ -186,7 +186,7 @@ class RunQueueSchedulerCompletion(RunQueueSchedulerSpeed):
     """
     A scheduler optimised to complete .bb files are quickly as possible. The
     priority map is sorted by task weight, but then reordered so once a given
-    .bb file starts to build, its completed as quickly as possible. This works
+    .bb file starts to build, it's completed as quickly as possible. This works
     well where disk space is at a premium and classes like OE's rm_work are in
     force.
     """
@@ -370,11 +370,11 @@ class RunQueueData:
 
         for listid in xrange(numTasks):
             task_done.append(False)
-            weight.append(0)
+            weight.append(1)
             deps_left.append(len(self.runq_revdeps[listid]))
 
         for listid in endpoints:
-            weight[listid] = 1
+            weight[listid] = 10
             task_done[listid] = True
 
         while True:
@@ -795,7 +795,7 @@ class RunQueueData:
                 for st in self.cooker.configuration.invalidate_stamp.split(','):
                     invalidate_task(fn, "do_%s" % st, True)
 
-        # Interate over the task list and call into the siggen code
+        # Iterate over the task list and call into the siggen code
         dealtwith = set()
         todeal = set(range(len(self.runq_fnid)))
         while len(todeal) > 0:
@@ -859,15 +859,18 @@ class RunQueue:
 
     def _start_worker(self, fakeroot = False, rqexec = None):
         logger.debug(1, "Starting bitbake-worker")
+        magic = "decafbad"
+        if self.cooker.configuration.profile:
+            magic = "decafbadbad"
         if fakeroot:
             fakerootcmd = self.cfgData.getVar("FAKEROOTCMD", True)
             fakerootenv = (self.cfgData.getVar("FAKEROOTBASEENV", True) or "").split()
             env = os.environ.copy()
             for key, value in (var.split('=') for var in fakerootenv):
                 env[key] = value
-            worker = subprocess.Popen([fakerootcmd, "bitbake-worker", "decafbad"], stdout=subprocess.PIPE, stdin=subprocess.PIPE, env=env)
+            worker = subprocess.Popen([fakerootcmd, "bitbake-worker", magic], stdout=subprocess.PIPE, stdin=subprocess.PIPE, env=env)
         else:
-            worker = subprocess.Popen(["bitbake-worker", "decafbad"], stdout=subprocess.PIPE, stdin=subprocess.PIPE)
+            worker = subprocess.Popen(["bitbake-worker", magic], stdout=subprocess.PIPE, stdin=subprocess.PIPE)
         bb.utils.nonblockingfd(worker.stdout)
         workerpipe = runQueuePipe(worker.stdout, None, self.cfgData, self, rqexec)
 
@@ -876,9 +879,7 @@ class RunQueue:
             "fakerootenv" : self.rqdata.dataCache.fakerootenv,
             "fakerootdirs" : self.rqdata.dataCache.fakerootdirs,
             "fakerootnoenv" : self.rqdata.dataCache.fakerootnoenv,
-            "hashes" : bb.parse.siggen.taskhash,
-            "hash_deps" : bb.parse.siggen.runtaskdeps,
-            "sigchecksums" : bb.parse.siggen.file_checksum_values,
+            "sigdata" : bb.parse.siggen.get_taskdata(),
             "runq_hash" : self.rqdata.runq_hash,
             "logdefaultdebug" : bb.msg.loggerDefaultDebugLevel,
             "logdefaultverbose" : bb.msg.loggerDefaultVerbose,
@@ -967,11 +968,11 @@ class RunQueue:
 
         stampfile = bb.build.stampfile(taskname, self.rqdata.dataCache, fn)
 
-        # If the stamp is missing its not current
+        # If the stamp is missing, it's not current
         if not os.access(stampfile, os.F_OK):
             logger.debug(2, "Stampfile %s not available", stampfile)
             return False
-        # If its a 'nostamp' task, it's not current
+        # If it's a 'nostamp' task, it's not current
         taskdep = self.rqdata.dataCache.task_deps[fn]
         if 'nostamp' in taskdep and taskname in taskdep['nostamp']:
             logger.debug(2, "%s.%s is nostamp\n", fn, taskname)
@@ -1065,7 +1066,7 @@ class RunQueue:
         if self.state is runQueueCleanUp:
            self.rqexe.finish()
 
-        if self.state is runQueueComplete or self.state is runQueueFailed:
+        if (self.state is runQueueComplete or self.state is runQueueFailed) and self.rqexe:
             self.teardown_workers()
             if self.rqexe.stats.failed:
                 logger.info("Tasks Summary: Attempted %d tasks of which %d didn't need to be rerun and %d failed.", self.rqexe.stats.completed + self.rqexe.stats.failed, self.rqexe.stats.skipped, self.rqexe.stats.failed)
@@ -1106,6 +1107,7 @@ class RunQueue:
 
     def finish_runqueue(self, now = False):
         if not self.rqexe:
+            self.state = runQueueComplete
             return
 
         if now:
@@ -1551,7 +1553,8 @@ class RunQueueExecuteTasks(RunQueueExecute):
                 bb.event.fire(startevent, self.cfgData)
                 self.runq_running[task] = 1
                 self.stats.taskActive()
-                bb.build.make_stamp(taskname, self.rqdata.dataCache, fn)
+                if not self.cooker.configuration.dry_run:
+                    bb.build.make_stamp(taskname, self.rqdata.dataCache, fn)
                 self.task_complete(task)
                 return True
             else:
@@ -1686,7 +1689,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
 
         process_endpoints(endpoints)
 
-        # Build a list of setscene tasks which as "unskippable"
+        # Build a list of setscene tasks which are "unskippable"
         # These are direct endpoints referenced by the build
         endpoints2 = {}
         sq_revdeps2 = []
@@ -1842,6 +1845,10 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
                 realtask = self.rqdata.runq_setscene[task]
                 realdep = self.rqdata.runq_setscene[dep]
                 logger.debug(2, "%s was unavailable and is a hard dependency of %s so skipping" % (self.rqdata.get_user_idstring(realtask), self.rqdata.get_user_idstring(realdep)))
+                self.scenequeue_updatecounters(dep, fail)
+                continue
+            if task not in self.sq_revdeps2[dep]:
+                # May already have been removed by the fail case above
                 continue
             self.sq_revdeps2[dep].remove(task)
             if len(self.sq_revdeps2[dep]) == 0:
@@ -1983,6 +1990,10 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
         logger.debug(1, 'We can skip tasks %s', sorted(self.rq.scenequeue_covered))
 
         self.rq.state = runQueueRunInit
+
+        completeevent = sceneQueueComplete(self.stats, self.rq)
+        bb.event.fire(completeevent, self.cfgData)
+
         return True
 
     def runqueue_process_waitpid(self, task, status):
@@ -2035,7 +2046,7 @@ class sceneQueueEvent(runQueueEvent):
 
 class runQueueTaskStarted(runQueueEvent):
     """
-    Event notifing a task was started
+    Event notifying a task was started
     """
     def __init__(self, task, stats, rq, noexec=False):
         runQueueEvent.__init__(self, task, stats, rq)
@@ -2043,7 +2054,7 @@ class runQueueTaskStarted(runQueueEvent):
 
 class sceneQueueTaskStarted(sceneQueueEvent):
     """
-    Event notifing a setscene task was started
+    Event notifying a setscene task was started
     """
     def __init__(self, task, stats, rq, noexec=False):
         sceneQueueEvent.__init__(self, task, stats, rq)
@@ -2051,7 +2062,7 @@ class sceneQueueTaskStarted(sceneQueueEvent):
 
 class runQueueTaskFailed(runQueueEvent):
     """
-    Event notifing a task failed
+    Event notifying a task failed
     """
     def __init__(self, task, stats, exitcode, rq):
         runQueueEvent.__init__(self, task, stats, rq)
@@ -2059,25 +2070,33 @@ class runQueueTaskFailed(runQueueEvent):
 
 class sceneQueueTaskFailed(sceneQueueEvent):
     """
-    Event notifing a setscene task failed
+    Event notifying a setscene task failed
     """
     def __init__(self, task, stats, exitcode, rq):
         sceneQueueEvent.__init__(self, task, stats, rq)
         self.exitcode = exitcode
 
+class sceneQueueComplete(sceneQueueEvent):
+    """
+    Event when all the sceneQueue tasks are complete
+    """
+    def __init__(self, stats, rq):
+        self.stats = stats.copy()
+        bb.event.Event.__init__(self)
+
 class runQueueTaskCompleted(runQueueEvent):
     """
-    Event notifing a task completed
+    Event notifying a task completed
     """
 
 class sceneQueueTaskCompleted(sceneQueueEvent):
     """
-    Event notifing a setscene task completed
+    Event notifying a setscene task completed
     """
 
 class runQueueTaskSkipped(runQueueEvent):
     """
-    Event notifing a task was skipped
+    Event notifying a task was skipped
     """
     def __init__(self, task, stats, rq, reason):
         runQueueEvent.__init__(self, task, stats, rq)

bitbake/lib/bb/server/process.py

@@ -38,14 +38,18 @@ from . import BitBakeBaseServer, BitBakeBaseServerConnection, BaseImplServer
 logger = logging.getLogger('BitBake')
 
 class ServerCommunicator():
-    def __init__(self, connection, event_handle):
+    def __init__(self, connection, event_handle, server):
         self.connection = connection
         self.event_handle = event_handle
+        self.server = server
 
     def runCommand(self, command):
         # @todo try/except
         self.connection.send(command)
 
+        if not self.server.is_alive():
+            raise SystemExit
+
         while True:
             # don't let the user ctrl-c while we're waiting for a response
             try:
@@ -139,6 +143,8 @@ class ProcessServer(Process, BaseImplServer):
                 raise
             except Exception:
                 logger.exception('Running idle function')
+                del self._idlefuns[function]
+                self.quit = True
 
         if nextsleep is not None:
             select.select(fds,[],[],nextsleep)
@@ -158,7 +164,7 @@ class BitBakeProcessServerConnection(BitBakeBaseServerConnection):
         self.procserver = serverImpl
         self.ui_channel = ui_channel
         self.event_queue = event_queue
-        self.connection = ServerCommunicator(self.ui_channel, self.procserver.event_handle)
+        self.connection = ServerCommunicator(self.ui_channel, self.procserver.event_handle, self.procserver)
         self.events = self.event_queue
 
     def sigterm_terminate(self):
@@ -197,14 +203,20 @@ class ProcessEventQueue(multiprocessing.queues.Queue):
 
     def waitEvent(self, timeout):
         if self.exit:
-            raise KeyboardInterrupt()
+            sys.exit(1)
         try:
+            if not self.server.is_alive():
+                self.setexit()
+                return None
             return self.get(True, timeout)
         except Empty:
             return None
 
     def getEvent(self):
         try:
+            if not self.server.is_alive():
+                self.setexit()
+                return None
             return self.get(False)
         except Empty:
             return None
@@ -219,6 +231,7 @@ class BitBakeServer(BitBakeBaseServer):
         self.ui_channel, self.server_channel = Pipe()
         self.event_queue = ProcessEventQueue(0)
         self.serverImpl = ProcessServer(self.server_channel, self.event_queue, None)
+        self.event_queue.server = self.serverImpl
 
     def detach(self):
         self.serverImpl.start()

bitbake/lib/bb/server/xmlrpc.py

@@ -80,7 +80,7 @@ class BBTransport(xmlrpclib.Transport):
 
 def _create_server(host, port, timeout = 60):
     t = BBTransport(timeout)
-    s = xmlrpclib.Server("http://%s:%d/" % (host, port), transport=t, allow_none=True)
+    s = xmlrpclib.ServerProxy("http://%s:%d/" % (host, port), transport=t, allow_none=True)
     return s, t
 
 class BitBakeServerCommands():
@@ -253,9 +253,13 @@ class XMLRPCServer(SimpleXMLRPCServer, BaseImplServer):
             socktimeout = self.socket.gettimeout() or nextsleep
             socktimeout = min(socktimeout, nextsleep)
             # Mirror what BaseServer handle_request would do
-            fd_sets = select.select(fds, [], [], socktimeout)
-            if fd_sets[0] and self in fd_sets[0]:
-                self._handle_request_noblock()
+            try:
+                fd_sets = select.select(fds, [], [], socktimeout)
+                if fd_sets[0] and self in fd_sets[0]:
+                    self._handle_request_noblock()
+            except IOError:
+                # we ignore interrupted calls
+                pass
 
         # Tell idle functions we're exiting
         for function, data in self._idlefuns.items():
@@ -277,13 +281,16 @@ class BitBakeXMLRPCServerConnection(BitBakeBaseServerConnection):
         self.observer_only = observer_only
         self.featureset = featureset
 
-    def connect(self):
-        if not self.observer_only:
-            token = self.connection.addClient()
-        else:
-            token = "observer"
+    def connect(self, token = None):
+        if token is None:
+            if self.observer_only:
+                token = "observer"
+            else:
+                token = self.connection.addClient()
+
         if token is None:
             return None
+
         self.transport.set_connection_token(token)
 
         self.events = uievent.BBUIEventQueue(self.connection, self.clientinfo)
@@ -332,7 +339,9 @@ class BitBakeServer(BitBakeBaseServer):
 
 class BitBakeXMLRPCClient(BitBakeBaseServer):
 
-    def __init__(self, observer_only = False):
+    def __init__(self, observer_only = False, token = None):
+        self.token = token
+
         self.observer_only = observer_only
         # if we need extra caches, just tell the server to load them all
         pass
@@ -340,37 +349,14 @@ class BitBakeXMLRPCClient(BitBakeBaseServer):
     def saveConnectionDetails(self, remote):
         self.remote = remote
 
-    def saveConnectionConfigParams(self, configParams):
-        self.configParams = configParams
-
     def establishConnection(self, featureset):
         # The format of "remote" must be "server:port"
         try:
             [host, port] = self.remote.split(":")
             port = int(port)
         except Exception as e:
-            bb.fatal("Failed to read remote definition (%s)" % str(e))
-
-        # use automatic port if port set to -1, meaning read it from
-        # the bitbake.lock file
-        if port == -1:
-            lock_location = "%s/bitbake.lock" % self.configParams.environment.get('BUILDDIR')
-            lock = bb.utils.lockfile(lock_location, False, False)
-            if lock:
-                # This means there is no server running which we can
-                # connect to on the local system.
-                bb.utils.unlockfile(lock)
-                return None
-
-            try:
-                lf = open(lock_location, 'r')
-                remotedef = lf.readline()
-                [host, port] = remotedef.split(":")
-                port = int(port)
-                lf.close()
-                self.remote = remotedef
-            except Exception as e:
-                bb.fatal("Failed to read bitbake.lock (%s)" % str(e))
+            bb.warn("Failed to read remote definition (%s)" % str(e))
+            raise e
 
         # We need our IP for the server connection. We get the IP
         # by trying to connect with the server
@@ -380,13 +366,15 @@ class BitBakeXMLRPCClient(BitBakeBaseServer):
             ip = s.getsockname()[0]
             s.close()
         except Exception as e:
-            bb.fatal("Could not create socket for %s:%s (%s)" % (host, port, str(e)))
+            bb.warn("Could not create socket for %s:%s (%s)" % (host, port, str(e)))
+            raise e
         try:
             self.serverImpl = XMLRPCProxyServer(host, port)
             self.connection = BitBakeXMLRPCServerConnection(self.serverImpl, (ip, 0), self.observer_only, featureset)
-            return self.connection.connect()
+            return self.connection.connect(self.token)
         except Exception as e:
-            bb.fatal("Could not connect to server at %s:%s (%s)" % (host, port, str(e)))
+            bb.warn("Could not connect to server at %s:%s (%s)" % (host, port, str(e)))
+            raise e
 
     def endSession(self):
         self.connection.removeClient()

bitbake/lib/bb/siggen.py

@@ -190,16 +190,18 @@ class SignatureGeneratorBasic(SignatureGenerator):
         taint = self.read_taint(fn, task, dataCache.stamp[fn])
         if taint:
             data = data + taint
+            logger.warn("%s is tainted from a forced run" % k)
 
         h = hashlib.md5(data).hexdigest()
         self.taskhash[k] = h
         #d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task])
         return h
 
-    def set_taskdata(self, hashes, deps, checksums):
-        self.runtaskdeps = deps
-        self.taskhash = hashes
-        self.file_checksum_values = checksums
+    def get_taskdata(self):
+       return (self.runtaskdeps, self.taskhash, self.file_checksum_values)
+
+    def set_taskdata(self, data):
+        self.runtaskdeps, self.taskhash, self.file_checksum_values = data
 
     def dump_sigtask(self, fn, task, stampbase, runtime):
         k = fn + "." + task
@@ -294,7 +296,7 @@ def dump_this_task(outfile, d):
 
 def clean_basepath(a):
     if a.startswith("virtual:"):
-        b = a.rsplit(":", 1)[0] + ":" + a.rsplit("/", 1)[1]
+        b = a.rsplit("/", 1)[1] + ":" + a.rsplit(":", 1)[0]
     else:
         b = a.rsplit("/", 1)[1]
     return b

bitbake/lib/bb/tests/data.py

@@ -253,6 +253,18 @@ class TestConcatOverride(unittest.TestCase):
         bb.data.update_data(self.d)
         self.assertEqual(self.d.getVar("TEST_TEST", True), "bar bar")
 
+    def test_empty_remove(self):
+        self.d.setVar("TEST", "")
+        self.d.setVar("TEST_remove", "val")
+        bb.data.update_data(self.d)
+        self.assertEqual(self.d.getVar("TEST", True), "")
+
+    def test_remove_expansion(self):
+        self.d.setVar("BAR", "Z")
+        self.d.setVar("TEST", "${BAR}/X Y")
+        self.d.setVar("TEST_remove", "${BAR}/X")
+        bb.data.update_data(self.d)
+        self.assertEqual(self.d.getVar("TEST", True), "Y")
 
 class TestOverrides(unittest.TestCase):
     def setUp(self):
@@ -294,3 +306,39 @@ class TestFlags(unittest.TestCase):
         self.assertEqual(self.d.getVarFlag("foo", "flag2"), None)
 
 
+class Contains(unittest.TestCase):
+    def setUp(self):
+        self.d = bb.data.init()
+        self.d.setVar("SOMEFLAG", "a b c")
+
+    def test_contains(self):
+        self.assertTrue(bb.utils.contains("SOMEFLAG", "a", True, False, self.d))
+        self.assertTrue(bb.utils.contains("SOMEFLAG", "b", True, False, self.d))
+        self.assertTrue(bb.utils.contains("SOMEFLAG", "c", True, False, self.d))
+
+        self.assertTrue(bb.utils.contains("SOMEFLAG", "a b", True, False, self.d))
+        self.assertTrue(bb.utils.contains("SOMEFLAG", "b c", True, False, self.d))
+        self.assertTrue(bb.utils.contains("SOMEFLAG", "c a", True, False, self.d))
+
+        self.assertTrue(bb.utils.contains("SOMEFLAG", "a b c", True, False, self.d))
+        self.assertTrue(bb.utils.contains("SOMEFLAG", "c b a", True, False, self.d))
+
+        self.assertFalse(bb.utils.contains("SOMEFLAG", "x", True, False, self.d))
+        self.assertFalse(bb.utils.contains("SOMEFLAG", "a x", True, False, self.d))
+        self.assertFalse(bb.utils.contains("SOMEFLAG", "x c b", True, False, self.d))
+        self.assertFalse(bb.utils.contains("SOMEFLAG", "x c b a", True, False, self.d))
+
+    def test_contains_any(self):
+        self.assertTrue(bb.utils.contains_any("SOMEFLAG", "a", True, False, self.d))
+        self.assertTrue(bb.utils.contains_any("SOMEFLAG", "b", True, False, self.d))
+        self.assertTrue(bb.utils.contains_any("SOMEFLAG", "c", True, False, self.d))
+
+        self.assertTrue(bb.utils.contains_any("SOMEFLAG", "a b", True, False, self.d))
+        self.assertTrue(bb.utils.contains_any("SOMEFLAG", "b c", True, False, self.d))
+        self.assertTrue(bb.utils.contains_any("SOMEFLAG", "c a", True, False, self.d))
+
+        self.assertTrue(bb.utils.contains_any("SOMEFLAG", "a x", True, False, self.d))
+        self.assertTrue(bb.utils.contains_any("SOMEFLAG", "x c", True, False, self.d))
+
+        self.assertFalse(bb.utils.contains_any("SOMEFLAG", "x", True, False, self.d))
+        self.assertFalse(bb.utils.contains_any("SOMEFLAG", "x y z", True, False, self.d))

bitbake/lib/bb/tests/fetch.py

@@ -444,6 +444,13 @@ class FetcherLocalTest(FetcherTest):
         tree = self.fetchUnpack(['file://dir/subdir/e'])
         self.assertEqual(tree, ['dir/subdir/e'])
 
+    def test_local_subdirparam(self):
+        tree = self.fetchUnpack(['file://a;subdir=bar'])
+        self.assertEqual(tree, ['bar/a'])
+
+    def test_local_deepsubdirparam(self):
+        tree = self.fetchUnpack(['file://dir/subdir/e;subdir=bar'])
+        self.assertEqual(tree, ['bar/dir/subdir/e'])
 
 class FetcherNetworkTest(FetcherTest):
 

bitbake/lib/bb/tinfoil.py

@@ -25,12 +25,12 @@ import bb.cache
 import bb.cooker
 import bb.providers
 import bb.utils
-from bb.cooker import state, BBCooker
+from bb.cooker import state, BBCooker, CookerFeatures
 from bb.cookerdata import CookerConfiguration, ConfigParameters
 import bb.fetch2
 
 class Tinfoil:
-    def __init__(self, output=sys.stdout):
+    def __init__(self, output=sys.stdout, tracking=False):
         # Needed to avoid deprecation warnings with python 2.6
         warnings.filterwarnings("ignore", category=DeprecationWarning)
 
@@ -48,7 +48,10 @@ class Tinfoil:
         configparams = TinfoilConfigParameters(parse_only=True)
         self.config.setConfigParameters(configparams)
         self.config.setServerRegIdleCallback(self.register_idle_function)
-        self.cooker = BBCooker(self.config)
+        features = []
+        if tracking:
+            features.append(CookerFeatures.BASEDATASTORE_TRACKING)
+        self.cooker = BBCooker(self.config, features)
         self.config_data = self.cooker.data
         bb.providers.logger.setLevel(logging.ERROR)
         self.cooker_data = None

bitbake/lib/bb/ui/buildinfohelper.py

@@ -22,7 +22,7 @@ import bb
 import re
 import ast
 
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "toaster.toastermain.settings")
+os.environ["DJANGO_SETTINGS_MODULE"] = "toaster.toastermain.settings"
 
 import toaster.toastermain.settings as toaster_django_settings
 from toaster.orm.models import Build, Task, Recipe, Layer_Version, Layer, Target, LogMessage, HelpText
@@ -46,7 +46,7 @@ class ORMWrapper(object):
         pass
 
 
-    def create_build_object(self, build_info):
+    def create_build_object(self, build_info, brbe):
         assert 'machine' in build_info
         assert 'distro' in build_info
         assert 'distro_version' in build_info
@@ -65,6 +65,13 @@ class ORMWrapper(object):
                                     build_name=build_info['build_name'],
                                     bitbake_version=build_info['bitbake_version'])
 
+        if brbe is not None:
+            from bldcontrol.models import BuildEnvironment, BuildRequest
+            br, be = brbe.split(":")
+            buildrequest = BuildRequest.objects.get(pk = br)
+            build.project_id = buildrequest.project_id
+            build.save()
+
         return build
 
     def create_target_objects(self, target_info):
@@ -473,6 +480,8 @@ class ORMWrapper(object):
                                 line_number = vh['line'],
                                 operation = vh['op'])
 
+class MockEvent: pass           # sometimes we mock an event, declare it here
+
 class BuildInfoHelper(object):
     """ This class gathers the build information from the server and sends it
         towards the ORM wrapper for storing in the database
@@ -480,6 +489,7 @@ class BuildInfoHelper(object):
         Keeps in memory all data that needs matching before writing it to the database
     """
 
+
     def __init__(self, server, has_build_history = False):
         self._configure_django()
         self.internal_state = {}
@@ -489,6 +499,8 @@ class BuildInfoHelper(object):
         self.orm_wrapper = ORMWrapper()
         self.has_build_history = has_build_history
         self.tmp_dir = self.server.runCommand(["getVariable", "TMPDIR"])[0]
+        self.brbe    = self.server.runCommand(["getVariable", "TOASTER_BRBE"])[0]
+
 
     def _configure_django(self):
         # Add toaster to sys path for importing modules
@@ -589,8 +601,8 @@ class BuildInfoHelper(object):
     ## external available methods to store information
 
     def store_layer_info(self, event):
-        assert 'data' in vars(event)
-        layerinfos = event.data
+        assert '_localdata' in vars(event)
+        layerinfos = event._localdata
         self.internal_state['lvs'] = {}
         for layer in layerinfos:
             self.internal_state['lvs'][self.orm_wrapper.get_update_layer_object(layerinfos[layer])] = layerinfos[layer]['version']
@@ -600,7 +612,8 @@ class BuildInfoHelper(object):
         assert '_pkgs' in vars(event)
         build_information = self._get_build_information()
 
-        build_obj = self.orm_wrapper.create_build_object(build_information)
+        build_obj = self.orm_wrapper.create_build_object(build_information, self.brbe)
+
         self.internal_state['build'] = build_obj
 
         # save layer version information for this build
@@ -619,14 +632,18 @@ class BuildInfoHelper(object):
         # Save build configuration
         self.orm_wrapper.save_build_variables(build_obj, self.server.runCommand(["getAllKeysWithFlags", ["doc", "func"]])[0])
 
+        return self.brbe
+
+
+
     def update_target_image_file(self, event):
         image_fstypes = self.server.runCommand(["getVariable", "IMAGE_FSTYPES"])[0]
         for t in self.internal_state['targets']:
             if t.is_image == True:
-                output_files = list(event.data.viewkeys())
+                output_files = list(event._localdata.viewkeys())
                 for output in output_files:
                     if t.target in output and output.split('.rootfs.')[1] in image_fstypes:
-                        self.orm_wrapper.save_target_image_file_information(t, output, event.data[output])
+                        self.orm_wrapper.save_target_image_file_information(t, output, event._localdata[output])
 
     def update_build_information(self, event, errors, warnings, taskfailures):
         if 'build' in self.internal_state:
@@ -634,8 +651,8 @@ class BuildInfoHelper(object):
 
 
     def store_license_manifest_path(self, event):
-        deploy_dir = event.data['deploy_dir']
-        image_name =  event.data['image_name']
+        deploy_dir = event._localdata['deploy_dir']
+        image_name =  event._localdata['image_name']
         path = deploy_dir + "/licenses/" + image_name + "/"
         for target in self.internal_state['targets']:
             if target.target in image_name:
@@ -683,7 +700,7 @@ class BuildInfoHelper(object):
 
 
     def store_tasks_stats(self, event):
-        for (taskfile, taskname, taskstats, recipename) in event.data:
+        for (taskfile, taskname, taskstats, recipename) in event._localdata:
             localfilepath = taskfile.split(":")[-1]
             assert localfilepath.startswith("/")
 
@@ -755,12 +772,11 @@ class BuildInfoHelper(object):
 
 
     def store_missed_state_tasks(self, event):
-        for (fn, taskname, taskhash, sstatefile) in event.data['missed']:
+        for (fn, taskname, taskhash, sstatefile) in event._localdata['missed']:
 
             identifier = fn + taskname + "_setscene"
             recipe_information = self._get_recipe_information_from_taskfile(fn)
             recipe = self.orm_wrapper.get_update_recipe_object(recipe_information)
-            class MockEvent: pass
             mevent = MockEvent()
             mevent.taskname = taskname
             mevent.taskhash = taskhash
@@ -774,12 +790,11 @@ class BuildInfoHelper(object):
 
             self.orm_wrapper.get_update_task_object(task_information)
 
-        for (fn, taskname, taskhash, sstatefile) in event.data['found']:
+        for (fn, taskname, taskhash, sstatefile) in event._localdata['found']:
 
             identifier = fn + taskname + "_setscene"
             recipe_information = self._get_recipe_information_from_taskfile(fn)
             recipe = self.orm_wrapper.get_update_recipe_object(recipe_information)
-            class MockEvent: pass
             mevent = MockEvent()
             mevent.taskname = taskname
             mevent.taskhash = taskhash
@@ -791,15 +806,15 @@ class BuildInfoHelper(object):
 
 
     def store_target_package_data(self, event):
-        assert 'data' in vars(event)
+        assert '_localdata' in vars(event)
         # for all image targets
         for target in self.internal_state['targets']:
             if target.is_image:
                 try:
-                    pkgdata = event.data['pkgdata']
-                    imgdata = event.data['imgdata'][target.target]
+                    pkgdata = event._localdata['pkgdata']
+                    imgdata = event._localdata['imgdata'][target.target]
                     self.orm_wrapper.save_target_package_information(self.internal_state['build'], target, imgdata, pkgdata, self.internal_state['recipes'])
-                    filedata = event.data['filedata'][target.target]
+                    filedata = event._localdata['filedata'][target.target]
                     self.orm_wrapper.save_target_file_information(self.internal_state['build'], target, filedata)
                 except KeyError:
                     # we must have not got the data for this image, nothing to save
@@ -909,56 +924,79 @@ class BuildInfoHelper(object):
 
 
     def store_build_package_information(self, event):
-        assert 'data' in vars(event)
-        package_info = event.data
+        assert '_localdata' in vars(event)
+        package_info = event._localdata
         self.orm_wrapper.save_build_package_information(self.internal_state['build'],
                             package_info,
                             self.internal_state['recipes'],
                             )
 
-    def _store_log_information(self, level, text):
-        log_information = {}
-        log_information['build'] = self.internal_state['build']
-        log_information['level'] = level
-        log_information['message'] = text
-        self.orm_wrapper.create_logmessage(log_information)
+    def _store_build_done(self):
+        br_id, be_id = self.brbe.split(":")
+        from bldcontrol.models import BuildEnvironment, BuildRequest
+        be = BuildEnvironment.objects.get(pk = be_id)
+        be.lock = BuildEnvironment.LOCK_LOCK
+        be.save()
+        br = BuildRequest.objects.get(pk = br_id)
+        br.state = BuildRequest.REQ_COMPLETED
+        br.build = self.internal_state['build']
+        br.save()
 
-    def store_log_info(self, text):
-        self._store_log_information(LogMessage.INFO, text)
-
-    def store_log_warn(self, text):
-        self._store_log_information(LogMessage.WARNING, text)
 
     def store_log_error(self, text):
-        self._store_log_information(LogMessage.ERROR, text)
+        mockevent = MockEvent()
+        mockevent.levelno = format.ERROR
+        mockevent.msg = text
+        self.store_log_event(mockevent)
 
     def store_log_event(self, event):
+        if event.levelno < format.WARNING:
+            return
+
+        if 'args' in vars(event):
+            event.msg = event.msg % event.args
+
+        if not 'build' in self.internal_state:
+            if self.brbe is None:
+                if not 'backlog' in self.internal_state:
+                    self.internal_state['backlog'] = []
+                self.internal_state['backlog'].append(event)
+            else:   # we're under Toaster control, post the errors to the build request
+                from bldcontrol.models import BuildRequest, BRError
+                br, be = brbe.split(":")
+                buildrequest = BuildRequest.objects.get(pk = br)
+                brerror = BRError.objects.create(req = buildrequest, errtype="build", errmsg = event.msg)
+
+            return
+
         if 'build' in self.internal_state and 'backlog' in self.internal_state:
             if len(self.internal_state['backlog']):
                 tempevent = self.internal_state['backlog'].pop()
-                print "Saving stored event ", tempevent
+                print "    Saving stored event ", tempevent
                 self.store_log_event(tempevent)
             else:
                 del self.internal_state['backlog']
 
-        if event.levelno < format.WARNING:
-            return
-
-        if not 'build' in self.internal_state:
-            print "Save event for later"
-            if not 'backlog' in self.internal_state:
-                self.internal_state['backlog'] = []
-            self.internal_state['backlog'].append(event)
-
-            return
         log_information = {}
         log_information['build'] = self.internal_state['build']
         if event.levelno >= format.ERROR:
-            log_information['level'] = LogMessage.ERROR
+            log_information['level'] = event.levelno
         elif event.levelno == format.WARNING:
             log_information['level'] = LogMessage.WARNING
+        elif event.levelno == format.INFO:
+            log_information['level'] = LogMessage.INFO
+        else:
+            log_information['level'] = event.levelno
+
         log_information['message'] = event.msg
         log_information['pathname'] = event.pathname
         log_information['lineno'] = event.lineno
         self.orm_wrapper.create_logmessage(log_information)
 
+    def close(self):
+        if self.brbe is not None:
+            buildinfohelper._store_build_done()
+
+        if 'backlog' in self.internal_state:
+            for event in self.internal_state['backlog']:
+                   print "NOTE: Unsaved log: ", event.msg

bitbake/lib/bb/ui/crumbs/hig/simplesettingsdialog.py

@@ -230,10 +230,7 @@ class SimpleSettingsDialog (CrumbsDialog, SettingsUIHelper):
         self.configuration.sstatemirror = ""
         for mirror in self.sstatemirrors_list:
             if mirror[1] != "" and mirror[2].startswith("file://"):
-                if mirror[1].endswith("\\1"):
-                    smirror = mirror[2] + " " + mirror[1] + " \\n "
-                else:
-                    smirror = mirror[2] + " " + mirror[1] + "\\1 \\n "
+                smirror = mirror[2] + " " + mirror[1] + " \\n "
                 self.configuration.sstatemirror += smirror
         self.configuration.bbthread = self.bb_spinner.get_value_as_int()
         self.configuration.pmake = self.pmake_spinner.get_value_as_int()

bitbake/lib/bb/ui/depexp.py

@@ -198,7 +198,7 @@ def main(server, eventHandler, params):
             print("Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.")
             return 1
         if 'msg' in cmdline and cmdline['msg']:
-            logger.error(cmdline['msg'])
+            print(cmdline['msg'])
             return 1
         cmdline = cmdline['action']
         if not cmdline or cmdline[0] != "generateDotGraph":

bitbake/lib/bb/ui/knotty.py

@@ -284,6 +284,7 @@ def main(server, eventHandler, params, tf = TerminalFilter):
 
     if not params.observe_only:
         params.updateFromServer(server)
+        params.updateToServer(server)
         cmdline = params.parseActions()
         if not cmdline:
             print("Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.")
@@ -321,8 +322,8 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                     break
                 termfilter.updateFooter()
                 event = eventHandler.waitEvent(0.25)
-            if event is None:
-                continue
+                if event is None:
+                    continue
             helper.eventHandler(event)
             if isinstance(event, bb.runqueue.runQueueExitWait):
                 if not main.shutdown:
@@ -351,7 +352,7 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                 # For "normal" logging conditions, don't show note logs from tasks
                 # but do show them if the user has changed the default log level to
                 # include verbose/debug messages
-                if event.taskpid != 0 and event.levelno <= format.NOTE:
+                if event.taskpid != 0 and event.levelno <= format.NOTE and (event.levelno < llevel or (event.levelno == format.NOTE and llevel != format.VERBOSE)):
                     continue
                 logger.handle(event)
                 continue
@@ -507,7 +508,11 @@ def main(server, eventHandler, params, tf = TerminalFilter):
             termfilter.clearFooter()
             # ignore interrupted io
             if ioerror.args[0] == 4:
-                pass
+                continue
+            sys.stderr.write(str(ioerror))
+            if not params.observe_only:
+                _, error = server.runCommand(["stateForceShutdown"])
+            main.shutdown = 2
         except KeyboardInterrupt:
             termfilter.clearFooter()
             if params.observe_only:
@@ -526,7 +531,11 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                     logger.error("Unable to cleanly shutdown: %s" % error)
             main.shutdown = main.shutdown + 1
             pass
-
+        except Exception as e:
+            sys.stderr.write(str(e))
+            if not params.observe_only:
+                _, error = server.runCommand(["stateForceShutdown"])
+            main.shutdown = 2
     summary = ""
     if taskfailures:
         summary += pluralise("\nSummary: %s task failed:",

bitbake/lib/bb/ui/toasterui.py

@@ -91,13 +91,16 @@ def main(server, eventHandler, params ):
     errors = 0
     warnings = 0
     taskfailures = []
+    first = True
 
     buildinfohelper = BuildInfoHelper(server, build_history_enabled)
 
-
     while True:
         try:
             event = eventHandler.waitEvent(0.25)
+            if first:
+                first = False
+                logger.info("ToasterUI waiting for events")
 
             if event is None:
                 if main.shutdown > 0:
@@ -218,19 +221,28 @@ def main(server, eventHandler, params ):
                                   bb.command.CommandExit)):
                 if (isinstance(event, bb.command.CommandFailed)):
                     event.levelno = format.ERROR
-                    event.msg = event.error
+                    event.msg = "Command Failed " + event.error
                     event.pathname = ""
                     event.lineno = 0
                     buildinfohelper.store_log_event(event)
                     errors += 1
 
                 buildinfohelper.update_build_information(event, errors, warnings, taskfailures)
+                buildinfohelper.close()
+
 
                 # we start a new build info
-                errors = 0
-                warnings = 0
-                taskfailures = []
-                buildinfohelper = BuildInfoHelper(server, build_history_enabled)
+                if buildinfohelper.brbe is not None:
+
+                    print "we are under BuildEnvironment management - after the build, we exit"
+                    server.terminateServer()
+                else:
+                    print "prepared for new build"
+                    errors = 0
+                    warnings = 0
+                    taskfailures = []
+                    buildinfohelper = BuildInfoHelper(server, build_history_enabled)
+
                 continue
 
             if isinstance(event, bb.event.MetadataEvent):

bitbake/lib/bb/ui/uievent.py

@@ -44,10 +44,27 @@ class BBUIEventQueue:
         server.register_function( self.send_event, "event.sendpickle" )
         server.socket.settimeout(1)
 
-        self.EventHandle = self.BBServer.registerEventHandler(self.host, self.port)
+        self.EventHandler = None
+        count_tries = 0
 
-        if (self.EventHandle == None):
-            bb.fatal("Could not register UI event handler")
+        # the event handler registration may fail here due to cooker being in invalid state
+        # this is a transient situation, and we should retry a couple of times before
+        # giving up
+
+        while self.EventHandler == None and count_tries < 5:
+            self.EventHandle = self.BBServer.registerEventHandler(self.host, self.port)
+
+            if (self.EventHandle != None):
+                break
+
+            bb.warn("Could not register UI event handler %s:%d, retry" % (self.host, self.port))
+            count_tries += 1
+            import time
+            time.sleep(1)
+
+
+        if self.EventHandle == None:
+            raise Exception("Could not register UI event handler")
 
         self.server = server
 

bitbake/lib/bb/utils.py

@@ -264,7 +264,7 @@ def _print_trace(body, line):
 def better_compile(text, file, realfile, mode = "exec"):
     """
     A better compile method. This method
-    will print  the offending lines.
+    will print the offending lines.
     """
     try:
         return compile(text, file, mode)
@@ -354,14 +354,11 @@ def better_exec(code, context, text = None, realfile = "<code>"):
         code = better_compile(code, realfile, realfile)
     try:
         exec(code, get_context(), context)
-    except bb.BBHandledException:
-        # Error already shown so passthrough
+    except (bb.BBHandledException, bb.parse.SkipRecipe, bb.build.FuncFailed, bb.data_smart.ExpansionError):
+        # Error already shown so passthrough, no need for traceback
         raise
     except Exception as e:
         (t, value, tb) = sys.exc_info()
-
-        if t in [bb.parse.SkipPackage, bb.build.FuncFailed]:
-            raise
         try:
             _print_exception(t, value, tb, realfile, text, context)
         except Exception as e:
@@ -533,7 +530,7 @@ def filter_environment(good_vars):
 def approved_variables():
     """
     Determine and return the list of whitelisted variables which are approved
-    to remain in the envrionment.
+    to remain in the environment.
     """
     if 'BB_PRESERVE_ENV' in os.environ:
         return os.environ.keys()
@@ -845,6 +842,19 @@ def contains(variable, checkvalues, truevalue, falsevalue, d):
         return truevalue
     return falsevalue
 
+def contains_any(variable, checkvalues, truevalue, falsevalue, d):
+    val = d.getVar(variable, True)
+    if not val:
+        return falsevalue
+    val = set(val.split())
+    if isinstance(checkvalues, basestring):
+        checkvalues = set(checkvalues.split())
+    else:
+        checkvalues = set(checkvalues)
+    if checkvalues & val:
+        return truevalue
+    return falsevalue
+
 def cpu_count():
     return multiprocessing.cpu_count()
 
@@ -852,21 +862,16 @@ def nonblockingfd(fd):
     fcntl.fcntl(fd, fcntl.F_SETFL, fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK)
 
 def process_profilelog(fn):
-    # Redirect stdout to capture profile information
     pout = open(fn + '.processed', 'w')
-    so = sys.stdout.fileno()
-    orig_so = os.dup(sys.stdout.fileno())
-    os.dup2(pout.fileno(), so)
    
     import pstats
-    p = pstats.Stats(fn)
+    p = pstats.Stats(fn, stream=pout)
     p.sort_stats('time')
     p.print_stats()
     p.print_callers()
     p.sort_stats('cumulative')
     p.print_stats()
 
-    os.dup2(orig_so, so)
     pout.flush()
     pout.close()  
 
@@ -874,5 +879,17 @@ def process_profilelog(fn):
 # Was present to work around multiprocessing pool bugs in python < 2.7.3
 #
 def multiprocessingpool(*args, **kwargs):
+
+    import multiprocessing.pool
+    #import multiprocessing.util
+    #multiprocessing.util.log_to_stderr(10)
+    # Deal with a multiprocessing bug where signals to the processes would be delayed until the work
+    # completes. Putting in a timeout means the signals (like SIGINT/SIGTERM) get processed.
+    def wrapper(func):
+        def wrap(self, timeout=None):
+            return func(self, timeout=timeout if timeout is not None else 1e100)
+        return wrap
+    multiprocessing.pool.IMapIterator.next = wrapper(multiprocessing.pool.IMapIterator.next)
+
     return multiprocessing.Pool(*args, **kwargs)
 

bitbake/lib/prserv/__init__.py

@@ -11,4 +11,4 @@ def init_logger(logfile, loglevel):
     logging.basicConfig(level=numeric_level, filename=logfile, format=FORMAT)
 
 class NotFoundError(Exception):
-    pass
+    pass

bitbake/lib/prserv/db.py

@@ -220,6 +220,7 @@ class PRData(object):
                 raise e
         self.connection=sqlite3.connect(self.filename, isolation_level="EXCLUSIVE", check_same_thread = False)
         self.connection.row_factory=sqlite3.Row
+        self.connection.execute("pragma synchronous = off;")
         self._tables={}
 
     def __del__(self):

bitbake/lib/prserv/serv.py

@@ -38,8 +38,17 @@ singleton = None
 class PRServer(SimpleXMLRPCServer):
     def __init__(self, dbfile, logfile, interface, daemon=True):
         ''' constructor '''
-        SimpleXMLRPCServer.__init__(self, interface,
-                                    logRequests=False, allow_none=True)
+        import socket
+        try:
+            SimpleXMLRPCServer.__init__(self, interface,
+                                        logRequests=False, allow_none=True)
+        except socket.error:
+            ip=socket.gethostbyname(interface[0])
+            port=interface[1]
+            msg="PR Server unable to bind to %s:%s\n" % (ip, port)
+            sys.stderr.write(msg)
+            raise PRServiceConfigError
+
         self.dbfile=dbfile
         self.daemon=daemon
         self.logfile=logfile

bitbake/lib/toaster/bldcontrol/admin.py

@@ -0,0 +1,8 @@
+from django.contrib import admin
+from django.contrib.admin.filters import RelatedFieldListFilter
+from .models import BuildEnvironment
+
+class BuildEnvironmentAdmin(admin.ModelAdmin):
+    pass
+
+admin.site.register(BuildEnvironment, BuildEnvironmentAdmin)

bitbake/lib/toaster/bldcontrol/bbcontroller.py

@@ -0,0 +1,183 @@
+#
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2014        Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import os
+import sys
+import re
+from django.db import transaction
+from django.db.models import Q
+from bldcontrol.models import BuildEnvironment, BRLayer, BRVariable, BRTarget, BRBitbake
+
+# load Bitbake components
+path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+sys.path.insert(0, path)
+import bb.server.xmlrpc
+
+class BitbakeController(object):
+    """ This is the basic class that controlls a bitbake server.
+        It is outside the scope of this class on how the server is started and aquired
+    """
+
+    def __init__(self, connection):
+        self.connection = connection
+
+    def _runCommand(self, command):
+        result, error = self.connection.connection.runCommand(command)
+        if error:
+            raise Exception(error)
+        return result
+
+    def disconnect(self):
+        return self.connection.removeClient()
+
+    def setVariable(self, name, value):
+        return self._runCommand(["setVariable", name, value])
+
+    def build(self, targets, task = None):
+        if task is None:
+            task = "build"
+        return self._runCommand(["buildTargets", targets, task])
+
+
+
+def getBuildEnvironmentController(**kwargs):
+    """ Gets you a BuildEnvironmentController that encapsulates a build environment,
+        based on the query dictionary sent in.
+
+        This is used to retrieve, for example, the currently running BE from inside
+        the toaster UI, or find a new BE to start a new build in it.
+
+        The return object MUST always be a BuildEnvironmentController.
+    """
+
+    from localhostbecontroller import LocalhostBEController
+    from sshbecontroller    import SSHBEController
+
+    be = BuildEnvironment.objects.filter(Q(**kwargs))[0]
+    if be.betype == BuildEnvironment.TYPE_LOCAL:
+        return LocalhostBEController(be)
+    elif be.betype == BuildEnvironment.TYPE_SSH:
+        return SSHBEController(be)
+    else:
+        raise Exception("FIXME: Implement BEC for type %s" % str(be.betype))
+
+
+def _getgitcheckoutdirectoryname(url):
+    """ Utility that returns the last component of a git path as directory
+    """
+    import re
+    components = re.split(r'[:\.\/]', url)
+    return components[-2] if components[-1] == "git" else components[-1]
+
+
+class BuildEnvironmentController(object):
+    """ BuildEnvironmentController (BEC) is the abstract class that defines the operations that MUST
+        or SHOULD be supported by a Build Environment. It is used to establish the framework, and must
+        not be instantiated directly by the user.
+
+        Use the "getBuildEnvironmentController()" function to get a working BEC for your remote.
+
+        How the BuildEnvironments are discovered is outside the scope of this class.
+
+        You must derive this class to teach Toaster how to operate in your own infrastructure.
+        We provide some specific BuildEnvironmentController classes that can be used either to
+        directly set-up Toaster infrastructure, or as a model for your own infrastructure set:
+
+            * Localhost controller will run the Toaster BE on the same account as the web server
+        (current user if you are using the the Django development web server)
+        on the local machine, with the "build/" directory under the "poky/" source checkout directory.
+        Bash is expected to be available.
+
+            * SSH controller will run the Toaster BE on a remote machine, where the current user
+        can connect without raise Exception("FIXME: implement")word (set up with either ssh-agent or raise Exception("FIXME: implement")phrase-less key authentication)
+
+    """
+    def __init__(self, be):
+        """ Takes a BuildEnvironment object as parameter that points to the settings of the BE.
+        """
+        self.be = be
+        self.connection = None
+
+
+    def startBBServer(self):
+        """ Starts a  BB server with Toaster toasterui set up to record the builds, an no controlling UI.
+            After this method executes, self.be bbaddress/bbport MUST point to a running and free server,
+            and the bbstate MUST be  updated to "started".
+        """
+        raise Exception("Must override in order to actually start the BB server")
+
+    def stopBBServer(self):
+        """ Stops the currently running BB server.
+            The bbstate MUST be updated to "stopped".
+            self.connection must be none.
+        """
+
+    def setLayers(self, bbs, ls):
+        """ Checks-out bitbake executor and layers from git repositories.
+            Sets the layer variables in the config file, after validating local layer paths.
+            The bitbakes must be a 1-length list of BRBitbake
+            The layer paths must be in a list of BRLayer object
+
+            a word of attention: by convention, the first layer for any build will be poky!
+        """
+        raise Exception("Must override setLayers")
+
+
+    def getBBController(self):
+        """ returns a BitbakeController to an already started server; this is the point where the server
+            starts if needed; or reconnects to the server if we can
+        """
+        if not self.connection:
+            self.startBBServer()
+            self.be.lock = BuildEnvironment.LOCK_RUNNING
+            self.be.save()
+
+        server = bb.server.xmlrpc.BitBakeXMLRPCClient()
+        server.initServer()
+        server.saveConnectionDetails("%s:%s" % (self.be.bbaddress, self.be.bbport))
+        self.connection = server.establishConnection([])
+
+        self.be.bbtoken = self.connection.transport.connection_token
+        self.be.save()
+
+        return BitbakeController(self.connection)
+
+    def getArtifact(path):
+        """ This call returns an artifact identified by the 'path'. How 'path' is interpreted as
+            up to the implementing BEC. The return MUST be a REST URL where a GET will actually return
+            the content of the artifact, e.g. for use as a "download link" in a web UI.
+        """
+        raise Exception("Must return the REST URL of the artifact")
+
+    def release(self):
+        """ This stops the server and releases any resources. After this point, all resources
+            are un-available for further reference
+        """
+        raise Exception("Must override BE release")
+
+class ShellCmdException(Exception):
+    pass
+
+
+class BuildSetupException(Exception):
+    pass
+

bitbake/lib/toaster/bldcontrol/localhostbecontroller.py

@@ -0,0 +1,191 @@
+#
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2014        Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import os
+import sys
+import re
+from django.db import transaction
+from django.db.models import Q
+from bldcontrol.models import BuildEnvironment, BRLayer, BRVariable, BRTarget, BRBitbake
+import subprocess
+
+from toastermain import settings
+
+from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException, _getgitcheckoutdirectoryname
+
+class LocalhostBEController(BuildEnvironmentController):
+    """ Implementation of the BuildEnvironmentController for the localhost;
+        this controller manages the default build directory,
+        the server setup and system start and stop for the localhost-type build environment
+
+    """
+
+    def __init__(self, be):
+        super(LocalhostBEController, self).__init__(be)
+        self.dburl = settings.getDATABASE_URL()
+        self.pokydirname = None
+        self.islayerset = False
+
+    def _shellcmd(self, command, cwd = None):
+        if cwd is None:
+            cwd = self.be.sourcedir
+
+        p = subprocess.Popen(command, cwd = cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        (out,err) = p.communicate()
+        if p.returncode:
+            if len(err) == 0:
+                err = "command: %s \n%s" % (command, out)
+            else:
+                err = "command: %s \n%s" % (command, err)
+            raise ShellCmdException(err)
+        else:
+            return out
+
+    def _createdirpath(self, path):
+        from os.path import dirname as DN
+        if path == "":
+            raise Exception("Invalid path creation specified.")
+        if not os.path.exists(DN(path)):
+            self._createdirpath(DN(path))
+        if not os.path.exists(path):
+            os.mkdir(path, 0755)
+
+    def _setupBE(self):
+        assert self.pokydirname and os.path.exists(self.pokydirname)
+        self._createdirpath(self.be.builddir)
+        self._shellcmd("bash -c \"source %s/oe-init-build-env %s\"" % (self.pokydirname, self.be.builddir))
+
+    def startBBServer(self):
+        assert self.pokydirname and os.path.exists(self.pokydirname)
+        assert self.islayerset
+        print("DEBUG: executing ", "bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb && sleep 1\"" % (self.pokydirname, self.be.builddir, self.dburl))
+        print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb && sleep 1\"" % (self.pokydirname, self.be.builddir, self.dburl))
+        # FIXME unfortunate sleep 1 - we need to make sure that bbserver is started and the toaster ui is connected
+        # but since they start async without any return, we just wait a bit
+        print "Started server"
+        assert self.be.sourcedir and os.path.exists(self.be.builddir)
+        self.be.bbaddress = "localhost"
+        self.be.bbport = "8200"
+        self.be.bbstate = BuildEnvironment.SERVER_STARTED
+        self.be.save()
+
+    def stopBBServer(self):
+        assert self.pokydirname and os.path.exists(self.pokydirname)
+        assert self.islayerset
+        print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && %s source toaster stop\"" %
+            (self.pokydirname, self.be.builddir, (lambda: "" if self.be.bbtoken is None else "BBTOKEN=%s" % self.be.bbtoken)()))
+        self.be.bbstate = BuildEnvironment.SERVER_STOPPED
+        self.be.save()
+        print "Stopped server"
+
+    def setLayers(self, bitbakes, layers):
+        """ a word of attention: by convention, the first layer for any build will be poky! """
+
+        assert self.be.sourcedir is not None
+        assert len(bitbakes) == 1
+        # set layers in the layersource
+
+        # 1. get a list of repos, and map dirpaths for each layer
+        gitrepos = {}
+        gitrepos[bitbakes[0].giturl] = []
+        gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) )
+        
+        for layer in layers:
+            # we don't process local URLs
+            if layer.giturl.startswith("file://"):
+                continue
+            if not layer.giturl in gitrepos:
+                gitrepos[layer.giturl] = []
+            gitrepos[layer.giturl].append( (layer.name, layer.dirpath, layer.commit))
+        for giturl in gitrepos.keys():
+            commitid = gitrepos[giturl][0][2]
+            for e in gitrepos[giturl]:
+                if commitid != e[2]:
+                    raise BuildSetupException("More than one commit per git url, unsupported configuration")
+
+
+        layerlist = []
+
+        # 2. checkout the repositories
+        for giturl in gitrepos.keys():
+            localdirname = os.path.join(self.be.sourcedir, _getgitcheckoutdirectoryname(giturl))
+            print "DEBUG: giturl ", giturl ,"checking out in current directory", localdirname
+
+            # make sure our directory is a git repository
+            if os.path.exists(localdirname):
+                if not giturl in self._shellcmd("git remote -v", localdirname):
+                    raise BuildSetupException("Existing git repository at %s, but with different remotes (not '%s'). Aborting." % (localdirname, giturl))
+            else:
+                self._shellcmd("git clone \"%s\" \"%s\"" % (giturl, localdirname))
+            # checkout the needed commit
+            commit = gitrepos[giturl][0][2]
+
+            # branch magic name "HEAD" will inhibit checkout
+            if commit != "HEAD":
+                print "DEBUG: checking out commit ", commit, "to", localdirname
+                self._shellcmd("git fetch --all && git checkout \"%s\"" % commit , localdirname)
+
+            # take the localdirname as poky dir if we can find the oe-init-build-env
+            if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")):
+                print "DEBUG: selected poky dir name", localdirname
+                self.pokydirname = localdirname
+
+            # verify our repositories
+            for name, dirpath, commit in gitrepos[giturl]:
+                localdirpath = os.path.join(localdirname, dirpath)
+                if not os.path.exists(localdirpath):
+                    raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit))
+
+                if name != "bitbake":
+                    layerlist.append(localdirpath)
+
+        print "DEBUG: current layer list ", layerlist
+
+        # 3. configure the build environment, so we have a conf/bblayers.conf
+        assert self.pokydirname is not None
+        self._setupBE()
+
+        # 4. update the bblayers.conf
+        bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf")
+        if not os.path.exists(bblayerconf):
+            raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf)
+
+        conflines = open(bblayerconf, "r").readlines()
+
+        bblayerconffile = open(bblayerconf, "w")
+        for i in xrange(len(conflines)):
+            if conflines[i].startswith("# line added by toaster"):
+                i += 2
+            else:
+                bblayerconffile.write(conflines[i])
+
+        bblayerconffile.write("\n# line added by toaster build control\nBBLAYERS = \"" + " ".join(layerlist) + "\"")
+        bblayerconffile.close()
+
+        self.islayerset = True
+        return True
+
+    def release(self):
+        assert self.be.sourcedir and os.path.exists(self.be.builddir)
+        import shutil
+        shutil.rmtree(os.path.join(self.be.sourcedir, "build"))
+        assert not os.path.exists(self.be.builddir)

bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py

@@ -0,0 +1,212 @@
+from django.core.management.base import NoArgsCommand, CommandError
+from django.db import transaction
+from orm.models import LayerSource, ToasterSetting, Branch, Layer, Layer_Version
+from orm.models import BitbakeVersion, Release, ReleaseDefaultLayer
+from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException
+from bldcontrol.models import BuildRequest, BuildEnvironment
+import os
+
+def DN(path):
+    if path is None:
+        return ""
+    else:
+        return os.path.dirname(path)
+
+
+class Command(NoArgsCommand):
+    args = ""
+    help = "Verifies that the configured settings are valid and usable, or prompts the user to fix the settings."
+
+    def _reduce_canon_path(self, path):
+        components = []
+        for c in path.split("/"):
+            if c == "..":
+                del components[-1]
+            elif c == ".":
+                pass
+            else:
+                components.append(c)
+        return "/".join(components)
+
+    def _find_first_path_for_file(self, startdirectory, filename, level = 0):
+        if level < 0:
+            return None
+        dirs = []
+        for i in os.listdir(startdirectory):
+            j = os.path.join(startdirectory, i)
+            if os.path.isfile(j):
+                if i == filename:
+                    return startdirectory
+            elif os.path.isdir(j):
+                dirs.append(j)
+        for j in dirs:
+            ret = self._find_first_path_for_file(j, filename, level - 1)
+            if ret is not None:
+                return ret
+        return None
+
+    def _get_suggested_sourcedir(self, be):
+        if be.betype != BuildEnvironment.TYPE_LOCAL:
+            return ""
+        return DN(DN(DN(self._find_first_path_for_file(self.guesspath, "toasterconf.json", 4))))
+
+    def _get_suggested_builddir(self, be):
+        if be.betype != BuildEnvironment.TYPE_LOCAL:
+            return ""
+        return DN(self._find_first_path_for_file(self.guesspath, "bblayers.conf", 3))
+
+    def _import_layer_config(self, baselayerdir):
+        filepath = os.path.join(baselayerdir, "meta/conf/toasterconf.json")
+        if not os.path.exists(filepath) or not os.path.isfile(filepath):
+            raise Exception("Failed to find toaster config file %s ." % filepath)
+
+        import json, pprint
+        data = json.loads(open(filepath, "r").read())
+
+        # verify config file validity before updating settings
+        for i in ['bitbake', 'releases', 'defaultrelease', 'config', 'layersources']:
+            assert i in data
+
+        # import bitbake data
+        for bvi in data['bitbake']:
+            bvo, created = BitbakeVersion.objects.get_or_create(name=bvi['name'])
+            bvo.giturl = bvi['giturl']
+            bvo.branch = bvi['branch']
+            bvo.dirpath = bvi['dirpath']
+            bvo.save()
+
+        # set the layer sources
+        for lsi in data['layersources']:
+            assert 'sourcetype' in lsi
+            assert 'apiurl' in lsi
+            assert 'name' in lsi
+            assert 'branches' in lsi
+
+            if lsi['sourcetype'] == LayerSource.TYPE_LAYERINDEX or lsi['apiurl'].startswith("/"):
+                apiurl = lsi['apiurl']
+            else:
+                apiurl = self._reduce_canon_path(os.path.join(DN(filepath), lsi['apiurl']))
+
+            try:
+                ls = LayerSource.objects.get(sourcetype = lsi['sourcetype'], apiurl = apiurl)
+            except LayerSource.DoesNotExist:
+                ls = LayerSource.objects.create(
+                    name = lsi['name'],
+                    sourcetype = lsi['sourcetype'],
+                    apiurl = apiurl
+                )
+
+            layerbranches = []
+            for branchname in lsi['branches']:
+                bo, created = Branch.objects.get_or_create(layer_source = ls, name = branchname)
+                layerbranches.append(bo)
+
+            if 'layers' in lsi:
+                for layerinfo in lsi['layers']:
+                    lo, created = Layer.objects.get_or_create(layer_source = ls, name = layerinfo['name'])
+                    if layerinfo['local_path'].startswith("/"):
+                        lo.local_path = layerinfo['local_path']
+                    else:
+                        lo.local_path = self._reduce_canon_path(os.path.join(DN(filepath), layerinfo['local_path']))
+                    lo.layer_index_url = layerinfo['layer_index_url']
+                    if 'vcs_url' in layerinfo:
+                        lo.vcs_url = layerinfo['vcs_url']
+                    lo.save()
+
+                    for branch in layerbranches:
+                        lvo, created = Layer_Version.objects.get_or_create(layer_source = ls,
+                                up_branch = branch,
+                                commit = branch.name,
+                                layer = lo)
+                        lvo.dirpath = layerinfo['dirpath']
+                        lvo.save()
+        # set releases
+        for ri in data['releases']:
+            bvo = BitbakeVersion.objects.get(name = ri['bitbake'])
+            assert bvo is not None
+
+            ro, created = Release.objects.get_or_create(name = ri['name'], bitbake_version = bvo)
+            ro.description = ri['description']
+            ro.branch = ri['branch']
+            ro.save()
+
+            for dli in ri['defaultlayers']:
+                lsi, layername = dli.split(":")
+                layer, created = Layer.objects.get_or_create( 
+                        layer_source = LayerSource.objects.get(name = lsi),
+                        name = layername
+                    )
+                ReleaseDefaultLayer.objects.get_or_create( release = ro, layer = layer)
+
+        # set default release
+        if ToasterSetting.objects.filter(name = "DEFAULT_RELEASE").count() > 0:
+            ToasterSetting.objects.filter(name = "DEFAULT_RELEASE").update(value = data['defaultrelease'])
+        else:
+            ToasterSetting.objects.create(name = "DEFAULT_RELEASE", value = data['defaultrelease'])
+
+        # set default config variables
+        for configname in data['config']:
+            if ToasterSetting.objects.filter(name = "DEFCONF_" + configname).count() > 0:
+                ToasterSetting.objects.filter(name = "DEFCONF_" + configname).update(value = data['config'][configname])
+            else:
+                ToasterSetting.objects.create(name = "DEFCONF_" + configname, value = data['config'][configname])
+
+    def handle(self, **options):
+        self.guesspath = DN(DN(DN(DN(DN(DN(DN(__file__)))))))
+
+        # we make sure we have builddir and sourcedir for all defined build envionments
+        for be in BuildEnvironment.objects.all():
+            def _verify_be():
+                is_changed = False
+                print("Verifying the Build Environment type %s id %d." % (be.get_betype_display(), be.pk))
+                if len(be.sourcedir) == 0:
+                    suggesteddir = self._get_suggested_sourcedir(be)
+                    be.sourcedir = raw_input(" -- Layer sources checkout directory may not be empty [guessed \"%s\"]:" % suggesteddir)
+                    if len(be.sourcedir) == 0 and len(suggesteddir) > 0:
+                        be.sourcedir = suggesteddir
+                    is_changed = True
+
+                if not be.sourcedir.startswith("/"):
+                    be.sourcedir = raw_input(" -- Layer sources checkout directory must be an absolute path:")
+                    is_changed = True
+
+                if len(be.builddir) == 0:
+                    suggesteddir = self._get_suggested_builddir(be)
+                    be.builddir = raw_input(" -- Build directory may not be empty [guessed \"%s\"]:" % suggesteddir)
+                    if len(be.builddir) == 0 and len(suggesteddir) > 0:
+                        be.builddir = suggesteddir
+                    is_changed = True
+
+                if not be.builddir.startswith("/"):
+                    be.builddir = raw_input(" -- Build directory must be an absolute path:")
+                    is_changed = True
+
+
+
+                if is_changed:
+                    print "Build configuration saved"
+                    be.save()
+
+                if is_changed and be.betype == BuildEnvironment.TYPE_LOCAL:
+                    baselayerdir = DN(DN(self._find_first_path_for_file(be.sourcedir, "toasterconf.json", 3)))
+                    if baselayerdir:
+                        i = raw_input(" -- Do you want to import basic layer configuration from \"%s\" ? (y/N):" % baselayerdir)
+                        if len(i) and i.upper()[0] == 'Y':
+                            self._import_layer_config(baselayerdir)
+                            # we run lsupdates after config update
+                            print "Updating information from the layer source, please wait."
+                            from django.core.management import call_command
+                            call_command("lsupdates")
+                        pass
+
+                return is_changed
+
+            while (_verify_be()):
+                pass
+
+        # verify that default settings are there
+        if ToasterSetting.objects.filter(name = 'DEFAULT_RELEASE').count() != 1:
+            ToasterSetting.objects.filter(name = 'DEFAULT_RELEASE').delete()
+            ToasterSetting.objects.get_or_create(name = 'DEFAULT_RELEASE', value = '')
+
+        return 0

bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py

@@ -0,0 +1,90 @@
+from django.core.management.base import NoArgsCommand, CommandError
+from django.db import transaction
+from orm.models import Build
+from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException, BuildSetupException
+from bldcontrol.models import BuildRequest, BuildEnvironment, BRError
+import os
+
+class Command(NoArgsCommand):
+    args    = ""
+    help    = "Schedules and executes build requests as possible. Does not return (interrupt with Ctrl-C)"
+
+
+    @transaction.commit_on_success
+    def _selectBuildEnvironment(self):
+        bec = getBuildEnvironmentController(lock = BuildEnvironment.LOCK_FREE)
+        bec.be.lock = BuildEnvironment.LOCK_LOCK
+        bec.be.save()
+        return bec
+
+    @transaction.commit_on_success
+    def _selectBuildRequest(self):
+        br = BuildRequest.objects.filter(state = BuildRequest.REQ_QUEUED).order_by('pk')[0]
+        br.state = BuildRequest.REQ_INPROGRESS
+        br.save()
+        return br
+
+    def schedule(self):
+        import traceback
+        try:
+            br = None
+            try:
+                # select the build environment and the request to build
+                br = self._selectBuildRequest()
+            except IndexError as e:
+                return
+            try:
+                bec = self._selectBuildEnvironment()
+            except IndexError as e:
+                # we could not find a BEC; postpone the BR
+                br.state = BuildRequest.REQ_QUEUED
+                br.save()
+                return
+
+            # set up the buid environment with the needed layers
+            print "Build %s, Environment %s" % (br, bec.be)
+            bec.setLayers(br.brbitbake_set.all(), br.brlayer_set.all())
+
+            # get the bb server running
+            bbctrl = bec.getBBController()
+
+            # let toasterui that this is a managed build
+            bbctrl.setVariable("TOASTER_BRBE", "%d:%d" % (br.pk, bec.be.pk))
+
+            # set the build configuration
+            for variable in br.brvariable_set.all():
+                bbctrl.setVariable(variable.name, variable.value)
+
+            # trigger the build command
+            bbctrl.build(list(map(lambda x:x.target, br.brtarget_set.all())))
+
+            print "Build launched, exiting"
+            # disconnect from the server
+            bbctrl.disconnect()
+
+            # cleanup to be performed by toaster when the deed is done
+
+
+        except Exception as e:
+            print " EE Error executing shell command\n", e
+            traceback.print_exc(e)
+            BRError.objects.create(req = br,
+                errtype = str(type(e)),
+                errmsg = str(e),
+                traceback = traceback.format_exc(e))
+            br.state = BuildRequest.REQ_FAILED
+            br.save()
+            bec.be.lock = BuildEnvironment.LOCK_FREE
+            bec.be.save()
+
+
+    def cleanup(self):
+        from django.utils import timezone
+        from datetime import timedelta
+        # environments locked for more than 30 seconds - they should be unlocked
+        BuildEnvironment.objects.filter(lock=BuildEnvironment.LOCK_LOCK).filter(updated__lt = timezone.now() - timedelta(seconds = 30)).update(lock = BuildEnvironment.LOCK_FREE)
+
+
+    def handle_noargs(self, **options):
+        self.cleanup()
+        self.schedule()

bitbake/lib/toaster/bldcontrol/migrations/0001_initial.py

@@ -0,0 +1,154 @@
+# -*- coding: utf-8 -*-
+from south.utils import datetime_utils as datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'BuildEnvironment'
+        db.create_table(u'bldcontrol_buildenvironment', (
+            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('address', self.gf('django.db.models.fields.CharField')(max_length=254)),
+            ('betype', self.gf('django.db.models.fields.IntegerField')()),
+            ('bbaddress', self.gf('django.db.models.fields.CharField')(max_length=254, blank=True)),
+            ('bbport', self.gf('django.db.models.fields.IntegerField')(default=-1)),
+            ('bbtoken', self.gf('django.db.models.fields.CharField')(max_length=126, blank=True)),
+            ('bbstate', self.gf('django.db.models.fields.IntegerField')(default=0)),
+            ('lock', self.gf('django.db.models.fields.IntegerField')(default=0)),
+            ('created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)),
+            ('updated', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, blank=True)),
+        ))
+        db.send_create_signal(u'bldcontrol', ['BuildEnvironment'])
+
+        # Adding model 'BuildRequest'
+        db.create_table(u'bldcontrol_buildrequest', (
+            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('project', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['orm.Project'])),
+            ('build', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['orm.Build'], null=True)),
+            ('state', self.gf('django.db.models.fields.IntegerField')(default=0)),
+            ('created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)),
+            ('updated', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, blank=True)),
+        ))
+        db.send_create_signal(u'bldcontrol', ['BuildRequest'])
+
+        # Adding model 'BRLayer'
+        db.create_table(u'bldcontrol_brlayer', (
+            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('req', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['bldcontrol.BuildRequest'])),
+            ('name', self.gf('django.db.models.fields.CharField')(max_length=100)),
+            ('giturl', self.gf('django.db.models.fields.CharField')(max_length=254)),
+            ('commit', self.gf('django.db.models.fields.CharField')(max_length=254)),
+        ))
+        db.send_create_signal(u'bldcontrol', ['BRLayer'])
+
+        # Adding model 'BRVariable'
+        db.create_table(u'bldcontrol_brvariable', (
+            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('req', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['bldcontrol.BuildRequest'])),
+            ('name', self.gf('django.db.models.fields.CharField')(max_length=100)),
+            ('value', self.gf('django.db.models.fields.TextField')(blank=True)),
+        ))
+        db.send_create_signal(u'bldcontrol', ['BRVariable'])
+
+        # Adding model 'BRTarget'
+        db.create_table(u'bldcontrol_brtarget', (
+            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('req', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['bldcontrol.BuildRequest'])),
+            ('target', self.gf('django.db.models.fields.CharField')(max_length=100)),
+            ('task', self.gf('django.db.models.fields.CharField')(max_length=100, null=True)),
+        ))
+        db.send_create_signal(u'bldcontrol', ['BRTarget'])
+
+
+    def backwards(self, orm):
+        # Deleting model 'BuildEnvironment'
+        db.delete_table(u'bldcontrol_buildenvironment')
+
+        # Deleting model 'BuildRequest'
+        db.delete_table(u'bldcontrol_buildrequest')
+
+        # Deleting model 'BRLayer'
+        db.delete_table(u'bldcontrol_brlayer')
+
+        # Deleting model 'BRVariable'
+        db.delete_table(u'bldcontrol_brvariable')
+
+        # Deleting model 'BRTarget'
+        db.delete_table(u'bldcontrol_brtarget')
+
+
+    models = {
+        u'bldcontrol.brlayer': {
+            'Meta': {'object_name': 'BRLayer'},
+            'commit': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
+            'giturl': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"})
+        },
+        u'bldcontrol.brtarget': {
+            'Meta': {'object_name': 'BRTarget'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}),
+            'target': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'task': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True'})
+        },
+        u'bldcontrol.brvariable': {
+            'Meta': {'object_name': 'BRVariable'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}),
+            'value': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        u'bldcontrol.buildenvironment': {
+            'Meta': {'object_name': 'BuildEnvironment'},
+            'address': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
+            'bbaddress': ('django.db.models.fields.CharField', [], {'max_length': '254', 'blank': 'True'}),
+            'bbport': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
+            'bbstate': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'bbtoken': ('django.db.models.fields.CharField', [], {'max_length': '126', 'blank': 'True'}),
+            'betype': ('django.db.models.fields.IntegerField', [], {}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'lock': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
+        },
+        u'bldcontrol.buildrequest': {
+            'Meta': {'object_name': 'BuildRequest'},
+            'build': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Build']", 'null': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'project': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Project']"}),
+            'state': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
+        },
+        u'orm.build': {
+            'Meta': {'object_name': 'Build'},
+            'bitbake_version': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+            'build_name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'completed_on': ('django.db.models.fields.DateTimeField', [], {}),
+            'cooker_log_path': ('django.db.models.fields.CharField', [], {'max_length': '500'}),
+            'distro': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'distro_version': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'errors_no': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'machine': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'outcome': ('django.db.models.fields.IntegerField', [], {'default': '2'}),
+            'project': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Project']", 'null': 'True'}),
+            'started_on': ('django.db.models.fields.DateTimeField', [], {}),
+            'timespent': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'warnings_no': ('django.db.models.fields.IntegerField', [], {'default': '0'})
+        },
+        u'orm.project': {
+            'Meta': {'object_name': 'Project'},
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
+        }
+    }
+
+    complete_apps = ['bldcontrol']

bitbake/lib/toaster/bldcontrol/migrations/0002_auto__add_field_buildenvironment_sourcedir__add_field_buildenvironment.py

@@ -0,0 +1,106 @@
+# -*- coding: utf-8 -*-
+from south.utils import datetime_utils as datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding field 'BuildEnvironment.sourcedir'
+        db.add_column(u'bldcontrol_buildenvironment', 'sourcedir',
+                      self.gf('django.db.models.fields.CharField')(default='', max_length=512, blank=True),
+                      keep_default=False)
+
+        # Adding field 'BuildEnvironment.builddir'
+        db.add_column(u'bldcontrol_buildenvironment', 'builddir',
+                      self.gf('django.db.models.fields.CharField')(default='', max_length=512, blank=True),
+                      keep_default=False)
+
+
+    def backwards(self, orm):
+        # Deleting field 'BuildEnvironment.sourcedir'
+        db.delete_column(u'bldcontrol_buildenvironment', 'sourcedir')
+
+        # Deleting field 'BuildEnvironment.builddir'
+        db.delete_column(u'bldcontrol_buildenvironment', 'builddir')
+
+
+    models = {
+        u'bldcontrol.brlayer': {
+            'Meta': {'object_name': 'BRLayer'},
+            'commit': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
+            'giturl': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"})
+        },
+        u'bldcontrol.brtarget': {
+            'Meta': {'object_name': 'BRTarget'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}),
+            'target': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'task': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True'})
+        },
+        u'bldcontrol.brvariable': {
+            'Meta': {'object_name': 'BRVariable'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}),
+            'value': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        u'bldcontrol.buildenvironment': {
+            'Meta': {'object_name': 'BuildEnvironment'},
+            'address': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
+            'bbaddress': ('django.db.models.fields.CharField', [], {'max_length': '254', 'blank': 'True'}),
+            'bbport': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
+            'bbstate': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'bbtoken': ('django.db.models.fields.CharField', [], {'max_length': '126', 'blank': 'True'}),
+            'betype': ('django.db.models.fields.IntegerField', [], {}),
+            'builddir': ('django.db.models.fields.CharField', [], {'max_length': '512', 'blank': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'lock': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'sourcedir': ('django.db.models.fields.CharField', [], {'max_length': '512', 'blank': 'True'}),
+            'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
+        },
+        u'bldcontrol.buildrequest': {
+            'Meta': {'object_name': 'BuildRequest'},
+            'build': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Build']", 'null': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'project': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Project']"}),
+            'state': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
+        },
+        u'orm.build': {
+            'Meta': {'object_name': 'Build'},
+            'bitbake_version': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+            'build_name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'completed_on': ('django.db.models.fields.DateTimeField', [], {}),
+            'cooker_log_path': ('django.db.models.fields.CharField', [], {'max_length': '500'}),
+            'distro': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'distro_version': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'errors_no': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'machine': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'outcome': ('django.db.models.fields.IntegerField', [], {'default': '2'}),
+            'project': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Project']", 'null': 'True'}),
+            'started_on': ('django.db.models.fields.DateTimeField', [], {}),
+            'timespent': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'warnings_no': ('django.db.models.fields.IntegerField', [], {'default': '0'})
+        },
+        u'orm.project': {
+            'Meta': {'object_name': 'Project'},
+            'branch': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'short_description': ('django.db.models.fields.CharField', [], {'max_length': '50', 'blank': 'True'}),
+            'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'user_id': ('django.db.models.fields.IntegerField', [], {'null': 'True'})
+        }
+    }
+
+    complete_apps = ['bldcontrol']

bitbake/lib/toaster/bldcontrol/migrations/0003_auto__add_field_brlayer_dirpath.py

@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+from south.utils import datetime_utils as datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding field 'BRLayer.dirpath'
+        db.add_column(u'bldcontrol_brlayer', 'dirpath',
+                      self.gf('django.db.models.fields.CharField')(default='', max_length=254),
+                      keep_default=False)
+
+
+    def backwards(self, orm):
+        # Deleting field 'BRLayer.dirpath'
+        db.delete_column(u'bldcontrol_brlayer', 'dirpath')
+
+
+    models = {
+        u'bldcontrol.brlayer': {
+            'Meta': {'object_name': 'BRLayer'},
+            'commit': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
+            'dirpath': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
+            'giturl': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"})
+        },
+        u'bldcontrol.brtarget': {
+            'Meta': {'object_name': 'BRTarget'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}),
+            'target': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'task': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True'})
+        },
+        u'bldcontrol.brvariable': {
+            'Meta': {'object_name': 'BRVariable'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}),
+            'value': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        u'bldcontrol.buildenvironment': {
+            'Meta': {'object_name': 'BuildEnvironment'},
+            'address': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
+            'bbaddress': ('django.db.models.fields.CharField', [], {'max_length': '254', 'blank': 'True'}),
+            'bbport': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
+            'bbstate': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'bbtoken': ('django.db.models.fields.CharField', [], {'max_length': '126', 'blank': 'True'}),
+            'betype': ('django.db.models.fields.IntegerField', [], {}),
+            'builddir': ('django.db.models.fields.CharField', [], {'max_length': '512', 'blank': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'lock': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'sourcedir': ('django.db.models.fields.CharField', [], {'max_length': '512', 'blank': 'True'}),
+            'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
+        },
+        u'bldcontrol.buildrequest': {
+            'Meta': {'object_name': 'BuildRequest'},
+            'build': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Build']", 'null': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'project': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Project']"}),
+            'state': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
+        },
+        u'orm.build': {
+            'Meta': {'object_name': 'Build'},
+            'bitbake_version': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+            'build_name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'completed_on': ('django.db.models.fields.DateTimeField', [], {}),
+            'cooker_log_path': ('django.db.models.fields.CharField', [], {'max_length': '500'}),
+            'distro': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'distro_version': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'errors_no': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'machine': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'outcome': ('django.db.models.fields.IntegerField', [], {'default': '2'}),
+            'project': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Project']", 'null': 'True'}),
+            'started_on': ('django.db.models.fields.DateTimeField', [], {}),
+            'timespent': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'warnings_no': ('django.db.models.fields.IntegerField', [], {'default': '0'})
+        },
+        u'orm.project': {
+            'Meta': {'object_name': 'Project'},
+            'branch': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'short_description': ('django.db.models.fields.CharField', [], {'max_length': '50', 'blank': 'True'}),
+            'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'user_id': ('django.db.models.fields.IntegerField', [], {'null': 'True'})
+        }
+    }
+
+    complete_apps = ['bldcontrol']

bitbake/lib/toaster/bldcontrol/migrations/0004_loadinitialdata.py

@@ -0,0 +1,104 @@
+# -*- coding: utf-8 -*-
+from south.utils import datetime_utils as datetime
+from south.db import db
+from south.v2 import DataMigration
+from django.db import models
+
+class Migration(DataMigration):
+
+    def forwards(self, orm):
+        "Write your forwards methods here."
+        # Note: Don't use "from appname.models import ModelName".
+        # Use orm.ModelName to refer to models in this application,
+        # and orm['appname.ModelName'] for models in other applications.
+        try:
+            orm.BuildEnvironment.objects.get(pk = 1)
+        except:
+            from django.utils import timezone
+            orm.BuildEnvironment.objects.create(pk = 1,
+                created = timezone.now(),
+                updated = timezone.now(),
+                betype = 0)
+
+    def backwards(self, orm):
+        "Write your backwards methods here."
+
+    models = {
+        u'bldcontrol.brlayer': {
+            'Meta': {'object_name': 'BRLayer'},
+            'commit': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
+            'dirpath': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
+            'giturl': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"})
+        },
+        u'bldcontrol.brtarget': {
+            'Meta': {'object_name': 'BRTarget'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}),
+            'target': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'task': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True'})
+        },
+        u'bldcontrol.brvariable': {
+            'Meta': {'object_name': 'BRVariable'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}),
+            'value': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        u'bldcontrol.buildenvironment': {
+            'Meta': {'object_name': 'BuildEnvironment'},
+            'address': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
+            'bbaddress': ('django.db.models.fields.CharField', [], {'max_length': '254', 'blank': 'True'}),
+            'bbport': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
+            'bbstate': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'bbtoken': ('django.db.models.fields.CharField', [], {'max_length': '126', 'blank': 'True'}),
+            'betype': ('django.db.models.fields.IntegerField', [], {}),
+            'builddir': ('django.db.models.fields.CharField', [], {'max_length': '512', 'blank': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'lock': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'sourcedir': ('django.db.models.fields.CharField', [], {'max_length': '512', 'blank': 'True'}),
+            'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
+        },
+        u'bldcontrol.buildrequest': {
+            'Meta': {'object_name': 'BuildRequest'},
+            'build': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Build']", 'null': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'project': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Project']"}),
+            'state': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
+        },
+        u'orm.build': {
+            'Meta': {'object_name': 'Build'},
+            'bitbake_version': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+            'build_name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'completed_on': ('django.db.models.fields.DateTimeField', [], {}),
+            'cooker_log_path': ('django.db.models.fields.CharField', [], {'max_length': '500'}),
+            'distro': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'distro_version': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'errors_no': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'machine': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'outcome': ('django.db.models.fields.IntegerField', [], {'default': '2'}),
+            'project': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Project']", 'null': 'True'}),
+            'started_on': ('django.db.models.fields.DateTimeField', [], {}),
+            'timespent': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'warnings_no': ('django.db.models.fields.IntegerField', [], {'default': '0'})
+        },
+        u'orm.project': {
+            'Meta': {'object_name': 'Project'},
+            'branch': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'short_description': ('django.db.models.fields.CharField', [], {'max_length': '50', 'blank': 'True'}),
+            'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'user_id': ('django.db.models.fields.IntegerField', [], {'null': 'True'})
+        }
+    }
+
+    complete_apps = ['bldcontrol']
+    symmetrical = True

bitbake/lib/toaster/bldcontrol/migrations/0005_auto__add_brerror.py

@@ -0,0 +1,112 @@
+# -*- coding: utf-8 -*-
+from south.utils import datetime_utils as datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'BRError'
+        db.create_table(u'bldcontrol_brerror', (
+            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('req', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['bldcontrol.BuildRequest'])),
+            ('errtype', self.gf('django.db.models.fields.CharField')(max_length=100)),
+            ('errmsg', self.gf('django.db.models.fields.TextField')()),
+            ('traceback', self.gf('django.db.models.fields.TextField')()),
+        ))
+        db.send_create_signal(u'bldcontrol', ['BRError'])
+
+
+    def backwards(self, orm):
+        # Deleting model 'BRError'
+        db.delete_table(u'bldcontrol_brerror')
+
+
+    models = {
+        u'bldcontrol.brerror': {
+            'Meta': {'object_name': 'BRError'},
+            'errmsg': ('django.db.models.fields.TextField', [], {}),
+            'errtype': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}),
+            'traceback': ('django.db.models.fields.TextField', [], {})
+        },
+        u'bldcontrol.brlayer': {
+            'Meta': {'object_name': 'BRLayer'},
+            'commit': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
+            'dirpath': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
+            'giturl': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"})
+        },
+        u'bldcontrol.brtarget': {
+            'Meta': {'object_name': 'BRTarget'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}),
+            'target': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'task': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True'})
+        },
+        u'bldcontrol.brvariable': {
+            'Meta': {'object_name': 'BRVariable'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}),
+            'value': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        u'bldcontrol.buildenvironment': {
+            'Meta': {'object_name': 'BuildEnvironment'},
+            'address': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
+            'bbaddress': ('django.db.models.fields.CharField', [], {'max_length': '254', 'blank': 'True'}),
+            'bbport': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
+            'bbstate': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'bbtoken': ('django.db.models.fields.CharField', [], {'max_length': '126', 'blank': 'True'}),
+            'betype': ('django.db.models.fields.IntegerField', [], {}),
+            'builddir': ('django.db.models.fields.CharField', [], {'max_length': '512', 'blank': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'lock': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'sourcedir': ('django.db.models.fields.CharField', [], {'max_length': '512', 'blank': 'True'}),
+            'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
+        },
+        u'bldcontrol.buildrequest': {
+            'Meta': {'object_name': 'BuildRequest'},
+            'build': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Build']", 'null': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'project': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Project']"}),
+            'state': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
+        },
+        u'orm.build': {
+            'Meta': {'object_name': 'Build'},
+            'bitbake_version': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+            'build_name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'completed_on': ('django.db.models.fields.DateTimeField', [], {}),
+            'cooker_log_path': ('django.db.models.fields.CharField', [], {'max_length': '500'}),
+            'distro': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'distro_version': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'errors_no': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'machine': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'outcome': ('django.db.models.fields.IntegerField', [], {'default': '2'}),
+            'project': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Project']", 'null': 'True'}),
+            'started_on': ('django.db.models.fields.DateTimeField', [], {}),
+            'timespent': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'warnings_no': ('django.db.models.fields.IntegerField', [], {'default': '0'})
+        },
+        u'orm.project': {
+            'Meta': {'object_name': 'Project'},
+            'branch': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'short_description': ('django.db.models.fields.CharField', [], {'max_length': '50', 'blank': 'True'}),
+            'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'user_id': ('django.db.models.fields.IntegerField', [], {'null': 'True'})
+        }
+    }
+
+    complete_apps = ['bldcontrol']

bitbake/lib/toaster/bldcontrol/migrations/0006_auto__add_brbitbake.py

@@ -0,0 +1,128 @@
+# -*- coding: utf-8 -*-
+from south.utils import datetime_utils as datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'BRBitbake'
+        db.create_table(u'bldcontrol_brbitbake', (
+            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('req', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['bldcontrol.BuildRequest'], unique=True)),
+            ('giturl', self.gf('django.db.models.fields.CharField')(max_length=254)),
+            ('commit', self.gf('django.db.models.fields.CharField')(max_length=254)),
+            ('dirpath', self.gf('django.db.models.fields.CharField')(max_length=254)),
+        ))
+        db.send_create_signal(u'bldcontrol', ['BRBitbake'])
+
+
+    def backwards(self, orm):
+        # Deleting model 'BRBitbake'
+        db.delete_table(u'bldcontrol_brbitbake')
+
+
+    models = {
+        u'bldcontrol.brbitbake': {
+            'Meta': {'object_name': 'BRBitbake'},
+            'commit': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
+            'dirpath': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
+            'giturl': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']", 'unique': 'True'})
+        },
+        u'bldcontrol.brerror': {
+            'Meta': {'object_name': 'BRError'},
+            'errmsg': ('django.db.models.fields.TextField', [], {}),
+            'errtype': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}),
+            'traceback': ('django.db.models.fields.TextField', [], {})
+        },
+        u'bldcontrol.brlayer': {
+            'Meta': {'object_name': 'BRLayer'},
+            'commit': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
+            'dirpath': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
+            'giturl': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"})
+        },
+        u'bldcontrol.brtarget': {
+            'Meta': {'object_name': 'BRTarget'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}),
+            'target': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'task': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True'})
+        },
+        u'bldcontrol.brvariable': {
+            'Meta': {'object_name': 'BRVariable'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}),
+            'value': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        u'bldcontrol.buildenvironment': {
+            'Meta': {'object_name': 'BuildEnvironment'},
+            'address': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
+            'bbaddress': ('django.db.models.fields.CharField', [], {'max_length': '254', 'blank': 'True'}),
+            'bbport': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
+            'bbstate': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'bbtoken': ('django.db.models.fields.CharField', [], {'max_length': '126', 'blank': 'True'}),
+            'betype': ('django.db.models.fields.IntegerField', [], {}),
+            'builddir': ('django.db.models.fields.CharField', [], {'max_length': '512', 'blank': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'lock': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'sourcedir': ('django.db.models.fields.CharField', [], {'max_length': '512', 'blank': 'True'}),
+            'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
+        },
+        u'bldcontrol.buildrequest': {
+            'Meta': {'object_name': 'BuildRequest'},
+            'build': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Build']", 'null': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'project': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Project']"}),
+            'state': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
+        },
+        u'orm.bitbakeversion': {
+            'Meta': {'object_name': 'BitbakeVersion'},
+            'branch': ('django.db.models.fields.CharField', [], {'max_length': '32'}),
+            'dirpath': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'giturl': ('django.db.models.fields.URLField', [], {'max_length': '200'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '32'})
+        },
+        u'orm.build': {
+            'Meta': {'object_name': 'Build'},
+            'bitbake_version': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+            'build_name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'completed_on': ('django.db.models.fields.DateTimeField', [], {}),
+            'cooker_log_path': ('django.db.models.fields.CharField', [], {'max_length': '500'}),
+            'distro': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'distro_version': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'errors_no': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'machine': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'outcome': ('django.db.models.fields.IntegerField', [], {'default': '2'}),
+            'project': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Project']", 'null': 'True'}),
+            'started_on': ('django.db.models.fields.DateTimeField', [], {}),
+            'timespent': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'warnings_no': ('django.db.models.fields.IntegerField', [], {'default': '0'})
+        },
+        u'orm.project': {
+            'Meta': {'object_name': 'Project'},
+            'bitbake_version': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.BitbakeVersion']"}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'short_description': ('django.db.models.fields.CharField', [], {'max_length': '50', 'blank': 'True'}),
+            'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'user_id': ('django.db.models.fields.IntegerField', [], {'null': 'True'})
+        }
+    }
+
+    complete_apps = ['bldcontrol']

bitbake/lib/toaster/bldcontrol/models.py

@@ -0,0 +1,98 @@
+from django.db import models
+from django.core.validators import MaxValueValidator, MinValueValidator
+from orm.models import Project, ProjectLayer, ProjectVariable, ProjectTarget, Build
+
+# a BuildEnvironment is the equivalent of the "build/" directory on the localhost
+class BuildEnvironment(models.Model):
+    SERVER_STOPPED = 0
+    SERVER_STARTED = 1
+    SERVER_STATE = (
+        (SERVER_STOPPED, "stopped"),
+        (SERVER_STARTED, "started"),
+    )
+
+    TYPE_LOCAL = 0
+    TYPE_SSH   = 1
+    TYPE = (
+        (TYPE_LOCAL, "local"),
+        (TYPE_SSH, "ssh"),
+    )
+
+    LOCK_FREE = 0
+    LOCK_LOCK = 1
+    LOCK_RUNNING = 2
+    LOCK_STATE = (
+        (LOCK_FREE, "free"),
+        (LOCK_LOCK, "lock"),
+        (LOCK_RUNNING, "running"),
+    )
+
+    address     = models.CharField(max_length = 254)
+    betype      = models.IntegerField(choices = TYPE)
+    bbaddress   = models.CharField(max_length = 254, blank = True)
+    bbport      = models.IntegerField(default = -1)
+    bbtoken     = models.CharField(max_length = 126, blank = True)
+    bbstate     = models.IntegerField(choices = SERVER_STATE, default = SERVER_STOPPED)
+    sourcedir   = models.CharField(max_length = 512, blank = True)
+    builddir    = models.CharField(max_length = 512, blank = True)
+    lock        = models.IntegerField(choices = LOCK_STATE, default = LOCK_FREE)
+    created     = models.DateTimeField(auto_now_add = True)
+    updated     = models.DateTimeField(auto_now = True)
+
+
+# a BuildRequest is a request that the scheduler will build using a BuildEnvironment
+# the build request queue is the table itself, ordered by state
+
+class BuildRequest(models.Model):
+    REQ_CREATED = 0
+    REQ_QUEUED = 1
+    REQ_INPROGRESS = 2
+    REQ_COMPLETED = 3
+    REQ_FAILED = 4
+
+    REQUEST_STATE = (
+        (REQ_CREATED, "created"),
+        (REQ_QUEUED, "queued"),
+        (REQ_INPROGRESS, "in progress"),
+        (REQ_COMPLETED, "completed"),
+        (REQ_FAILED, "failed"),
+    )
+
+    project     = models.ForeignKey(Project)
+    build       = models.ForeignKey(Build, null = True)     # TODO: toasterui should set this when Build is created
+    state       = models.IntegerField(choices = REQUEST_STATE, default = REQ_CREATED)
+    created     = models.DateTimeField(auto_now_add = True)
+    updated     = models.DateTimeField(auto_now = True)
+
+
+# These tables specify the settings for running an actual build.
+# They MUST be kept in sync with the tables in orm.models.Project*
+
+class BRLayer(models.Model):
+    req         = models.ForeignKey(BuildRequest)
+    name        = models.CharField(max_length = 100)
+    giturl      = models.CharField(max_length = 254)
+    commit      = models.CharField(max_length = 254)
+    dirpath     = models.CharField(max_length = 254)
+
+class BRBitbake(models.Model):
+    req         = models.ForeignKey(BuildRequest, unique = True)    # only one bitbake for a request
+    giturl      = models.CharField(max_length =254)
+    commit      = models.CharField(max_length = 254)
+    dirpath     = models.CharField(max_length = 254)
+
+class BRVariable(models.Model):
+    req         = models.ForeignKey(BuildRequest)
+    name        = models.CharField(max_length=100)
+    value       = models.TextField(blank = True)
+
+class BRTarget(models.Model):
+    req         = models.ForeignKey(BuildRequest)
+    target      = models.CharField(max_length=100)
+    task        = models.CharField(max_length=100, null=True)
+
+class BRError(models.Model):
+    req         = models.ForeignKey(BuildRequest)
+    errtype     = models.CharField(max_length=100)
+    errmsg      = models.TextField()
+    traceback   = models.TextField()

bitbake/lib/toaster/bldcontrol/sshbecontroller.py

@@ -0,0 +1,193 @@
+#
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2014        Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import sys
+import re
+from django.db import transaction
+from django.db.models import Q
+from bldcontrol.models import BuildEnvironment, BRLayer, BRVariable, BRTarget, BRBitbake
+import subprocess
+
+from toastermain import settings
+
+from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException, _getgitcheckoutdirectoryname
+
+def DN(path):
+    return "/".join(path.split("/")[0:-1])
+
+class SSHBEController(BuildEnvironmentController):
+    """ Implementation of the BuildEnvironmentController for the localhost;
+        this controller manages the default build directory,
+        the server setup and system start and stop for the localhost-type build environment
+
+    """
+
+    def __init__(self, be):
+        super(SSHBEController, self).__init__(be)
+        self.dburl = settings.getDATABASE_URL()
+        self.pokydirname = None
+        self.islayerset = False
+
+    def _shellcmd(self, command, cwd = None):
+        if cwd is None:
+            cwd = self.be.sourcedir
+
+        p = subprocess.Popen("ssh %s 'cd %s && %s'" % (self.be.address, cwd, command), stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+        (out,err) = p.communicate()
+        if p.returncode:
+            if len(err) == 0:
+                err = "command: %s \n%s" % (command, out)
+            else:
+                err = "command: %s \n%s" % (command, err)
+            raise ShellCmdException(err)
+        else:
+            return out.strip()
+
+    def _pathexists(self, path):
+        try:
+            self._shellcmd("test -e \"%s\"" % path)
+            return True
+        except ShellCmdException as e:
+            return False
+
+    def _pathcreate(self, path):
+        self._shellcmd("mkdir -p \"%s\"" % path)
+
+    def _setupBE(self):
+        assert self.pokydirname and self._pathexists(self.pokydirname)
+        self._pathcreate(self.be.builddir)
+        self._shellcmd("bash -c \"source %s/oe-init-build-env %s\"" % (self.pokydirname, self.be.builddir))
+
+    def startBBServer(self):
+        assert self.pokydirname and self._pathexists(self.pokydirname)
+        assert self.islayerset
+        print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb && sleep 1\"" % (self.pokydirname, self.be.builddir, self.dburl))
+        # FIXME unfortunate sleep 1 - we need to make sure that bbserver is started and the toaster ui is connected
+        # but since they start async without any return, we just wait a bit
+        print "Started server"
+        assert self.be.sourcedir and self._pathexists(self.be.builddir)
+        self.be.bbaddress = self.be.address.split("@")[-1]
+        self.be.bbport = "8200"
+        self.be.bbstate = BuildEnvironment.SERVER_STARTED
+        self.be.save()
+
+    def stopBBServer(self):
+        assert self.pokydirname and self._pathexists(self.pokydirname)
+        assert self.islayerset
+        print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && %s source toaster stop\"" %
+            (self.pokydirname, self.be.builddir, (lambda: "" if self.be.bbtoken is None else "BBTOKEN=%s" % self.be.bbtoken)()))
+        self.be.bbstate = BuildEnvironment.SERVER_STOPPED
+        self.be.save()
+        print "Stopped server"
+
+    def setLayers(self, bitbakes, layers):
+        """ a word of attention: by convention, the first layer for any build will be poky! """
+
+        assert self.be.sourcedir is not None
+        assert len(bitbakes) == 1
+        # set layers in the layersource
+
+        # 1. get a list of repos, and map dirpaths for each layer
+        gitrepos = {}
+        gitrepos[bitbakes[0].giturl] = []
+        gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) )
+        
+        for layer in layers:
+            # we don't process local URLs
+            if layer.giturl.startswith("file://"):
+                continue
+            if not layer.giturl in gitrepos:
+                gitrepos[layer.giturl] = []
+            gitrepos[layer.giturl].append( (layer.name, layer.dirpath, layer.commit))
+        for giturl in gitrepos.keys():
+            commitid = gitrepos[giturl][0][2]
+            for e in gitrepos[giturl]:
+                if commitid != e[2]:
+                    raise BuildSetupException("More than one commit per git url, unsupported configuration")
+
+        layerlist = []
+
+        # 2. checkout the repositories
+        for giturl in gitrepos.keys():
+            import os
+            localdirname = os.path.join(self.be.sourcedir, _getgitcheckoutdirectoryname(giturl))
+            print "DEBUG: giturl ", giturl ,"checking out in current directory", localdirname
+
+            # make sure our directory is a git repository
+            if self._pathexists(localdirname):
+                if not giturl in self._shellcmd("git remote -v", localdirname):
+                    raise BuildSetupException("Existing git repository at %s, but with different remotes (not '%s'). Aborting." % (localdirname, giturl))
+            else:
+                self._shellcmd("git clone \"%s\" \"%s\"" % (giturl, localdirname))
+            # checkout the needed commit
+            commit = gitrepos[giturl][0][2]
+
+            # branch magic name "HEAD" will inhibit checkout
+            if commit != "HEAD":
+                print "DEBUG: checking out commit ", commit, "to", localdirname
+                self._shellcmd("git fetch --all && git checkout \"%s\"" % commit , localdirname)
+
+            # take the localdirname as poky dir if we can find the oe-init-build-env
+            if self.pokydirname is None and self._pathexists(os.path.join(localdirname, "oe-init-build-env")):
+                print "DEBUG: selected poky dir name", localdirname
+                self.pokydirname = localdirname
+
+            # verify our repositories
+            for name, dirpath, commit in gitrepos[giturl]:
+                localdirpath = os.path.join(localdirname, dirpath)
+                if not self._pathexists(localdirpath):
+                    raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit))
+
+                if name != "bitbake":
+                    layerlist.append(localdirpath)
+
+        print "DEBUG: current layer list ", layerlist
+
+        # 3. configure the build environment, so we have a conf/bblayers.conf
+        assert self.pokydirname is not None
+        self._setupBE()
+
+        # 4. update the bblayers.conf
+        bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf")
+        if not self._pathexists(bblayerconf):
+            raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf)
+
+        conflines = open(bblayerconf, "r").readlines()
+
+        bblayerconffile = open(bblayerconf, "w")
+        for i in xrange(len(conflines)):
+            if conflines[i].startswith("# line added by toaster"):
+                i += 2
+            else:
+                bblayerconffile.write(conflines[i])
+
+        bblayerconffile.write("\n# line added by toaster build control\nBBLAYERS = \"" + " ".join(layerlist) + "\"")
+        bblayerconffile.close()
+
+        self.islayerset = True
+        return True
+
+    def release(self):
+        assert self.be.sourcedir and self._pathexists(self.be.builddir)
+        import shutil
+        shutil.rmtree(os.path.join(self.be.sourcedir, "build"))
+        assert not self._pathexists(self.be.builddir)

bitbake/lib/toaster/bldcontrol/tests.py

@@ -0,0 +1,149 @@
+"""
+This file demonstrates writing tests using the unittest module. These will pass
+when you run "manage.py test".
+
+Replace this with more appropriate tests for your application.
+"""
+
+from django.test import TestCase
+
+from bldcontrol.bbcontroller import BitbakeController
+from bldcontrol.localhostbecontroller import LocalhostBEController
+from bldcontrol.sshbecontroller import SSHBEController
+from bldcontrol.models import BuildEnvironment, BuildRequest
+from bldcontrol.management.commands.runbuilds import Command
+
+import socket
+import subprocess
+
+# standard poky data hardcoded for testing
+BITBAKE_LAYERS = [type('bitbake_info', (object,), { "giturl": "git://git.yoctoproject.org/poky.git", "dirpath": "", "commit": "HEAD"})]
+POKY_LAYERS = [
+    type('poky_info', (object,), { "name": "meta", "giturl": "git://git.yoctoproject.org/poky.git", "dirpath": "meta", "commit": "HEAD"}),
+    type('poky_info', (object,), { "name": "meta-yocto", "giturl": "git://git.yoctoproject.org/poky.git", "dirpath": "meta-yocto", "commit": "HEAD"}),
+    type('poky_info', (object,), { "name": "meta-yocto-bsp", "giturl": "git://git.yoctoproject.org/poky.git", "dirpath": "meta-yocto-bsp", "commit": "HEAD"}),
+    ]
+
+
+
+# we have an abstract test class designed to ensure that the controllers use a single interface
+# specific controller tests only need to override the _getBuildEnvironment() method
+
+class BEControllerTests(object):
+
+    def _serverForceStop(self, bc):
+        err = bc._shellcmd("netstat  -tapn 2>/dev/null | grep 8200 | awk '{print $7}' | sort -fu | cut -d \"/\" -f 1 | grep -v -- - | tee /dev/fd/2 | xargs -r kill")
+        self.assertTrue(err == '', "bitbake server pid %s not stopped" % err)
+
+    def test_serverStartAndStop(self):
+        obe =  self._getBuildEnvironment()
+        bc = self._getBEController(obe)
+        bc.setLayers(BITBAKE_LAYERS, POKY_LAYERS) # setting layers, skip any layer info
+
+        hostname = self.test_address.split("@")[-1]
+
+        # test start server and stop
+        self.assertTrue(socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex((hostname, 8200)), "Port already occupied")
+        bc.startBBServer()
+        self.assertFalse(socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex((hostname, 8200)), "Server not answering")
+
+        bc.stopBBServer()
+        self.assertTrue(socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex((hostname, 8200)), "Server not stopped")
+
+        self._serverForceStop(bc)
+
+    def test_getBBController(self):
+        obe = self._getBuildEnvironment()
+        bc = self._getBEController(obe)
+        bc.setLayers(BITBAKE_LAYERS, POKY_LAYERS) # setting layers, skip any layer info
+
+        bbc = bc.getBBController()
+        self.assertTrue(isinstance(bbc, BitbakeController))
+        # test set variable, use no build marker -1 for BR value
+        try:
+            bbc.setVariable("TOASTER_BRBE", "%d:%d" % (-1, obe.pk))
+        except Exception as e :
+            self.fail("setVariable raised %s", e)
+
+        bc.stopBBServer()
+
+        self._serverForceStop(bc)
+
+class LocalhostBEControllerTests(TestCase, BEControllerTests):
+    def __init__(self, *args):
+        super(LocalhostBEControllerTests, self).__init__(*args)
+        # hardcoded for Alex's machine; since the localhost BE is machine-dependent,
+        # I found no good way to abstractize this
+        self.test_sourcedir = "/home/ddalex/ssd/yocto"
+        self.test_builddir = "/home/ddalex/ssd/yocto/build"
+        self.test_address = "localhost"
+
+    def _getBuildEnvironment(self):
+        return BuildEnvironment.objects.create(
+                lock = BuildEnvironment.LOCK_FREE,
+                betype = BuildEnvironment.TYPE_LOCAL,
+                address = self.test_address,
+                sourcedir = self.test_sourcedir,
+                builddir = self.test_builddir )
+
+    def _getBEController(self, obe):
+        return LocalhostBEController(obe)
+
+class SSHBEControllerTests(TestCase, BEControllerTests):
+    def __init__(self, *args):
+        super(SSHBEControllerTests, self).__init__(*args)
+        self.test_address = "ddalex-desktop.local"
+        # hardcoded for ddalex-desktop.local machine; since the localhost BE is machine-dependent,
+        # I found no good way to abstractize this
+        self.test_sourcedir = "/home/ddalex/ssd/yocto"
+        self.test_builddir = "/home/ddalex/ssd/yocto/build"
+
+    def _getBuildEnvironment(self):
+        return BuildEnvironment.objects.create(
+                lock = BuildEnvironment.LOCK_FREE,
+                betype = BuildEnvironment.TYPE_SSH,
+                address = self.test_address,
+                sourcedir = self.test_sourcedir,
+                builddir = self.test_builddir )
+
+    def _getBEController(self, obe):
+        return SSHBEController(obe)
+
+    def test_pathExists(self):
+        obe = BuildEnvironment.objects.create(betype = BuildEnvironment.TYPE_SSH, address= self.test_address)
+        sbc = SSHBEController(obe)
+        self.assertTrue(sbc._pathexists("/"))
+        self.assertFalse(sbc._pathexists("/.deadbeef"))
+        self.assertTrue(sbc._pathexists(sbc._shellcmd("pwd")))
+
+
+class RunBuildsCommandTests(TestCase):
+    def test_bec_select(self):
+        """
+        Tests that we can find and lock a build environment
+        """
+
+        obe = BuildEnvironment.objects.create(lock = BuildEnvironment.LOCK_FREE, betype = BuildEnvironment.TYPE_LOCAL)
+        command = Command()
+        bec = command._selectBuildEnvironment()
+
+        # make sure we select the object we've just built
+        self.assertTrue(bec.be.id == obe.id, "Environment is not properly selected")
+        # we have a locked environment
+        self.assertTrue(bec.be.lock == BuildEnvironment.LOCK_LOCK, "Environment is not locked")
+        # no more selections possible here
+        self.assertRaises(IndexError, command._selectBuildEnvironment)
+
+    def test_br_select(self):
+        from orm.models import Project, Release, BitbakeVersion
+        p = Project.objects.create_project("test", Release.objects.get_or_create(name = "HEAD", bitbake_version = BitbakeVersion.objects.get_or_create(name="HEAD", branch="HEAD")[0])[0])
+        obr = BuildRequest.objects.create(state = BuildRequest.REQ_QUEUED, project = p)
+        command = Command()
+        br = command._selectBuildRequest()
+
+        # make sure we select the object we've just built
+        self.assertTrue(obr.id == br.id, "Request is not properly selected")
+        # we have a locked environment
+        self.assertTrue(br.state == BuildRequest.REQ_INPROGRESS, "Request is not updated")
+        # no more selections possible here
+        self.assertRaises(IndexError, command._selectBuildRequest)

bitbake/lib/toaster/bldcontrol/views.py

@@ -0,0 +1 @@
+# Create your views here.

bitbake/lib/toaster/bldviewer/static/css/bootstrap.css

@@ -4794,4 +4794,4 @@ td.visible-print {
   td.hidden-print {
     display: none !important;
   }
-}
+}

bitbake/lib/toaster/bldviewer/templates/simple_build.html

@@ -6,7 +6,7 @@
 
 {% block pagetable %}
 
-    {% load projecttags %}
+    {% load simple_projecttags %}
         <tr>
             <th>Outcome</th>
             <th>Started On</th>

bitbake/lib/toaster/bldviewer/templates/simple_layer.html

@@ -5,7 +5,7 @@
 {% endblock %}
 
 {% block pagetable %}
-    {% load projecttags %}
+    {% load simple_projecttags %}
 
         <tr>
             <th>Name</th>

bitbake/lib/toaster/bldviewer/templates/simple_recipe.html

@@ -8,7 +8,7 @@
 {% endblock %}
 
 {% block pagetable %}
-    {% load projecttags %}
+    {% load simple_projecttags %}
 
         <tr>
         </tr>

bitbake/lib/toaster/orm/admin.py

@@ -0,0 +1,17 @@
+from django.contrib import admin
+from django.contrib.admin.filters import RelatedFieldListFilter
+from .models import Branch, LayerSource, ToasterSetting
+
+class LayerSourceAdmin(admin.ModelAdmin):
+    pass
+
+class BranchAdmin(admin.ModelAdmin):
+    pass
+
+class ToasterSettingAdmin(admin.ModelAdmin):
+    pass
+
+admin.site.register(LayerSource, LayerSourceAdmin)
+admin.site.register(Branch, BranchAdmin)
+admin.site.register(ToasterSetting, ToasterSettingAdmin)
+

bitbake/lib/toaster/orm/management/commands/lsupdates.py

@@ -0,0 +1,12 @@
+from django.core.management.base import NoArgsCommand, CommandError
+from orm.models import LayerSource
+import os
+
+class Command(NoArgsCommand):
+    args    = ""
+    help    = "Updates locally cached information from all LayerSources"
+
+
+    def handle_noargs(self, **options):
+        for ls in LayerSource.objects.all():
+            ls.update()