libpam: Fix patch broken during upgrade

"0x200" became "0200" during the upgrade to libpam 1.2.1 in:
http://git.yoctoproject.org/cgit/cgit.cgi/poky/commit/?id=88dd997d9941b63ae9eead6690ecf2b785c0740c
and this broke the IMAGE_FEATURES like debug-tweaks.

I've converted all the values to octal here to match the original
header file convention and make it clearer.

[YOCTO #8033]

(From OE-Core rev: 588e19058f631a1cc78002e1969a5459cd626afb)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

.gitignore

@@ -21,3 +21,6 @@ documentation/user-manual/user-manual.html
 documentation/user-manual/user-manual.pdf
 documentation/user-manual/user-manual.tgz
 pull-*/
+bitbake/lib/toaster/contrib/tts/backlog.txt
+bitbake/lib/toaster/contrib/tts/log/*
+bitbake/lib/toaster/contrib/tts/.cache/*

README.hardware

@@ -352,19 +352,22 @@ Setup instructions
 ------------------
 
 You will need the following:
-* NFS root setup on your workstation
-* TFTP server installed on your workstation
 * RJ45 -> serial ("rollover") cable connected from your PC to the CONSOLE
-  port on the board
+  port on the device
 * Ethernet connected to the first ethernet port on the board
 
+If using NFS as part of the setup process, you will also need:
+* NFS root setup on your workstation
+* TFTP server installed on your workstation (if fetching the kernel from
+  TFTP, see below).
+
 --- Preparation ---
 
 Build an image (e.g. core-image-minimal) using "edgerouter" as the MACHINE.
 In the following instruction it is based on core-image-minimal. Another target
 may be similiar with it.
 
---- Booting from NFS root ---
+--- Booting from NFS root / kernel via TFTP ---
 
 Load the kernel, and boot the system as follows:
 

bitbake/bin/bitbake

@@ -33,17 +33,21 @@ except RuntimeError as exc:
     sys.exit(str(exc))
 
 from bb import cookerdata
-from bb.main import bitbake_main, BitBakeConfigParameters
+from bb.main import bitbake_main, BitBakeConfigParameters, BBMainException
+
+__version__ = "1.27.1"
 
 if __name__ == "__main__":
+    if __version__ != bb.__version__:
+        sys.exit("Bitbake core version and program version mismatch!")
     try:
-        ret = bitbake_main(BitBakeConfigParameters(sys.argv),
-                           cookerdata.CookerConfiguration())
+        sys.exit(bitbake_main(BitBakeConfigParameters(sys.argv),
+                              cookerdata.CookerConfiguration()))
+    except BBMainException as err:
+        sys.exit(err)
     except bb.BBHandledException:
-        ret = 1
+        sys.exit(1)
     except Exception:
-        ret = 1
         import traceback
         traceback.print_exc()
-    sys.exit(ret)
-
+        sys.exit(1)

bitbake/bin/bitbake-diffsigs

@@ -46,6 +46,12 @@ logger = logger_create('bitbake-diffsigs')
 def find_compare_task(bbhandler, pn, taskname):
     """ Find the most recent signature files for the specified PN/task and compare them """
 
+    def get_hashval(siginfo):
+        if siginfo.endswith('.siginfo'):
+            return siginfo.rpartition(':')[2].partition('_')[0]
+        else:
+            return siginfo.rpartition('.')[2]
+
     if not hasattr(bb.siggen, 'find_siginfo'):
         logger.error('Metadata does not support finding signature data files')
         sys.exit(1)
@@ -54,7 +60,7 @@ def find_compare_task(bbhandler, pn, taskname):
         taskname = 'do_%s' % taskname
 
     filedates = bb.siggen.find_siginfo(pn, taskname, None, bbhandler.config_data)
-    latestfiles = sorted(filedates.keys(), key=lambda f: filedates[f])[-2:]
+    latestfiles = sorted(filedates.keys(), key=lambda f: filedates[f])[-3:]
     if not latestfiles:
         logger.error('No sigdata files found matching %s %s' % (pn, taskname))
         sys.exit(1)
@@ -62,6 +68,16 @@ def find_compare_task(bbhandler, pn, taskname):
         logger.error('Only one matching sigdata file found for the specified task (%s %s)' % (pn, taskname))
         sys.exit(1)
     else:
+        # It's possible that latestfiles contain 3 elements and the first two have the same hash value.
+        # In this case, we delete the second element.
+        # The above case is actually the most common one. Because we may have sigdata file and siginfo
+        # file having the same hash value. Comparing such two files makes no sense.
+        if len(latestfiles) == 3:
+            hash0 = get_hashval(latestfiles[0])
+            hash1 = get_hashval(latestfiles[1])
+            if hash0 == hash1:
+                latestfiles.pop(1)
+
         # Define recursion callback
         def recursecb(key, hash1, hash2):
             hashes = [hash1, hash2]

bitbake/bin/bitbake-layers

@@ -501,7 +501,13 @@ skipped recipes will also be listed, with a " (skipped)" suffix.
 
             if len(allproviders[p]) > 1 or not show_multi_provider_only:
                 pref = preferred_versions[p]
-                preffile = bb.cache.Cache.virtualfn2realfn(pref[1])[0]
+                realfn = bb.cache.Cache.virtualfn2realfn(pref[1])
+                preffile = realfn[0]
+                # We only display once per recipe, we should prefer non extended versions of the
+                # recipe if present (so e.g. in OpenEmbedded, openssl rather than nativesdk-openssl
+                # which would otherwise sort first).
+                if realfn[1] and realfn[0] in self.bbhandler.cooker.recipecache.pkg_fn:
+                    continue
                 if preffile not in preffiles:
                     preflayer = self.get_file_layer(preffile)
                     multilayer = False

bitbake/bin/bitbake-selftest

@@ -26,24 +26,30 @@ except RuntimeError as exc:
     sys.exit(str(exc))
 
 def usage():
-    print('usage: %s [testname1 [testname2]...]' % os.path.basename(sys.argv[0]))
+    print('usage: [BB_SKIP_NETTESTS=yes] %s [-v] [testname1 [testname2]...]' % os.path.basename(sys.argv[0]))
 
-if len(sys.argv) > 1:
+verbosity = 1
+
+tests = sys.argv[1:]
+if '-v' in sys.argv:
+    tests.remove('-v')
+    verbosity = 2
+
+if tests:
     if '--help' in sys.argv[1:]:
         usage()
         sys.exit(0)
-
-    tests = sys.argv[1:]
 else:
     tests = ["bb.tests.codeparser",
              "bb.tests.cow",
              "bb.tests.data",
              "bb.tests.fetch",
+             "bb.tests.parse",
              "bb.tests.utils"]
 
 for t in tests:
     t = '.'.join(t.split('.')[:3])
     __import__(t)
 
-unittest.main(argv=["bitbake-selftest"] + tests)
+unittest.main(argv=["bitbake-selftest"] + tests, verbosity=verbosity)
 

bitbake/bin/bitbake-worker

@@ -24,6 +24,15 @@ if sys.argv[1] == "decafbadbad":
     except:
         import profile
 
+# Unbuffer stdout to avoid log truncation in the event
+# of an unorderly exit as well as to provide timely
+# updates to log files for use with tail
+try:
+    if sys.stdout.name == '<stdout>':
+        sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
+except:
+    pass
+
 logger = logging.getLogger("BitBake")
 
 try:

bitbake/bin/image-writer

@@ -34,7 +34,7 @@ from bb.ui.crumbs.hig.deployimagedialog import DeployImageDialog
 from bb.ui.crumbs.hig.imageselectiondialog import ImageSelectionDialog
 
 # I put all the fs bitbake supported here. Need more test.
-DEPLOYABLE_IMAGE_TYPES = ["jffs2", "cramfs", "ext2", "ext3", "btrfs", "squashfs", "ubi", "vmdk"]
+DEPLOYABLE_IMAGE_TYPES = ["jffs2", "cramfs", "ext2", "ext3", "ext4", "btrfs", "squashfs", "ubi", "vmdk"]
 Title = "USB Image Writer"
 
 class DeployWindow(gtk.Window):

bitbake/bin/toaster

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/sh
 # (c) 2013 Intel Corp.
 
 # This program is free software; you can redistribute it and/or modify
@@ -28,79 +28,84 @@
 
 # Helper function to kill a background toaster development server
 
-function webserverKillAll()
+webserverKillAll()
 {
-	local pidfile
-	for pidfile in ${BUILDDIR}/.toastermain.pid; do
-		if [ -f ${pidfile} ]; then
-		while kill -0 $(< ${pidfile}) 2>/dev/null; do
-			kill -SIGTERM -$(< ${pidfile}) 2>/dev/null
-			sleep 1;
-			# Kill processes if they are still running - may happen in interactive shells
-			ps fux | grep "python.*manage.py runserver" | awk '{print $2}' | xargs kill
-		done;
-		rm  ${pidfile}
-		fi
-	done
+    local pidfile
+    for pidfile in ${BUILDDIR}/.toastermain.pid; do
+        if [ -f ${pidfile} ]; then
+            pid=`cat ${pidfile}`
+            while kill -0 $pid 2>/dev/null; do
+                kill -SIGTERM -$pid 2>/dev/null
+                sleep 1
+                # Kill processes if they are still running - may happen in interactive shells
+                ps fux | grep "python.*manage.py runserver" | awk '{print $2}' | xargs kill
+            done
+            rm  ${pidfile}
+        fi
+    done
 }
 
-function webserverStartAll()
+webserverStartAll()
 {
-	# do not start if toastermain points to a valid process
-	if ! cat "${BUILDDIR}/.toastermain.pid" 2>/dev/null | xargs -I{} kill -0 {} ; then
+    # do not start if toastermain points to a valid process
+    if ! cat "${BUILDDIR}/.toastermain.pid" 2>/dev/null | xargs -I{} kill -0 {} ; then
+        retval=1
+        rm "${BUILDDIR}/.toastermain.pid"
+    fi
+
+    retval=0
+    if [ "$TOASTER_MANAGED" '=' '1' ]; then
+        python $BBBASEDIR/lib/toaster/manage.py syncdb || retval=1
+    else
+        python $BBBASEDIR/lib/toaster/manage.py syncdb --noinput || retval=1
+    fi
+    python $BBBASEDIR/lib/toaster/manage.py migrate orm || retval=2
+    if [ $retval -eq 1 ]; then
+        echo "Failed db sync, stopping system start" 1>&2
+    elif [ $retval -eq 2 ]; then
+        printf "\nError on migration, trying to recover... \n"
+        python $BBBASEDIR/lib/toaster/manage.py migrate orm 0001_initial --fake
+        retval=0
+        python $BBBASEDIR/lib/toaster/manage.py migrate orm || retval=1
+    fi
+    if [ "$TOASTER_MANAGED" = '1' ]; then
+        python $BBBASEDIR/lib/toaster/manage.py migrate bldcontrol || retval=1
+        python $BBBASEDIR/lib/toaster/manage.py checksettings  --traceback || retval=1
+    fi
+    if [ $retval -eq 0 ]; then
+        echo "Starting webserver..."
+        python $BBBASEDIR/lib/toaster/manage.py runserver "0.0.0.0:$WEB_PORT" </dev/null >>${BUILDDIR}/toaster_web.log 2>&1 & echo $! >${BUILDDIR}/.toastermain.pid
+        sleep 1
+        if ! cat "${BUILDDIR}/.toastermain.pid" | xargs -I{} kill -0 {} ; then
             retval=1
             rm "${BUILDDIR}/.toastermain.pid"
+        else
+            echo "Webserver address:  http://0.0.0.0:$WEB_PORT/"
         fi
-
-        retval=0
-        python $BBBASEDIR/lib/toaster/manage.py syncdb || retval=1
-        python $BBBASEDIR/lib/toaster/manage.py migrate orm || retval=2
-        if [ $retval -eq 1 ]; then
-                echo "Failed db sync, stopping system start" 1>&2
-        elif [ $retval -eq 2 ]; then
-            echo -e "\nError on migration, trying to recover... \n"
-            python $BBBASEDIR/lib/toaster/manage.py migrate orm 0001_initial --fake
-            retval=0
-            python $BBBASEDIR/lib/toaster/manage.py migrate orm || retval=1
-        fi
-	if [ "x$TOASTER_MANAGED" == "x1" ]; then
-	        python $BBBASEDIR/lib/toaster/manage.py migrate bldcontrol || retval=1
-		python $BBBASEDIR/lib/toaster/manage.py checksettings  --traceback || retval=1
-	fi
-        if [ $retval -eq 0 ]; then
-	    echo "Starting webserver..."
-            python $BBBASEDIR/lib/toaster/manage.py runserver "0.0.0.0:$WEB_PORT" </dev/null >>${BUILDDIR}/toaster_web.log 2>&1 & echo $! >${BUILDDIR}/.toastermain.pid
-            sleep 1
-            if ! cat "${BUILDDIR}/.toastermain.pid" | xargs -I{} kill -0 {} ; then
-                retval=1
-                rm "${BUILDDIR}/.toastermain.pid"
-	    else
-		echo "Webserver address:  http://0.0.0.0:$WEB_PORT/"
-            fi
-        fi
-        return $retval
+    fi
+    return $retval
 }
 
 # Helper functions to add a special configuration file
 
-function addtoConfiguration()
+addtoConfiguration()
 {
-        file=$1
-        shift
-        echo "#Created by toaster start script" > ${BUILDDIR}/conf/$file
-        for var in "$@"; do echo $var >> ${BUILDDIR}/conf/$file; done
+    file=$1
+    shift
+    echo "#Created by toaster start script" > ${BUILDDIR}/conf/$file
+    for var in "$@"; do echo $var >> ${BUILDDIR}/conf/$file; done
 }
 
 INSTOPSYSTEM=0
 
 # define the stop command
-function stop_system()
+stop_system()
 {
     # prevent reentry
-    if [ $INSTOPSYSTEM == 1 ]; then return; fi
+    if [ $INSTOPSYSTEM -eq 1 ]; then return; fi
     INSTOPSYSTEM=1
     if [ -f ${BUILDDIR}/.toasterui.pid ]; then
-        kill $(< ${BUILDDIR}/.toasterui.pid ) 2>/dev/null
+        kill `cat ${BUILDDIR}/.toasterui.pid` 2>/dev/null
         rm ${BUILDDIR}/.toasterui.pid
     fi
     BBSERVER=0.0.0.0:-1 bitbake -m
@@ -113,29 +118,29 @@ function stop_system()
     INSTOPSYSTEM=0
 }
 
-function check_pidbyfile() {
-    [ -e $1 ] && kill -0 $(< $1) 2>/dev/null
+check_pidbyfile() {
+    [ -e $1 ] && kill -0 `cat $1` 2>/dev/null
 }
 
 
-function notify_chldexit() {
-    if [ $NOTOASTERUI == 0 ]; then
+notify_chldexit() {
+    if [ $NOTOASTERUI -eq 0 ]; then
         check_pidbyfile ${BUILDDIR}/.toasterui.pid && return
         stop_system
     fi
 }
 
 
-function verify_prereq() {
-	# Verify prerequisites
+verify_prereq() {
+    # Verify prerequisites
 
     if ! echo "import django; print (1,) == django.VERSION[0:1] and django.VERSION[1:2][0] in (6,)" | python 2>/dev/null | grep True >/dev/null; then
-        echo -e "This program needs Django 1.6. Please install with\n\npip install django==1.6\n"
+        printf "This program needs Django 1.6. Please install with\n\npip install django==1.6\n"
         return 2
     fi
 
     if ! echo "import south; print reduce(lambda x, y: 2 if x==2 else 0 if x == 0 else y, map(lambda x: 1+cmp(x[1]-x[0],0), zip([0,8,4], map(int,south.__version__.split(\".\"))))) > 0" | python 2>/dev/null | grep True >/dev/null; then
-        echo -e "This program needs South 0.8.4. Please install with\n\npip install south==0.8.4\n"
+        printf "This program needs South 0.8.4. Please install with\n\npip install south==0.8.4\n"
         return 2
     fi
     return 0
@@ -143,14 +148,23 @@ function verify_prereq() {
 
 
 # read command line parameters
+if [ -n "$BASH_SOURCE" ] ; then
+    TOASTER=${BASH_SOURCE}
+elif [ -n "$ZSH_NAME" ] ; then
+    TOASTER=${(%):-%x}
+else
+    TOASTER=$0
+fi
+
+BBBASEDIR=`dirname $TOASTER`/..
 
-BBBASEDIR=`dirname ${BASH_SOURCE}`/..
 RUNNING=0
 
 NOTOASTERUI=0
 WEBSERVER=1
 TOASTER_BRBE=""
 WEB_PORT="8000"
+NOBROWSER=0
 
 for param in $*; do
     case $param in
@@ -160,6 +174,9 @@ for param in $*; do
     noweb )
             WEBSERVER=0
     ;;
+    nobrowser )
+            NOBROWSER=1
+    ;;
     brbe=* )
             TOASTER_BRBE=$'\n'"TOASTER_BRBE=\""${param#*=}"\""
     ;;
@@ -168,71 +185,67 @@ for param in $*; do
     esac
 done
 
+[ -n "${BASH_SOURCE}" ] && SRCFILE=${BASH_SOURCE} || SRCFILE=$_
 
-if [ -z "$ZSH_NAME" ] && [ `basename \"$0\"` = `basename \"$BASH_SOURCE\"` ]; then
+if [ `basename \"$0\"` = `basename \"${SRCFILE}\"` ]; then
     # We are called as standalone. We refuse to run in a build environment - we need the interactive mode for that.
     # Start just the web server, point the web browser to the interface, and start any Django services.
 
     if ! verify_prereq; then
-        echo -e "Error: Could not verify that the needed dependencies are installed. Please use virtualenv and pip to install dependencies listed in toaster-requirements.txt" 1>&2;
-        exit 1;
+        echo "Error: Could not verify that the needed dependencies are installed. Please use virtualenv and pip to install dependencies listed in toaster-requirements.txt" 1>&2
+        exit 1
     fi
 
     if [ -n "$BUILDDIR" ]; then
-	echo -e "Error: It looks like you sourced oe-init-build-env. Toaster cannot start in build mode from an oe-core build environment.\n You should be starting Toaster from a new terminal window." 1>&2;
-        exit 1;
-    fi
-
-    if [ "x`which daemon`" == "x" ]; then
-        echo -e "Failed dependency; toaster needs the 'daemon' program in order to be able to start builds'. Please install the 'daemon' program from your distribution repositories or http://www.libslack.org/daemon/" 1>&2;
-	exit 1;
+        printf "Error: It looks like you sourced oe-init-build-env. Toaster cannot start in build mode from an oe-core build environment.\n You should be starting Toaster from a new terminal window." 1>&2
+        exit 1
     fi
 
     # Define a fake builddir where only the pid files are actually created. No real builds will take place here.
     BUILDDIR=/tmp/toaster_$$
     if [ -d "$BUILDDIR" ]; then
-        echo -e "Previous toaster run directory $BUILDDIR found, cowardly refusing to start. Please remove the directory when that toaster instance is over" 2>&1
-        exit 1;
+        echo "Previous toaster run directory $BUILDDIR found, cowardly refusing to start. Please remove the directory when that toaster instance is over" 2>&1
+        exit 1
     fi
 
     mkdir -p "$BUILDDIR"
 
     RUNNING=1
-    function trap_ctrlc() {
+    trap_ctrlc() {
         echo "** Stopping system"
         webserverKillAll
         RUNNING=0
     }
 
-    function do_cleanup() {
+    do_cleanup() {
         find "$BUILDDIR" -type f | xargs rm
         rmdir "$BUILDDIR"
     }
-    function cleanup() {
+    cleanup() {
         if grep -ir error "$BUILDDIR" >/dev/null; then
             if grep -irn "That port is already in use" "$BUILDDIR"; then
                 echo "You can use the \"webport=PORTNUMBER\" parameter to start Toaster on a different port (port $WEB_PORT is already in use)"
                 do_cleanup
             else
-                echo -e "\nErrors found in the Toaster log files present in '$BUILDDIR'. Directory will not be cleaned.\n Please review the errors and notify toaster@yoctoproject.org or submit a bug https://bugzilla.yoctoproject.org/enter_bug.cgi?product=Toaster"
+                printf "\nErrors found in the Toaster log files present in '$BUILDDIR'. Directory will not be cleaned.\n Please review the errors and notify toaster@yoctoproject.org or submit a bug https://bugzilla.yoctoproject.org/enter_bug.cgi?product=Toaster"
             fi
         else
             echo "No errors found, removing the run directory '$BUILDDIR'"
             do_cleanup
-        fi;
+        fi
     }
     TOASTER_MANAGED=1
     export TOASTER_MANAGED=1
     if [ $WEBSERVER -gt 0 ] && ! webserverStartAll; then
-        echo "Failed to start the web server, stopping" 1>&2;
+        echo "Failed to start the web server, stopping" 1>&2
         cleanup
-        exit 1;
+        exit 1
     fi
-    if [ $WEBSERVER -gt 0 ]; then
+    if [ $WEBSERVER -gt 0 ] && [ $NOBROWSER -eq 0 ]  ; then
         echo "Starting browser..."
         xdg-open http://127.0.0.1:$WEB_PORT/ >/dev/null 2>&1 &
     fi
-    trap trap_ctrlc SIGINT
+    trap trap_ctrlc 2
     echo "Toaster is now running. You can stop it with Ctrl-C"
     while [ $RUNNING -gt 0 ]; do
         python $BBBASEDIR/lib/toaster/manage.py runbuilds 2>&1 | tee -a "$BUILDDIR/toaster.log"
@@ -245,27 +258,27 @@ fi
 
 
 if ! verify_prereq; then
-	echo -e "Error: Could not verify that the needed dependencies are installed. Please use virtualenv and pip to install dependencies listed in toaster-requirements.txt" 1>&2;
-	return 1;
+    echo "Error: Could not verify that the needed dependencies are installed. Please use virtualenv and pip to install dependencies listed in toaster-requirements.txt" 1>&2
+    return 1
 fi
 
 
 # We make sure we're running in the current shell and in a good environment
-if [ -z "$BUILDDIR" ] || [ -z `which bitbake` ]; then
-    echo "Error: Build environment is not setup or bitbake is not in path." 1>&2;
+if [ -z "$BUILDDIR" ] ||  ! which bitbake >/dev/null 2>&1 ; then
+    echo "Error: Build environment is not setup or bitbake is not in path." 1>&2
     return 2
 fi
 
 
 # Determine the action. If specified by arguments, fine, if not, toggle it
-if [ "x$1" == "xstart" ] || [ "x$1" == "xstop" ]; then
+if [ "$1" = 'start' ] || [ "$1" = 'stop' ]; then
     CMD="$1"
 else
     if [ -z "$BBSERVER" ]; then
         CMD="start"
     else
         CMD="stop"
-    fi;
+    fi
 fi
 
 echo "The system will $CMD."
@@ -274,16 +287,16 @@ echo "The system will $CMD."
 
 lock=1
 if [ -e $BUILDDIR/bitbake.lock ]; then
-    (flock -n 200 ) 200<$BUILDDIR/bitbake.lock || lock=0
+    python -c "import fcntl; fcntl.flock(open(\"$BUILDDIR/bitbake.lock\"), fcntl.LOCK_EX|fcntl.LOCK_NB)" 2>/dev/null || lock=0
 fi
 
-if [ ${CMD} == "start" ] && [ $lock -eq 0 ]; then
+if [ ${CMD} = 'start' ] && [ $lock -eq 0 ]; then
     echo "Error: bitbake lock state error. File locks show that the system is on." 1>&2
     echo "Please wait for the current build to finish, stop and then start the system again." 1>&2
     return 3
 fi
 
-if [ ${CMD} == "start" ] && [ -e $BUILDDIR/.toastermain.pid ] && kill -0 `cat $BUILDDIR/.toastermain.pid`; then
+if [ ${CMD} = 'start' ] && [ -e $BUILDDIR/.toastermain.pid ] && kill -0 `cat $BUILDDIR/.toastermain.pid`; then
     echo "Warning: bitbake appears to be dead, but the Toaster web server is running. Something fishy is going on." 1>&2
     echo "Cleaning up the web server to start from a clean slate."
     webserverKillAll
@@ -303,7 +316,7 @@ case $CMD in
         unset BBSERVER
         PREREAD=""
         if [ -e ${BUILDDIR}/conf/toaster-pre.conf ]; then
-		rm ${BUILDDIR}/conf/toaster-pre.conf
+            rm ${BUILDDIR}/conf/toaster-pre.conf
         fi
         bitbake $PREREAD --postread conf/toaster.conf --server-only -t xmlrpc -B 0.0.0.0:0
         if [ $? -ne 0 ]; then
@@ -311,7 +324,7 @@ case $CMD in
             echo "Bitbake server start failed"
         else
             export BBSERVER=0.0.0.0:-1
-            if [ $NOTOASTERUI == 0 ]; then        # we start the TOASTERUI only if not inhibited
+            if [ $NOTOASTERUI -eq 0 ]; then        # we start the TOASTERUI only if not inhibited
                 bitbake --observe-only -u toasterui >>${BUILDDIR}/toaster_ui.log 2>&1 & echo $! >${BUILDDIR}/.toasterui.pid
             fi
         fi
@@ -337,4 +350,3 @@ case $CMD in
     ;;
 esac
 
-

bitbake/bin/toaster-eventreplay

@@ -26,6 +26,7 @@
 # as a build eventlog, and the ToasterUI is used to process events in the file
 # and log data in the database
 
+from __future__ import print_function
 import os
 import sys, logging
 
@@ -39,12 +40,6 @@ from bb.ui import toasterui
 import sys
 import logging
 
-logger = logging.getLogger(__name__)
-console = logging.StreamHandler(sys.stdout)
-format_str = "%(levelname)s: %(message)s"
-logging.basicConfig(format=format_str)
-
-
 import json, pickle
 
 
@@ -168,12 +163,12 @@ class MockConfigParameters():
 # run toaster ui on our mock bitbake class
 if __name__ == "__main__":
     if len(sys.argv) < 2:
-        logger.error("Usage: %s event.log " % sys.argv[0])
+        print("Usage: %s event.log " % sys.argv[0])
         sys.exit(1)
 
     file_name = sys.argv[-1]
     mock_connection = FileReadEventsServerConnection(file_name)
     configParams = MockConfigParameters()
 
-    # run the main program
-    toasterui.main(mock_connection.connection, mock_connection.events, configParams)
+    # run the main program and set exit code to the returned value
+    sys.exit(toasterui.main(mock_connection.connection, mock_connection.events, configParams))

bitbake/doc/bitbake-user-manual/bitbake-user-manual-customization.xsl

@@ -1,7 +1,15 @@
 <?xml version='1.0'?>
 <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns="http://www.w3.org/1999/xhtml" xmlns:fo="http://www.w3.org/1999/XSL/Format" version="1.0">
 
-  <xsl:import href="http://docbook.sourceforge.net/release/xsl/current/xhtml/docbook.xsl" />
+  <xsl:import href="http://downloads.yoctoproject.org/mirror/docbook-mirror/docbook-xsl-1.76.1/xhtml/docbook.xsl" />
+
+<!--
+
+  <xsl:import href="../template/1.76.1/docbook-xsl-1.76.1/xhtml/docbook.xsl" />
+
+  <xsl:import href="http://docbook.sourceforge.net/release/xsl/1.76.1/xhtml/docbook.xsl" />
+
+-->
 
   <xsl:include href="../template/permalinks.xsl"/>
   <xsl:include href="../template/section.title.xsl"/>

bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.xml

@@ -21,7 +21,7 @@
         The execution process is launched using the following command
         form:
         <literallayout class='monospaced'>
-     $ bitbake &lt;target&gt;
+     $ bitbake <replaceable>target</replaceable>
         </literallayout>
         For information on the BitBake command and its options,
         see
@@ -37,14 +37,16 @@
             </para>
 
             <para>
-                A common way to determine this value for your build host is to run:
+                A common method to determine this value for your build host is to run
+                the following:
                 <literallayout class='monospaced'>
      $ grep processor /proc/cpuinfo
                 </literallayout>
-                and count the number of processors displayed. Note that the number of
-                processors will take into account hyper-threading, so that a quad-core
-                build host with hyper-threading will most likely show eight processors,
-                which is the value you would then assign to that variable.
+                This command returns the number of processors, which takes into 
+                account hyper-threading. 
+                Thus, a quad-core build host with hyper-threading most likely 
+                shows eight processors, which is the value you would then assign to
+                <filename>BB_NUMBER_THREADS</filename>.
             </para>
 
             <para>
@@ -285,8 +287,8 @@
             <link linkend='var-PN'><filename>PN</filename></link> and
             <link linkend='var-PV'><filename>PV</filename></link>:
             <literallayout class='monospaced'>
-     PN = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE'),d)[0] or 'defaultpkgname'}"
-     PV = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE'),d)[1] or '1.0'}"
+     PN = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}"
+     PV = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[1] or '1.0'}"
             </literallayout>
             In this example, a recipe called "something_1.2.3.bb" would set
             <filename>PN</filename> to "something" and
@@ -782,13 +784,13 @@
             make some dependency and hash information available to the build.
             This information includes:
             <itemizedlist>
-                <listitem><para><filename>BB_BASEHASH_task-&lt;taskname&gt;</filename>:
+                <listitem><para><filename>BB_BASEHASH_task-</filename><replaceable>taskname</replaceable>:
                     The base hashes for each task in the recipe.
                     </para></listitem>
-                <listitem><para><filename>BB_BASEHASH_&lt;filename:taskname&gt;</filename>:
+                <listitem><para><filename>BB_BASEHASH_</filename><replaceable>filename</replaceable><filename>:</filename><replaceable>taskname</replaceable>:
                     The base hashes for each dependent task.
                     </para></listitem>
-                <listitem><para><filename>BBHASHDEPS_&lt;filename:taskname&gt;</filename>:
+                <listitem><para><filename>BBHASHDEPS_</filename><replaceable>filename</replaceable><filename>:</filename><replaceable>taskname</replaceable>:
                     The task dependencies for each task.
                     </para></listitem>
                 <listitem><para><filename>BB_TASKHASH</filename>:

bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml

@@ -157,8 +157,8 @@
             <filename>SRC_URI</filename> variable with the appropriate
             varflags as follows:
             <literallayout class='monospaced'>
-     SRC_URI[md5sum] = "value"
-     SRC_URI[sha256sum] = "value"
+     SRC_URI[md5sum] = "<replaceable>value</replaceable>"
+     SRC_URI[sha256sum] = "<replaceable>value</replaceable>"
             </literallayout>
             You can also specify the checksums as parameters on the
             <filename>SRC_URI</filename> as shown below:
@@ -325,6 +325,25 @@
      SRC_URI = "ftp://you@oe.handhelds.org/home/you/secret.plan"
                 </literallayout>
             </para>
+            <note>
+               Because URL parameters are delimited by semi-colons, this can
+               introduce ambiguity when parsing URLs that also contain semi-colons,
+               for example:
+                <literallayout class='monospaced'>
+     SRC_URI = "http://abc123.org/git/?p=gcc/gcc.git;a=snapshot;h=a5dd47"
+                </literallayout>
+               Such URLs should should be modified by replacing semi-colons with '&amp;' characters:
+               <literallayout class='monospaced'>
+     SRC_URI = "http://abc123.org/git/?p=gcc/gcc.git&amp;a=snapshot&amp;h=a5dd47"
+                </literallayout>
+                In most cases this should work. Treating semi-colons and '&amp;' in queries
+                identically is recommended by the World Wide Web Consortium (W3C).
+                Note that due to the nature of the URL, you may have to specify the name
+                of the downloaded file as well:
+              <literallayout class='monospaced'>
+     SRC_URI = "http://abc123.org/git/?p=gcc/gcc.git&amp;a=snapshot&amp;h=a5dd47;downloadfilename=myfile.bz2"
+                </literallayout>
+            </note>
         </section>
 
         <section id='cvs-fetcher'>
@@ -628,7 +647,7 @@
                 <literallayout class='monospaced'>
      SRC_URI = "ccrc://cc.example.org/ccrc;vob=/example_vob;module=/example_module"
      SRCREV = "EXAMPLE_CLEARCASE_TAG"
-     PV = "${@d.getVar("SRCREV").replace("/", "+")}"
+     PV = "${@d.getVar("SRCREV", False).replace("/", "+")}"
                 </literallayout>
                 The fetcher uses the <filename>rcleartool</filename> or
                 <filename>cleartool</filename> remote client, depending on
@@ -646,13 +665,19 @@
                         </para></listitem>
                     <listitem><para><emphasis><filename>module</filename></emphasis>:
                         The module, which must include the
-                        prepending "/" character, in the selected VOB
-                        The <filename>module</filename> and <filename>vob</filename>
-                        options are combined to create the following load rule in
-                        the view config spec:
-                        <literallayout class='monospaced'>
-     load &lt;vob&gt;&lt;module&gt;
-                        </literallayout>
+                        prepending "/" character, in the selected VOB.
+                        <note>
+                            The <filename>module</filename> and <filename>vob</filename>
+                            options are combined to create the <filename>load</filename> rule in
+                            the view config spec.
+                            As an example, consider the <filename>vob</filename> and 
+                            <filename>module</filename> values from the 
+                            <filename>SRC_URI</filename> statement at the start of this section.
+                            Combining those values results in the following:
+                            <literallayout class='monospaced'>
+     load /example_vob/example_module
+                            </literallayout>
+                        </note>
                         </para></listitem>
                     <listitem><para><emphasis><filename>proto</filename></emphasis>:
                         The protocol, which can be either <filename>http</filename> or

bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.xml

@@ -221,7 +221,7 @@
                 <para>From your shell, enter the following commands to set and
                 export the <filename>BBPATH</filename> variable:
                 <literallayout class='monospaced'>
-     $ BBPATH="&lt;projectdirectory&gt;"
+     $ BBPATH="<replaceable>projectdirectory</replaceable>"
      $ export BBPATH
                 </literallayout>
                 Use your actual project directory in the command.

bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml

@@ -327,8 +327,8 @@
                 The following lines select the values of a package name
                 and its version number, respectively:
                 <literallayout class='monospaced'>
-     PN = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE'),d)[0] or 'defaultpkgname'}"
-     PV = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE'),d)[1] or '1.0'}"
+     PN = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}"
+     PV = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[1] or '1.0'}"
                 </literallayout>
             </para>
         </section>
@@ -952,7 +952,7 @@
                     <listitem><para>
                         The class needs to define the function as follows:
                         <literallayout class='monospaced'>
-     &lt;classname&gt;_&lt;functionname&gt;
+     <replaceable>classname</replaceable><filename>_</filename><replaceable>functionname</replaceable>
                         </literallayout>
                         For example, if you have a class file
                         <filename>bar.bbclass</filename> and a function named
@@ -966,7 +966,7 @@
                         The class needs to contain the <filename>EXPORT_FUNCTIONS</filename>
                         statement as follows:
                         <literallayout class='monospaced'>
-     EXPORT_FUNCTIONS &lt;functionname&gt;
+     EXPORT_FUNCTIONS <replaceable>functionname</replaceable>
                         </literallayout>
                         For example, continuing with the same example, the
                         statement in the <filename>bar.bbclass</filename> would be
@@ -1065,13 +1065,41 @@
             <title>Deleting a Task</title>
 
             <para>
-                As well as being able to add tasks, tasks can also be deleted.
-                This is done simply with <filename>deltask</filename> command.
+                As well as being able to add tasks, you can delete them.
+                Simply use the <filename>deltask</filename> command to
+                delete a task.
                 For example, to delete the example task used in the previous
                 sections, you would use:
                 <literallayout class='monospaced'>
      deltask printdate
                 </literallayout>
+                If you delete a task using the <filename>deltask</filename>
+                command and the task has dependencies, the dependencies are
+                not reconnected.
+                For example, suppose you have three tasks named
+                <filename>do_a</filename>, <filename>do_b</filename>, and
+                <filename>do_c</filename>.
+                Furthermore, <filename>do_c</filename> is dependent on
+                <filename>do_b</filename>, which in turn is dependent on
+                <filename>do_a</filename>.
+                Given this scenario, if you use <filename>deltask</filename>
+                to delete <filename>do_b</filename>, the implicit dependency
+                relationship between <filename>do_c</filename> and
+                <filename>do_a</filename> through <filename>do_b</filename>
+                no longer exists, and <filename>do_c</filename> dependencies
+                are not updated to include <filename>do_a</filename>.
+                Thus, <filename>do_c</filename> is free to run before
+                <filename>do_a</filename>.
+            </para>
+
+            <para>
+                If you want dependencies such as these to remain intact, use
+                the <filename>noexec</filename> varflag to disable the task
+                instead of using the <filename>deltask</filename> command to
+                delete it:
+                <literallayout class='monospaced'>
+     do_b[noexec] = "1"
+                </literallayout>
             </para>
         </section>
 
@@ -1135,7 +1163,7 @@
             <para>
                 The <filename>BB_ORIGENV</filename> variable returns a datastore
                 object that can be queried using the standard datastore operators
-                such as <filename>getVar()</filename>.
+                such as <filename>getVar(, False)</filename>.
                 The datastore object is useful, for example, to find the original
                 <filename>DISPLAY</filename> variable.
                 Here is an example:
@@ -1164,7 +1192,7 @@
             BitBake reads and writes varflags to the datastore using the following
             command forms:
             <literallayout class='monospaced'>
-     &lt;variable&gt; = d.getVarFlags("&lt;variable&gt;")
+     <replaceable>variable</replaceable> = d.getVarFlags("<replaceable>variable</replaceable>")
      self.d.setVarFlags("FOO", {"func": True})
             </literallayout>
         </para>
@@ -1185,11 +1213,36 @@
             Tasks support a number of these flags which control various
             functionality of the task:
             <itemizedlist>
+                <listitem><para><emphasis>cleandirs:</emphasis>
+                    Empty directories that should created before the task runs.
+                    </para></listitem>
+                <listitem><para><emphasis>depends:</emphasis>
+                    Controls inter-task dependencies.
+                    See the
+                    <link linkend='var-DEPENDS'><filename>DEPENDS</filename></link>
+                    variable and the
+                    "<link linkend='inter-task-dependencies'>Inter-Task Dependencies</link>"
+                    section for more information.
+                    </para></listitem>
+                <listitem><para><emphasis>deptask:</emphasis>
+                    Controls task build-time dependencies.
+                    See the
+                    <link linkend='var-DEPENDS'><filename>DEPENDS</filename></link>
+                    variable and the
+                    "<link linkend='build-dependencies'>Build Dependencies</link>"
+                    section for more information.
+                    </para></listitem>
                 <listitem><para><emphasis>dirs:</emphasis>
                     Directories that should be created before the task runs.
                     </para></listitem>
-                <listitem><para><emphasis>cleandirs:</emphasis>
-                    Empty directories that should created before the task runs.
+                <listitem><para><emphasis>lockfiles:</emphasis>
+                     Specifies one or more lockfiles to lock while the task
+                     executes.
+                     Only one task may hold a lockfile, and any task that
+                     attempts to lock an already locked file will block until
+                     the lock is released.
+                     You can use this variable flag to accomplish mutual
+                     exclusion.
                     </para></listitem>
                 <listitem><para><emphasis>noexec:</emphasis>
                     Marks the tasks as being empty and no execution required.
@@ -1201,15 +1254,20 @@
                     Tells BitBake to not generate a stamp file for a task,
                     which implies the task should always be executed.
                     </para></listitem>
-                <listitem><para><emphasis>umask:</emphasis>
-                    The umask to run the task under.
+                <listitem><para><emphasis>postfuncs:</emphasis>
+                    List of functions to call after the completion of the task.
                     </para></listitem>
-                <listitem><para><emphasis>deptask:</emphasis>
-                    Controls task build-time dependencies.
+                <listitem><para><emphasis>prefuncs:</emphasis>
+                    List of functions to call before the task executes.
+                    </para></listitem>
+                <listitem><para><emphasis>rdepends:</emphasis>
+                    Controls inter-task runtime dependencies.
                     See the
-                    <link linkend='var-DEPENDS'><filename>DEPENDS</filename></link>
-                    variable and the
-                    "<link linkend='build-dependencies'>Build Dependencies</link>"
+                    <link linkend='var-RDEPENDS'><filename>RDEPENDS</filename></link>
+                    variable, the
+                    <link linkend='var-RRECOMMENDS'><filename>RRECOMMENDS</filename></link>
+                    variable, and the
+                    "<link linkend='inter-task-dependencies'>Inter-Task Dependencies</link>"
                     section for more information.
                     </para></listitem>
                 <listitem><para><emphasis>rdeptask:</emphasis>
@@ -1222,6 +1280,11 @@
                     "<link linkend='runtime-dependencies'>Runtime Dependencies</link>"
                     section for more information.
                     </para></listitem>
+                <listitem><para><emphasis>recideptask:</emphasis>
+                    When set in conjunction with
+                    <filename>recrdeptask</filename>, specifies a task that
+                    should be inspected for additional dependencies.
+                    </para></listitem>
                 <listitem><para><emphasis>recrdeptask:</emphasis>
                     Controls task recursive runtime dependencies.
                     See the
@@ -1232,35 +1295,14 @@
                     "<link linkend='recursive-dependencies'>Recursive Dependencies</link>"
                     section for more information.
                     </para></listitem>
-                <listitem><para><emphasis>depends:</emphasis>
-                    Controls inter-task dependencies.
-                    See the
-                    <link linkend='var-DEPENDS'><filename>DEPENDS</filename></link>
-                    variable and the
-                    "<link linkend='inter-task-dependencies'>Inter-Task Dependencies</link>"
-                    section for more information.
-                    </para></listitem>
-                <listitem><para><emphasis>rdepends:</emphasis>
-                    Controls inter-task runtime dependencies.
-                    See the
-                    <link linkend='var-RDEPENDS'><filename>RDEPENDS</filename></link>
-                    variable, the
-                    <link linkend='var-RRECOMMENDS'><filename>RRECOMMENDS</filename></link>
-                    variable, and the
-                    "<link linkend='inter-task-dependencies'>Inter-Task Dependencies</link>"
-                    section for more information.
-                    </para></listitem>
-                <listitem><para><emphasis>postfuncs:</emphasis>
-                    List of functions to call after the completion of the task.
-                    </para></listitem>
-                <listitem><para><emphasis>prefuncs:</emphasis>
-                    List of functions to call before the task executes.
-                    </para></listitem>
                 <listitem><para><emphasis>stamp-extra-info:</emphasis>
                     Extra stamp information to append to the task's stamp.
                     As an example, OpenEmbedded uses this flag to allow
                     machine-specific tasks.
                     </para></listitem>
+                <listitem><para><emphasis>umask:</emphasis>
+                    The umask to run the task under.
+                    </para></listitem>
             </itemizedlist>
         </para>
 
@@ -1280,16 +1322,16 @@
                     does not allow BitBake to automatically determine
                     that the variable is referred to.
                     </para></listitem>
-                <listitem><para><emphasis>vardepvalue:</emphasis>
-                    If set, instructs BitBake to ignore the actual
-                    value of the variable and instead use the specified
-                    value when calculating the variable's signature.
-                    </para></listitem>
                 <listitem><para><emphasis>vardepsexclude:</emphasis>
                     Specifies a space-separated list of variables
                     that should be excluded from a variable's dependencies
                     for the purposes of calculating its signature.
                     </para></listitem>
+                <listitem><para><emphasis>vardepvalue:</emphasis>
+                    If set, instructs BitBake to ignore the actual
+                    value of the variable and instead use the specified
+                    value when calculating the variable's signature.
+                    </para></listitem>
                 <listitem><para><emphasis>vardepvalueexclude:</emphasis>
                     Specifies a pipe-separated list of strings to exclude
                     from the variable's value when calculating the

bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.xml

@@ -102,6 +102,56 @@
             </glossdef>
         </glossentry>
 
+        <glossentry id='var-BB_ALLOWED_NETWORKS'><glossterm>BB_ALLOWED_NETWORKS</glossterm>
+            <glossdef>
+                <para>
+                    Specifies a space-delimited list of hosts that the fetcher
+                    is allowed to use to obtain the required source code.
+                    Following are considerations surrounding this variable:
+                    <itemizedlist>
+                        <listitem><para>
+                            This host list is only used if
+                            <link linkend='var-BB_NO_NETWORK'><filename>BB_NO_NETWORK</filename></link>
+                            is either not set or set to "0".
+                            </para></listitem>
+                        <listitem><para>
+                            Limited support for wildcard matching against the
+                            beginning of host names exists.
+                            For example, the following setting matches
+                            <filename>git.gnu.org</filename>,
+                            <filename>ftp.gnu.org</filename>, and
+                            <filename>foo.git.gnu.org</filename>.
+                            <literallayout class='monospaced'>
+     BB_ALLOWED_NETWORKS = "*.gnu.org"
+                            </literallayout>
+                            </para></listitem>
+                        <listitem><para>
+                            Mirrors not in the host list are skipped and
+                            logged in debug.
+                            </para></listitem>
+                        <listitem><para>
+                            Attempts to access networks not in the host list
+                            cause a failure.
+                            </para></listitem>
+                    </itemizedlist>
+                    Using <filename>BB_ALLOWED_NETWORKS</filename> in
+                    conjunction with
+                    <link linkend='var-PREMIRRORS'><filename>PREMIRRORS</filename></link>
+                    is very useful.
+                    Adding the host you want to use to
+                    <filename>PREMIRRORS</filename> results in the source code
+                    being fetched from an allowed location and avoids raising
+                    an error when a host that is not allowed is in a
+                    <link linkend='var-SRC_URI'><filename>SRC_URI</filename></link>
+                    statement.
+                    This is because the fetcher does not attempt to use the
+                    host listed in <filename>SRC_URI</filename> after a
+                    successful fetch from the
+                    <filename>PREMIRRORS</filename> occurs.
+                </para>
+            </glossdef>
+        </glossentry>
+
         <glossentry id='var-BB_CONSOLELOG'><glossterm>BB_CONSOLELOG</glossterm>
             <glossdef>
                 <para>
@@ -872,7 +922,7 @@
                     that run on the target <filename>MACHINE</filename>;
                     "nativesdk", which targets the SDK machine instead of
                     <filename>MACHINE</filename>; and "mulitlibs" in the form
-                    "<filename>multilib:&lt;multilib_name&gt;</filename>".
+                    "<filename>multilib:</filename><replaceable>multilib_name</replaceable>".
                 </para>
 
                 <para>
@@ -884,7 +934,7 @@
                     metadata:
                     <literallayout class='monospaced'>
      BBCLASSEXTEND =+ "native nativesdk"
-     BBCLASSEXTEND =+ "multilib:&lt;multilib_name&gt;"
+     BBCLASSEXTEND =+ "multilib:<replaceable>multilib_name</replaceable>"
                     </literallayout>
                 </para>
              </glossdef>
@@ -1016,6 +1066,20 @@
             </glossdef>
         </glossentry>
 
+        <glossentry id='var-BBLAYERS_FETCH_DIR'><glossterm>BBLAYERS_FETCH_DIR</glossterm>
+            <glossdef>
+                <para>
+                    Sets the base location where layers are stored.
+                    By default, this location is set to
+                    <filename>${COREBASE}</filename>.
+                    This setting is used in conjunction with
+                    <filename>bitbake-layers layerindex-fetch</filename> and
+                    tells <filename>bitbake-layers</filename> where to place
+                    the fetched layers.
+                </para>
+            </glossdef>
+        </glossentry>
+
         <glossentry id='var-BBMASK'><glossterm>BBMASK</glossterm>
             <glossdef>
                 <para>
@@ -1091,9 +1155,9 @@
                     Set the variable as you would any environment variable
                     and then run BitBake:
                     <literallayout class='monospaced'>
-     $ BBPATH="&lt;build_directory&gt;"
+     $ BBPATH="<replaceable>build_directory</replaceable>"
      $ export BBPATH
-     $ bitbake &lt;target&gt;
+     $ bitbake <replaceable>target</replaceable>
                     </literallayout>
                 </para>
             </glossdef>
@@ -1888,7 +1952,7 @@
                     Here is the general syntax to specify versions with
                     the <filename>RDEPENDS</filename> variable:
                     <literallayout class='monospaced'>
-     RDEPENDS_${PN} = "&lt;package&gt; (&lt;operator&gt; &lt;version&gt;)"
+     RDEPENDS_${PN} = "<replaceable>package</replaceable> (<replaceable>operator</replaceable> <replaceable>version</replaceable>)"
                     </literallayout>
                     For <filename>operator</filename>, you can specify the
                     following:
@@ -1954,7 +2018,7 @@
                     Here is the general syntax to specify versions with
                     the <filename>RRECOMMENDS</filename> variable:
                     <literallayout class='monospaced'>
-     RRECOMMENDS_${PN} = "&lt;package&gt; (&lt;operator&gt; &lt;version&gt;)"
+     RRECOMMENDS_${PN} = "<replaceable>package</replaceable> (<replaceable>operator</replaceable> <replaceable>version</replaceable>)"
                     </literallayout>
                     For <filename>operator</filename>, you can specify the
                     following:

bitbake/lib/bb/__init__.py

@@ -21,7 +21,7 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-__version__ = "1.25.0"
+__version__ = "1.27.1"
 
 import sys
 if sys.version_info < (2, 7, 3):
@@ -94,11 +94,11 @@ def note(*args):
 def warn(*args):
     logger.warn(''.join(args))
 
-def error(*args):
-    logger.error(''.join(args))
+def error(*args, **kwargs):
+    logger.error(''.join(args), extra=kwargs)
 
-def fatal(*args):
-    logger.critical(''.join(args))
+def fatal(*args, **kwargs):
+    logger.critical(''.join(args), extra=kwargs)
     raise BBHandledException()
 
 def deprecated(func, name=None, advice=""):

bitbake/lib/bb/build.py

@@ -31,6 +31,7 @@ import logging
 import shlex
 import glob
 import time
+import stat
 import bb
 import bb.msg
 import bb.process
@@ -42,6 +43,20 @@ logger = logging.getLogger('BitBake.Build')
 
 NULL = open(os.devnull, 'r+')
 
+__mtime_cache = {}
+
+def cached_mtime_noerror(f):
+    if f not in __mtime_cache:
+        try:
+            __mtime_cache[f] = os.stat(f)[stat.ST_MTIME]
+        except OSError:
+            return 0
+    return __mtime_cache[f]
+
+def reset_cache():
+    global __mtime_cache
+    __mtime_cache = {}
+
 # When we execute a Python function, we'd like certain things
 # in all namespaces, hence we add them to __builtins__.
 # If we do not do this and use the exec globals, they will
@@ -144,7 +159,7 @@ class LogTee(object):
 def exec_func(func, d, dirs = None):
     """Execute a BB 'function'"""
 
-    body = d.getVar(func)
+    body = d.getVar(func, False)
     if not body:
         if body is None:
             logger.warn("Function %s doesn't exist", func)
@@ -298,7 +313,7 @@ def exec_func_shell(func, d, runfile, cwd=None):
 # cleanup
 ret=$?
 trap '' 0
-exit $?
+exit $ret
 ''')
 
     os.chmod(runfile, 0775)
@@ -314,14 +329,52 @@ exit $?
     else:
         logfile = sys.stdout
 
-    bb.debug(2, "Executing shell function %s" % func)
+    def readfifo(data):
+        lines = data.split('\0')
+        for line in lines:
+            splitval = line.split(' ', 1)
+            cmd = splitval[0]
+            if len(splitval) > 1:
+                value = splitval[1]
+            else:
+                value = ''
+            if cmd == 'bbplain':
+                bb.plain(value)
+            elif cmd == 'bbnote':
+                bb.note(value)
+            elif cmd == 'bbwarn':
+                bb.warn(value)
+            elif cmd == 'bberror':
+                bb.error(value)
+            elif cmd == 'bbfatal':
+                # The caller will call exit themselves, so bb.error() is
+                # what we want here rather than bb.fatal()
+                bb.error(value)
+            elif cmd == 'bbfatal_log':
+                bb.error(value, forcelog=True)
+            elif cmd == 'bbdebug':
+                splitval = value.split(' ', 1)
+                level = int(splitval[0])
+                value = splitval[1]
+                bb.debug(level, value)
 
-    try:
-        with open(os.devnull, 'r+') as stdin:
-            bb.process.run(cmd, shell=False, stdin=stdin, log=logfile)
-    except bb.process.CmdError:
-        logfn = d.getVar('BB_LOGFILE', True)
-        raise FuncFailed(func, logfn)
+    tempdir = d.getVar('T', True)
+    fifopath = os.path.join(tempdir, 'fifo.%s' % os.getpid())
+    if os.path.exists(fifopath):
+        os.unlink(fifopath)
+    os.mkfifo(fifopath)
+    with open(fifopath, 'r+') as fifo:
+        try:
+            bb.debug(2, "Executing shell function %s" % func)
+
+            try:
+                with open(os.devnull, 'r+') as stdin:
+                    bb.process.run(cmd, shell=False, stdin=stdin, log=logfile, extrafiles=[(fifo,readfifo)])
+            except bb.process.CmdError:
+                logfn = d.getVar('BB_LOGFILE', True)
+                raise FuncFailed(func, logfn)
+        finally:
+            os.unlink(fifopath)
 
     bb.debug(2, "Shell function %s finished" % func)
 
@@ -395,7 +448,10 @@ def _exec_task(fn, task, d, quieterr):
             self.triggered = False
             logging.Handler.__init__(self, logging.ERROR)
         def emit(self, record):
-            self.triggered = True
+            if getattr(record, 'forcelog', False):
+                self.triggered = False
+            else:
+                self.triggered = True
 
     # Handle logfiles
     si = open('/dev/null', 'r')
@@ -535,7 +591,7 @@ def stamp_internal(taskname, d, file_name, baseonly=False):
     stamp = bb.parse.siggen.stampfile(stamp, file_name, taskname, extrainfo)
 
     stampdir = os.path.dirname(stamp)
-    if bb.parse.cached_mtime_noerror(stampdir) == 0:
+    if cached_mtime_noerror(stampdir) == 0:
         bb.utils.mkdirhier(stampdir)
 
     return stamp
@@ -631,7 +687,7 @@ def stampfile(taskname, d, file_name = None):
     return stamp_internal(taskname, d, file_name)
 
 def add_tasks(tasklist, deltasklist, d):
-    task_deps = d.getVar('_task_deps')
+    task_deps = d.getVar('_task_deps', False)
     if not task_deps:
         task_deps = {}
     if not 'tasks' in task_deps:
@@ -681,7 +737,7 @@ def addtask(task, before, after, d):
         task = "do_" + task
 
     d.setVarFlag(task, "task", 1)
-    bbtasks = d.getVar('__BBTASKS') or []
+    bbtasks = d.getVar('__BBTASKS', False) or []
     if not task in bbtasks:
         bbtasks.append(task)
     d.setVar('__BBTASKS', bbtasks)
@@ -704,7 +760,7 @@ def deltask(task, d):
     if task[:3] != "do_":
         task = "do_" + task
 
-    bbtasks = d.getVar('__BBDELTASKS') or []
+    bbtasks = d.getVar('__BBDELTASKS', False) or []
     if not task in bbtasks:
         bbtasks.append(task)
     d.setVar('__BBDELTASKS', bbtasks)

bitbake/lib/bb/cache.py

@@ -659,25 +659,25 @@ class Cache(object):
         """
         chdir_back = False
 
-        from bb import data, parse
+        from bb import parse
 
         # expand tmpdir to include this topdir
-        data.setVar('TMPDIR', data.getVar('TMPDIR', config, 1) or "", config)
+        config.setVar('TMPDIR', config.getVar('TMPDIR', True) or "")
         bbfile_loc = os.path.abspath(os.path.dirname(bbfile))
         oldpath = os.path.abspath(os.getcwd())
         parse.cached_mtime_noerror(bbfile_loc)
-        bb_data = data.init_db(config)
+        bb_data = config.createCopy()
         # The ConfHandler first looks if there is a TOPDIR and if not
         # then it would call getcwd().
         # Previously, we chdir()ed to bbfile_loc, called the handler
         # and finally chdir()ed back, a couple of thousand times. We now
         # just fill in TOPDIR to point to bbfile_loc if there is no TOPDIR yet.
-        if not data.getVar('TOPDIR', bb_data):
+        if not bb_data.getVar('TOPDIR', False):
             chdir_back = True
-            data.setVar('TOPDIR', bbfile_loc, bb_data)
+            bb_data.setVar('TOPDIR', bbfile_loc)
         try:
             if appends:
-                data.setVar('__BBAPPEND', " ".join(appends), bb_data)
+                bb_data.setVar('__BBAPPEND', " ".join(appends))
             bb_data = parse.handle(bbfile, bb_data)
             if chdir_back:
                 os.chdir(oldpath)

bitbake/lib/bb/codeparser.py

@@ -92,6 +92,9 @@ class pythonCacheLine(object):
         for c in sorted(self.contains.keys()):
             l = l + (c, hash(self.contains[c]))
         return hash(l)
+    def __repr__(self):
+        return " ".join([str(self.refs), str(self.execs), str(self.contains)]) 
+
 
 class shellCacheLine(object):
     def __init__(self, execs):
@@ -105,6 +108,8 @@ class shellCacheLine(object):
         self.__init__(execs)
     def __hash__(self):
         return hash(self.execs)
+    def __repr__(self):
+        return str(self.execs)
 
 class CodeParserCache(MultiProcessCache):
     cache_file_name = "bb_codeparser.dat"
@@ -240,6 +245,9 @@ class PythonParser():
         self.unhandled_message = "while parsing %s, %s" % (name, self.unhandled_message)
 
     def parse_python(self, node):
+        if not node or not node.strip():
+            return
+
         h = hash(str(node))
 
         if h in codeparsercache.pythoncache:

bitbake/lib/bb/cooker.py

@@ -35,9 +35,11 @@ from contextlib import closing
 from functools import wraps
 from collections import defaultdict
 import bb, bb.exceptions, bb.command
-from bb import utils, data, parse, event, cache, providers, taskdata, runqueue
+from bb import utils, data, parse, event, cache, providers, taskdata, runqueue, build
 import Queue
 import signal
+import subprocess
+import errno
 import prserv.serv
 import pyinotify
 
@@ -151,9 +153,7 @@ class BBCooker:
 
         # Take a lock so only one copy of bitbake can run against a given build
         # directory at a time
-        lockfile = self.data.expand("${TOPDIR}/bitbake.lock")
-        self.lock = bb.utils.lockfile(lockfile, False, False)
-        if not self.lock:
+        if not self.lockBitbake():
             bb.fatal("Only one copy of bitbake should be run against a build directory")
         try:
             self.lock.seek(0)
@@ -349,6 +349,10 @@ class BBCooker:
         if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset:
             self.disableDataTracking()
 
+        self.data.renameVar("__depends", "__base_depends")
+        self.add_filewatch(self.data.getVar("__base_depends", False), self.configwatcher)
+
+
     def enableDataTracking(self):
         self.configuration.tracking = True
         if hasattr(self, "data"):
@@ -386,7 +390,7 @@ class BBCooker:
 
         replaced = False
         #do not save if nothing changed
-        if str(val) == self.data.getVar(var):
+        if str(val) == self.data.getVar(var, False):
             return
 
         conf_files = self.data.varhistory.get_variable_files(var)
@@ -398,7 +402,7 @@ class BBCooker:
                 listval += "%s   " % value
             val = listval
 
-        topdir = self.data.getVar("TOPDIR")
+        topdir = self.data.getVar("TOPDIR", False)
 
         #comment or replace operations made on var
         for conf_file in conf_files:
@@ -453,7 +457,7 @@ class BBCooker:
 
     def removeConfigurationVar(self, var):
         conf_files = self.data.varhistory.get_variable_files(var)
-        topdir = self.data.getVar("TOPDIR")
+        topdir = self.data.getVar("TOPDIR", False)
 
         for conf_file in conf_files:
             if topdir in conf_file:
@@ -493,7 +497,7 @@ class BBCooker:
 
     def parseConfiguration(self):
         # Set log file verbosity
-        verboselogs = bb.utils.to_boolean(self.data.getVar("BB_VERBOSE_LOGS", "0"))
+        verboselogs = bb.utils.to_boolean(self.data.getVar("BB_VERBOSE_LOGS", False))
         if verboselogs:
             bb.msg.loggerVerboseLogs = True
 
@@ -583,7 +587,7 @@ class BBCooker:
             if pkgs_to_build[0] in set(ignore.split()):
                 bb.fatal("%s is in ASSUME_PROVIDED" % pkgs_to_build[0])
 
-            taskdata, runlist, pkgs_to_build = self.buildTaskData(pkgs_to_build, None, self.configuration.abort)
+            taskdata, runlist, pkgs_to_build = self.buildTaskData(pkgs_to_build, None, self.configuration.abort, allowincomplete=True)
 
             targetid = taskdata.getbuild_id(pkgs_to_build[0])
             fnid = taskdata.build_targets[targetid][0]
@@ -613,10 +617,10 @@ class BBCooker:
         data.expandKeys(envdata)
         for e in envdata.keys():
             if data.getVarFlag( e, 'python', envdata ):
-                logger.plain("\npython %s () {\n%s}\n", e, data.getVar(e, envdata, 1))
+                logger.plain("\npython %s () {\n%s}\n", e, envdata.getVar(e, True))
 
 
-    def buildTaskData(self, pkgs_to_build, task, abort):
+    def buildTaskData(self, pkgs_to_build, task, abort, allowincomplete=False):
         """
         Prepare a runqueue and taskdata object for iteration over pkgs_to_build
         """
@@ -631,7 +635,7 @@ class BBCooker:
         localdata = data.createCopy(self.data)
         bb.data.update_data(localdata)
         bb.data.expandKeys(localdata)
-        taskdata = bb.taskdata.TaskData(abort, skiplist=self.skiplist)
+        taskdata = bb.taskdata.TaskData(abort, skiplist=self.skiplist, allowincomplete=allowincomplete)
 
         current = 0
         runlist = []
@@ -656,7 +660,7 @@ class BBCooker:
 
         # We set abort to False here to prevent unbuildable targets raising
         # an exception when we're just generating data
-        taskdata, runlist, pkgs_to_build = self.buildTaskData(pkgs_to_build, task, False)
+        taskdata, runlist, pkgs_to_build = self.buildTaskData(pkgs_to_build, task, False, allowincomplete=True)
 
         return runlist, taskdata
 
@@ -908,8 +912,8 @@ class BBCooker:
                            for appends in appends_without_recipes
                            for append in appends)
             msg = 'No recipes available for:\n%s' % '\n'.join(appendlines)
-            warn_only = data.getVar("BB_DANGLINGAPPENDS_WARNONLY", \
-                 self.data, False) or "no"
+            warn_only = self.data.getVar("BB_DANGLINGAPPENDS_WARNONLY", \
+                 False) or "no"
             if warn_only.lower() in ("1", "yes", "true"):
                 bb.warn(msg)
             else:
@@ -956,8 +960,8 @@ class BBCooker:
         # Generate a list of parsed configuration files by searching the files
         # listed in the __depends and __base_depends variables with a .conf suffix.
         conffiles = []
-        dep_files = self.data.getVar('__base_depends') or []
-        dep_files = dep_files + (self.data.getVar('__depends') or [])
+        dep_files = self.data.getVar('__base_depends', False) or []
+        dep_files = dep_files + (self.data.getVar('__depends', False) or [])
 
         for f in dep_files:
             if f[0].endswith(".conf"):
@@ -1174,9 +1178,12 @@ class BBCooker:
         """
         Setup any variables needed before starting a build
         """
-        if not self.data.getVar("BUILDNAME"):
-            self.data.setVar("BUILDNAME", time.strftime('%Y%m%d%H%M'))
-        self.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S', time.gmtime()))
+        t = time.gmtime() 
+        if not self.data.getVar("BUILDNAME", False):
+            self.data.setVar("BUILDNAME", "${DATE}${TIME}")
+        self.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S', t))
+        self.data.setVar("DATE", time.strftime('%Y%m%d', t))
+        self.data.setVar("TIME", time.strftime('%H%M%S', t))
 
     def matchFiles(self, bf):
         """
@@ -1275,7 +1282,7 @@ class BBCooker:
         taskdata = bb.taskdata.TaskData(self.configuration.abort)
         taskdata.add_provider(self.data, self.recipecache, item)
 
-        buildname = self.data.getVar("BUILDNAME")
+        buildname = self.data.getVar("BUILDNAME", True)
         bb.event.fire(bb.event.BuildStarted(buildname, [item]), self.expanded_data)
 
         # Execute the runqueue
@@ -1303,8 +1310,8 @@ class BBCooker:
                 return False
 
             if not retval:
-                self.command.finishAsyncCommand(msg)
                 bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runq_fnid), buildname, item, failures), self.expanded_data)
+                self.command.finishAsyncCommand(msg)
                 return False
             if retval is True:
                 return True
@@ -1336,18 +1343,19 @@ class BBCooker:
                 return False
 
             if not retval:
-                self.command.finishAsyncCommand(msg)
                 bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runq_fnid), buildname, targets, failures), self.data)
+                self.command.finishAsyncCommand(msg)
                 return False
             if retval is True:
                 return True
             return retval
 
+        build.reset_cache()
         self.buildSetVars()
 
         taskdata, runlist, fulltargetlist = self.buildTaskData(targets, task, self.configuration.abort)
 
-        buildname = self.data.getVar("BUILDNAME")
+        buildname = self.data.getVar("BUILDNAME", False)
         bb.event.fire(bb.event.BuildStarted(buildname, fulltargetlist), self.data)
 
         rq = bb.runqueue.RunQueue(self, self.data, self.recipecache, taskdata, runlist)
@@ -1401,7 +1409,7 @@ class BBCooker:
             if base_image is None:
                 imagefile.write("inherit core-image\n")
             else:
-                topdir = self.data.getVar("TOPDIR")
+                topdir = self.data.getVar("TOPDIR", False)
                 if topdir in base_image:
                     base_image = require_line.split()[1]
                 imagefile.write("require " + base_image + "\n")
@@ -1460,9 +1468,6 @@ class BBCooker:
             self.collection = CookerCollectFiles(self.recipecache.bbfile_config_priorities)
             (filelist, masked) = self.collection.collect_bbfiles(self.data, self.expanded_data)
 
-            self.data.renameVar("__depends", "__base_depends")
-            self.add_filewatch(self.data.getVar("__base_depends"), self.configwatcher)
-
             self.parser = CookerParser(self, filelist, masked)
             self.parsecache_valid = True
 
@@ -1476,6 +1481,11 @@ class BBCooker:
             self.handlePrefProviders()
             self.recipecache.bbfile_priority = self.collection.collection_priorities(self.recipecache.pkg_fn, self.data)
             self.state = state.running
+
+            # Send an event listing all stamps reachable after parsing
+            # which the metadata may use to clean up stale data
+            event = bb.event.ReachableStamps(self.recipecache.stamp)
+            bb.event.fire(event, self.expanded_data)
             return None
 
         return True
@@ -1525,6 +1535,34 @@ class BBCooker:
     def post_serve(self):
         prserv.serv.auto_shutdown(self.data)
         bb.event.fire(CookerExit(), self.expanded_data)
+        lockfile = self.lock.name
+        self.lock.close()
+        self.lock = None
+
+        while not self.lock:
+            with bb.utils.timeout(3):
+                self.lock = bb.utils.lockfile(lockfile, shared=False, retry=False, block=True)
+                if not self.lock:
+                    # Some systems may not have lsof available
+                    procs = None
+                    try:
+                        procs = subprocess.check_output(["lsof", '-w', lockfile], stderr=subprocess.STDOUT)
+                    except OSError as e:
+                        if e.errno != errno.ENOENT:
+                            raise
+                    if procs is None:
+                        # Fall back to fuser if lsof is unavailable
+                        try:
+                            procs = subprocess.check_output(["fuser", '-v', lockfile], stderr=subprocess.STDOUT)
+                        except OSError as e:
+                            if e.errno != errno.ENOENT:
+                                raise
+
+                    msg = "Delaying shutdown due to active processes which appear to be holding bitbake.lock"
+                    if procs:
+                        msg += ":\n%s" % str(procs)
+                    print(msg)
+
 
     def shutdown(self, force = False):
         if force:
@@ -1538,6 +1576,19 @@ class BBCooker:
     def reset(self):
         self.initConfigurationData()
 
+    def lockBitbake(self):
+        if not hasattr(self, 'lock'):
+            self.lock = None
+            if self.data:
+                lockfile = self.data.expand("${TOPDIR}/bitbake.lock")
+                if lockfile:
+                    self.lock = bb.utils.lockfile(lockfile, False, False)
+        return self.lock
+
+    def unlockBitbake(self):
+        if hasattr(self, 'lock') and self.lock:
+            bb.utils.unlockfile(self.lock)
+
 def server_main(cooker, func, *args):
     cooker.pre_serve()
 
@@ -1573,6 +1624,7 @@ class CookerExit(bb.event.Event):
 class CookerCollectFiles(object):
     def __init__(self, priorities):
         self.appendlist = {}
+        self.bbappends = []
         self.appliedappendlist = []
         self.bbfile_config_priorities = priorities
 
@@ -1667,6 +1719,7 @@ class CookerCollectFiles(object):
         # Build a list of .bbappend files for each .bb file
         for f in bbappend:
             base = os.path.basename(f).replace('.bbappend', '.bb')
+            self.bbappends.append((base, f))
             if not base in self.appendlist:
                self.appendlist[base] = []
             if f not in self.appendlist[base]:
@@ -1692,11 +1745,11 @@ class CookerCollectFiles(object):
         """
         filelist = []
         f = os.path.basename(fn)
-        for bbappend in self.appendlist:
+        for b in self.bbappends:
+            (bbappend, filename) = b
             if (bbappend == f) or ('%' in bbappend and bbappend.startswith(f[:bbappend.index('%')])):
                 self.appliedappendlist.append(bbappend)
-                for filename in self.appendlist[bbappend]:
-                    filelist.append(filename)
+                filelist.append(filename)
         return filelist
 
     def collection_priorities(self, pkgfns, d):
@@ -1716,10 +1769,10 @@ class CookerCollectFiles(object):
                 unmatched.add(regex)
 
         def findmatch(regex):
-            for bbfile in self.appendlist:
-                for append in self.appendlist[bbfile]:
-                    if regex.match(append):
-                        return True
+            for b in self.bbappends:
+                (bbfile, append) = b
+                if regex.match(append):
+                    return True
             return False
 
         for unmatch in unmatched.copy():
@@ -1795,8 +1848,6 @@ class Parser(multiprocessing.Process):
         finally:
             logfile = "profile-parse-%s.log" % multiprocessing.current_process().name
             prof.dump_stats(logfile)
-            bb.utils.process_profilelog(logfile)
-            print("Raw profiling information saved to %s and processed statistics to %s.processed" % (logfile, logfile))
 
     def realrun(self):
         if self.init:
@@ -1866,6 +1917,7 @@ class CookerParser(object):
         self.current = 0
         self.num_processes = int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS", True) or
                                  multiprocessing.cpu_count())
+        self.process_names = []
 
         self.bb_cache = bb.cache.Cache(self.cfgdata, self.cfghash, cooker.caches_array)
         self.fromcache = []
@@ -1901,6 +1953,7 @@ class CookerParser(object):
             for i in range(0, self.num_processes):
                 parser = Parser(self.jobs, self.result_queue, self.parser_quit, init, self.cooker.configuration.profile)
                 parser.start()
+                self.process_names.append(parser.name)
                 self.processes.append(parser)
 
             self.results = itertools.chain(self.results, self.parse_generator())
@@ -1944,6 +1997,16 @@ class CookerParser(object):
         multiprocessing.util.Finalize(None, sync.join, exitpriority=-100)
         bb.codeparser.parser_cache_savemerge(self.cooker.data)
         bb.fetch.fetcher_parse_done(self.cooker.data)
+        if self.cooker.configuration.profile:
+            profiles = []
+            for i in self.process_names:
+                logfile = "profile-parse-%s.log" % i
+                if os.path.exists(logfile):
+                    profiles.append(logfile)
+
+            pout = "profile-parse.log.processed"
+            bb.utils.process_profilelog(profiles, pout = pout)
+            print("Processed parsing statistics saved to %s" % (pout))
 
     def load_cached(self):
         for filename, appends in self.fromcache:

bitbake/lib/bb/cookerdata.py

@@ -173,9 +173,12 @@ def catch_parse_error(func):
     def wrapped(fn, *args):
         try:
             return func(fn, *args)
-        except (IOError, bb.parse.ParseError, bb.data_smart.ExpansionError) as exc:
+        except IOError as exc:
             import traceback
-            parselog.critical( traceback.format_exc())
+            parselog.critical(traceback.format_exc())
+            parselog.critical("Unable to parse %s: %s" % (fn, exc))
+            sys.exit(1)
+        except (bb.parse.ParseError, bb.data_smart.ExpansionError) as exc:
             parselog.critical("Unable to parse %s: %s" % (fn, exc))
             sys.exit(1)
     return wrapped
@@ -269,8 +272,11 @@ class CookerDataBuilder(object):
             layers = (data.getVar('BBLAYERS', True) or "").split()
 
             data = bb.data.createCopy(data)
+            approved = bb.utils.approved_variables()
             for layer in layers:
                 parselog.debug(2, "Adding layer %s", layer)
+                if 'HOME' in approved and '~' in layer:
+                    layer = os.path.expanduser(layer)
                 data.setVar('LAYERDIR', layer)
                 data = parse_config_file(os.path.join(layer, "conf", "layer.conf"), data)
                 data.expandVarref('LAYERDIR')
@@ -298,15 +304,15 @@ class CookerDataBuilder(object):
 
         # Nomally we only register event handlers at the end of parsing .bb files
         # We register any handlers we've found so far here...
-        for var in data.getVar('__BBHANDLERS') or []:
-            bb.event.register(var, data.getVar(var),  (data.getVarFlag(var, "eventmask", True) or "").split())
+        for var in data.getVar('__BBHANDLERS', False) or []:
+            bb.event.register(var, data.getVar(var, False),  (data.getVarFlag(var, "eventmask", True) or "").split())
 
         if data.getVar("BB_WORKERCONTEXT", False) is None:
             bb.fetch.fetcher_init(data)
         bb.codeparser.parser_cache_init(data)
         bb.event.fire(bb.event.ConfigParsed(), data)
 
-        if data.getVar("BB_INVALIDCONF") is True:
+        if data.getVar("BB_INVALIDCONF", False) is True:
             data.setVar("BB_INVALIDCONF", False)
             self.parseConfigurationFiles(self.prefiles, self.postfiles)
             return

bitbake/lib/bb/data.py

@@ -84,7 +84,7 @@ def setVar(var, value, d):
     d.setVar(var, value)
 
 
-def getVar(var, d, exp = 0):
+def getVar(var, d, exp = False):
     """Gets the value of a variable"""
     return d.getVar(var, exp)
 
@@ -159,13 +159,12 @@ def expandKeys(alterdata, readdata = None):
 
     # These two for loops are split for performance to maximise the
     # usefulness of the expand cache
-
-    for key in todolist:
+    for key in sorted(todolist):
         ekey = todolist[key]
-        newval = alterdata.getVar(ekey, 0)
-        if newval:
-            val = alterdata.getVar(key, 0)
-            if val is not None and newval is not None:
+        newval = alterdata.getVar(ekey, False)
+        if newval is not None:
+            val = alterdata.getVar(key, False)
+            if val is not None:
                 bb.warn("Variable key %s (%s) replaces original key %s (%s)." % (key, val, ekey, newval))
         alterdata.renameVar(key, ekey)
 
@@ -175,7 +174,7 @@ def inheritFromOS(d, savedenv, permitted):
     for s in savedenv.keys():
         if s in permitted:
             try:
-                d.setVar(s, getVar(s, savedenv, True), op = 'from env')
+                d.setVar(s, savedenv.getVar(s, True), op = 'from env')
                 if s in exportlist:
                     d.setVarFlag(s, "export", True, op = 'auto env export')
             except TypeError:
@@ -183,39 +182,39 @@ def inheritFromOS(d, savedenv, permitted):
 
 def emit_var(var, o=sys.__stdout__, d = init(), all=False):
     """Emit a variable to be sourced by a shell."""
-    if getVarFlag(var, "python", d):
-        return 0
+    if d.getVarFlag(var, "python"):
+        return False
 
-    export = getVarFlag(var, "export", d)
-    unexport = getVarFlag(var, "unexport", d)
-    func = getVarFlag(var, "func", d)
+    export = d.getVarFlag(var, "export")
+    unexport = d.getVarFlag(var, "unexport")
+    func = d.getVarFlag(var, "func")
     if not all and not export and not unexport and not func:
-        return 0
+        return False
 
     try:
         if all:
-            oval = getVar(var, d, 0)
-        val = getVar(var, d, 1)
+            oval = d.getVar(var, False)
+        val = d.getVar(var, True)
     except (KeyboardInterrupt, bb.build.FuncFailed):
         raise
     except Exception as exc:
         o.write('# expansion of %s threw %s: %s\n' % (var, exc.__class__.__name__, str(exc)))
-        return 0
+        return False
 
     if all:
-        d.varhistory.emit(var, oval, val, o)
+        d.varhistory.emit(var, oval, val, o, d)
 
     if (var.find("-") != -1 or var.find(".") != -1 or var.find('{') != -1 or var.find('}') != -1 or var.find('+') != -1) and not all:
-        return 0
+        return False
 
-    varExpanded = expand(var, d)
+    varExpanded = d.expand(var)
 
     if unexport:
         o.write('unset %s\n' % varExpanded)
-        return 0
+        return False
 
     if val is None:
-        return 0
+        return False
 
     val = str(val)
 
@@ -224,7 +223,7 @@ def emit_var(var, o=sys.__stdout__, d = init(), all=False):
         val = val[3:] # Strip off "() "
         o.write("%s() %s\n" % (varExpanded, val))
         o.write("export -f %s\n" % (varExpanded))
-        return 1
+        return True
 
     if func:
         # NOTE: should probably check for unbalanced {} within the var
@@ -240,7 +239,7 @@ def emit_var(var, o=sys.__stdout__, d = init(), all=False):
     alter = re.sub('\n', ' \\\n', alter)
     alter = re.sub('\\$', '\\\\$', alter)
     o.write('%s="%s"\n' % (varExpanded, alter))
-    return 0
+    return False
 
 def emit_env(o=sys.__stdout__, d = init(), all=False):
     """Emits all items in the data store in a format such that it can be sourced by a shell."""
@@ -272,8 +271,9 @@ def emit_func(func, o=sys.__stdout__, d = init()):
 
     keys = (key for key in d.keys() if not key.startswith("__") and not d.getVarFlag(key, "func"))
     for key in keys:
-        emit_var(key, o, d, False) and o.write('\n')
+        emit_var(key, o, d, False)
 
+    o.write('\n')
     emit_var(func, o, d, False) and o.write('\n')
     newdeps = bb.codeparser.ShellParser(func, logger).parse_shell(d.getVar(func, True))
     newdeps |= set((d.getVarFlag(func, "vardeps", True) or "").split())
@@ -420,7 +420,7 @@ def generate_dependencies(d):
     deps = {}
     values = {}
 
-    tasklist = d.getVar('__BBTASKS') or []
+    tasklist = d.getVar('__BBTASKS', False) or []
     for task in tasklist:
         deps[task], values[task] = build_dependencies(task, keys, shelldeps, varflagsexcl, d)
         newdeps = deps[task]
@@ -438,7 +438,7 @@ def generate_dependencies(d):
     return tasklist, deps, values
 
 def inherits_class(klass, d):
-    val = getVar('__inherit_cache', d) or []
+    val = d.getVar('__inherit_cache', False) or []
     needle = os.path.join('classes', '%s.bbclass' % klass)
     for v in val:
         if v.endswith(needle):

bitbake/lib/bb/data_smart.py

@@ -54,27 +54,36 @@ def infer_caller_details(loginfo, parent = False, varval = True):
         return
     # Infer caller's likely values for variable (var) and value (value), 
     # to reduce clutter in the rest of the code.
-    if varval and ('variable' not in loginfo or 'detail' not in loginfo):
+    above = None
+    def set_above():
         try:
             raise Exception
         except Exception:
             tb = sys.exc_info()[2]
             if parent:
-                above = tb.tb_frame.f_back.f_back
+                return tb.tb_frame.f_back.f_back.f_back
             else:
-                above = tb.tb_frame.f_back
-            lcls = above.f_locals.items()
+                return tb.tb_frame.f_back.f_back
+
+    if varval and ('variable' not in loginfo or 'detail' not in loginfo):
+        if not above:
+            above = set_above()
+        lcls = above.f_locals.items()
         for k, v in lcls:
             if k == 'value' and 'detail' not in loginfo:
                 loginfo['detail'] = v
             if k == 'var' and 'variable' not in loginfo:
                 loginfo['variable'] = v
     # Infer file/line/function from traceback
+    # Don't use traceback.extract_stack() since it fills the line contents which
+    # we don't need and that hits stat syscalls
     if 'file' not in loginfo:
-        depth = 3    
-        if parent:
-            depth = 4
-        file, line, func, text = traceback.extract_stack(limit = depth)[0]
+        if not above:
+            above = set_above()
+        f = above.f_back
+        line = f.f_lineno
+        file = f.f_code.co_filename
+        func = f.f_code.co_name
         loginfo['file'] = file
         loginfo['line'] = line
         if func not in loginfo:
@@ -231,6 +240,10 @@ class VariableHistory(object):
 
         if var not in self.variables:
             self.variables[var] = []
+        if not isinstance(self.variables[var], list):
+            return
+        if 'nodups' in loginfo and loginfo in self.variables[var]:
+            return
         self.variables[var].append(loginfo.copy())
 
     def variable(self, var):
@@ -239,8 +252,20 @@ class VariableHistory(object):
         else:
             return []
 
-    def emit(self, var, oval, val, o):
+    def emit(self, var, oval, val, o, d):
         history = self.variable(var)
+
+        # Append override history
+        if var in d.overridedata:
+            for (r, override) in d.overridedata[var]:
+                for event in self.variable(r):
+                    loginfo = event.copy()
+                    if 'flag' in loginfo and not loginfo['flag'].startswith("_"):
+                        continue
+                    loginfo['variable'] = var
+                    loginfo['op'] = 'override[%s]:%s' % (override, loginfo['op'])
+                    history.append(loginfo)
+
         commentVal = re.sub('\n', '\n#', str(oval))
         if history:
             if len(history) == 1:
@@ -296,24 +321,24 @@ class VariableHistory(object):
                 self.variables[var] = []
 
 class DataSmart(MutableMapping):
-    def __init__(self, special = None, seen = None ):
+    def __init__(self):
         self.dict = {}
 
-        if special is None:
-            special = COWDictBase.copy()
-        if seen is None:
-            seen = COWDictBase.copy()
-
         self.inchistory = IncludeHistory()
         self.varhistory = VariableHistory(self)
         self._tracking = False
 
-        # cookie monster tribute
-        self._special_values = special
-        self._seen_overrides = seen
-
         self.expand_cache = {}
 
+        # cookie monster tribute
+        # Need to be careful about writes to overridedata as
+        # its only a shallow copy, could influence other data store
+        # copies!
+        self.overridedata = {}
+        self.overrides = None
+        self.overridevars = set(["OVERRIDES", "FILE"])
+        self.inoverride = False
+
     def enableTracking(self):
         self._tracking = True
 
@@ -354,97 +379,25 @@ class DataSmart(MutableMapping):
     def expand(self, s, varname = None):
         return self.expandWithRefs(s, varname).value
 
-
     def finalize(self, parent = False):
+        return
+
+    def internal_finalize(self, parent = False):
         """Performs final steps upon the datastore, including application of overrides"""
+        self.overrides = None
 
-        overrides = (self.getVar("OVERRIDES", True) or "").split(":") or []
-        finalize_caller = {
-            'op': 'finalize',
-        }
-        infer_caller_details(finalize_caller, parent = parent, varval = False)
-
-        #
-        # Well let us see what breaks here. We used to iterate
-        # over each variable and apply the override and then
-        # do the line expanding.
-        # If we have bad luck - which we will have - the keys
-        # where in some order that is so important for this
-        # method which we don't have anymore.
-        # Anyway we will fix that and write test cases this
-        # time.
-
-        #
-        # First we apply all overrides
-        # Then we will handle _append and _prepend and store the _remove
-        # information for later.
-        #
-
-        # We only want to report finalization once per variable overridden.
-        finalizes_reported = {}
-
-        for o in overrides:
-            # calculate '_'+override
-            l = len(o) + 1
-
-            # see if one should even try
-            if o not in self._seen_overrides:
-                continue
-
-            vars = self._seen_overrides[o].copy()
-            for var in vars:
-                name = var[:-l]
-                try:
-                    # Report only once, even if multiple changes.
-                    if name not in finalizes_reported:
-                        finalizes_reported[name] = True
-                        finalize_caller['variable'] = name
-                        finalize_caller['detail'] = 'was: ' + str(self.getVar(name, False))
-                        self.varhistory.record(**finalize_caller)
-                    # Copy history of the override over.
-                    for event in self.varhistory.variable(var):
-                        loginfo = event.copy()
-                        loginfo['variable'] = name
-                        loginfo['op'] = 'override[%s]:%s' % (o, loginfo['op'])
-                        self.varhistory.record(**loginfo)
-                    self.setVar(name, self.getVar(var, False), op = 'finalize', file = 'override[%s]' % o, line = '')
-                    self.delVar(var)
-                except Exception:
-                    logger.info("Untracked delVar")
-
-        # now on to the appends and prepends, and stashing the removes
-        for op in __setvar_keyword__:
-            if op in self._special_values:
-                appends = self._special_values[op] or []
-                for append in appends:
-                    keep = []
-                    for (a, o) in self.getVarFlag(append, op) or []:
-                        match = True
-                        if o:
-                            for o2 in o.split("_"):
-                                if not o2 in overrides:
-                                    match = False
-                        if not match:
-                            keep.append((a ,o))
-                            continue
-
-                        if op == "_append":
-                            sval = self.getVar(append, False) or ""
-                            sval += a
-                            self.setVar(append, sval)
-                        elif op == "_prepend":
-                            sval = a + (self.getVar(append, False) or "")
-                            self.setVar(append, sval)
-                        elif op == "_remove":
-                            removes = self.getVarFlag(append, "_removeactive", False) or []
-                            removes.extend(a.split())
-                            self.setVarFlag(append, "_removeactive", removes, ignore=True)
-
-                    # We save overrides that may be applied at some later stage
-                    if keep:
-                        self.setVarFlag(append, op, keep, ignore=True)
-                    else:
-                        self.delVarFlag(append, op, ignore=True)
+    def need_overrides(self):
+        if self.overrides is None:
+            if self.inoverride:
+                return
+            self.inoverride = True
+            # Can end up here recursively so setup dummy values
+            self.overrides = []
+            self.overridesset = set()
+            self.overrides = (self.getVar("OVERRIDES", True) or "").split(":") or []
+            self.overridesset = set(self.overrides)
+            self.inoverride = False
+            self.expand_cache = {}
 
     def initVar(self, var):
         self.expand_cache = {}
@@ -475,6 +428,10 @@ class DataSmart(MutableMapping):
 
     def setVar(self, var, value, **loginfo):
         #print("var=" + str(var) + "  val=" + str(value))
+        parsing=False
+        if 'parsing' in loginfo:
+            parsing=True
+
         if 'op' not in loginfo:
             loginfo['op'] = "set"
         self.expand_cache = {}
@@ -496,52 +453,82 @@ class DataSmart(MutableMapping):
             self.varhistory.record(**loginfo)
             # todo make sure keyword is not __doc__ or __module__
             # pay the cookie monster
-            try:
-                self._special_values[keyword].add(base)
-            except KeyError:
-                self._special_values[keyword] = set()
-                self._special_values[keyword].add(base)
 
+            # more cookies for the cookie monster
+            if '_' in var:
+                self._setvar_update_overrides(base, **loginfo)
+
+
+            if base in self.overridevars:
+                self.overridevars.update(self.expandWithRefs(value, var).references)
+                self.internal_finalize(True)
             return
 
         if not var in self.dict:
             self._makeShadowCopy(var)
 
+        if not parsing:
+            if "_append" in self.dict[var]:
+                del self.dict[var]["_append"]
+            if "_prepend" in self.dict[var]:
+                del self.dict[var]["_prepend"]
+            if var in self.overridedata:
+                active = []
+                self.need_overrides()
+                for (r, o) in self.overridedata[var]:
+                    if o in self.overridesset:
+                        active.append(r)
+                    elif "_" in o:
+                        if set(o.split("_")).issubset(self.overridesset):
+                            active.append(r)
+                for a in active:
+                    self.delVar(a)
+                del self.overridedata[var]
+
         # more cookies for the cookie monster
         if '_' in var:
-            self._setvar_update_overrides(var)
+            self._setvar_update_overrides(var, **loginfo)
 
         # setting var
         self.dict[var]["_content"] = value
         self.varhistory.record(**loginfo)
 
-    def _setvar_update_overrides(self, var):
+        if var in self.overridevars:
+            self.overridevars.update(self.expandWithRefs(value, var).references)
+            self.internal_finalize(True)
+
+    def _setvar_update_overrides(self, var, **loginfo):
         # aka pay the cookie monster
         override = var[var.rfind('_')+1:]
         shortvar = var[:var.rfind('_')]
         while override:
-            if override not in self._seen_overrides:
-                self._seen_overrides[override] = set()
-            self._seen_overrides[override].add( var )
+            if shortvar not in self.overridedata:
+                self.overridedata[shortvar] = []
+            if [var, override] not in self.overridedata[shortvar]:
+                # Force CoW by recreating the list first
+                self.overridedata[shortvar] = list(self.overridedata[shortvar])
+                self.overridedata[shortvar].append([var, override])
             override = None
             if "_" in shortvar:
                 override = var[shortvar.rfind('_')+1:]
                 shortvar = var[:shortvar.rfind('_')]
+                if len(shortvar) == 0:
+                    override = None
 
-    def getVar(self, var, expand=False, noweakdefault=False):
-        return self.getVarFlag(var, "_content", expand, noweakdefault)
+    def getVar(self, var, expand=False, noweakdefault=False, parsing=False):
+        return self.getVarFlag(var, "_content", expand, noweakdefault, parsing)
 
     def renameVar(self, key, newkey, **loginfo):
         """
         Rename the variable key to newkey
         """
-        val = self.getVar(key, 0)
+        val = self.getVar(key, 0, parsing=True)
         if val is not None:
             loginfo['variable'] = newkey
             loginfo['op'] = 'rename from %s' % key
             loginfo['detail'] = val
             self.varhistory.record(**loginfo)
-            self.setVar(newkey, val, ignore=True)
+            self.setVar(newkey, val, ignore=True, parsing=True)
 
         for i in (__setvar_keyword__):
             src = self.getVarFlag(key, i)
@@ -552,9 +539,14 @@ class DataSmart(MutableMapping):
             dest.extend(src)
             self.setVarFlag(newkey, i, dest, ignore=True)
 
-            if i in self._special_values and key in self._special_values[i]:
-                self._special_values[i].remove(key)
-                self._special_values[i].add(newkey)
+        if key in self.overridedata:
+            self.overridedata[newkey] = []
+            for (v, o) in self.overridedata[key]:
+                self.overridedata[newkey].append([v.replace(key, newkey), o])
+                self.renameVar(v, v.replace(key, newkey))
+
+        if '_' in newkey and val is None:
+            self._setvar_update_overrides(newkey, **loginfo)
 
         loginfo['variable'] = key
         loginfo['op'] = 'rename (to)'
@@ -565,14 +557,12 @@ class DataSmart(MutableMapping):
     def appendVar(self, var, value, **loginfo):
         loginfo['op'] = 'append'
         self.varhistory.record(**loginfo)
-        newvalue = (self.getVar(var, False) or "") + value
-        self.setVar(var, newvalue, ignore=True)
+        self.setVar(var + "_append", value, ignore=True, parsing=True)
 
     def prependVar(self, var, value, **loginfo):
         loginfo['op'] = 'prepend'
         self.varhistory.record(**loginfo)
-        newvalue = value + (self.getVar(var, False) or "")
-        self.setVar(var, newvalue, ignore=True)
+        self.setVar(var + "_prepend", value, ignore=True, parsing=True)
 
     def delVar(self, var, **loginfo):
         loginfo['detail'] = ""
@@ -580,12 +570,28 @@ class DataSmart(MutableMapping):
         self.varhistory.record(**loginfo)
         self.expand_cache = {}
         self.dict[var] = {}
+        if var in self.overridedata:
+            del self.overridedata[var]
         if '_' in var:
             override = var[var.rfind('_')+1:]
-            if override and override in self._seen_overrides and var in self._seen_overrides[override]:
-                self._seen_overrides[override].remove(var)
+            shortvar = var[:var.rfind('_')]
+            while override:
+                try:
+                    if shortvar in self.overridedata:
+                        # Force CoW by recreating the list first
+                        self.overridedata[shortvar] = list(self.overridedata[shortvar])
+                        self.overridedata[shortvar].remove([var, override])
+                except ValueError as e:
+                    pass
+                override = None
+                if "_" in shortvar:
+                    override = var[shortvar.rfind('_')+1:]
+                    shortvar = var[:shortvar.rfind('_')]
+                    if len(shortvar) == 0:
+                         override = None
 
     def setVarFlag(self, var, flag, value, **loginfo):
+        self.expand_cache = {}
         if 'op' not in loginfo:
             loginfo['op'] = "set"
         loginfo['flag'] = flag
@@ -595,7 +601,7 @@ class DataSmart(MutableMapping):
         self.dict[var][flag] = value
 
         if flag == "_defaultval" and '_' in var:
-            self._setvar_update_overrides(var)
+            self._setvar_update_overrides(var, **loginfo)
 
         if flag == "unexport" or flag == "export":
             if not "__exportlist" in self.dict:
@@ -604,14 +610,71 @@ class DataSmart(MutableMapping):
                 self.dict["__exportlist"]["_content"] = set()
             self.dict["__exportlist"]["_content"].add(var)
 
-    def getVarFlag(self, var, flag, expand=False, noweakdefault=False):
+    def getVarFlag(self, var, flag, expand=False, noweakdefault=False, parsing=False):
         local_var = self._findVar(var)
         value = None
-        if local_var is not None:
+        if flag == "_content" and var in self.overridedata and not parsing:
+            match = False
+            active = {}
+            self.need_overrides()
+            for (r, o) in self.overridedata[var]:
+                # What about double overrides both with "_" in the name?
+                if o in self.overridesset:
+                    active[o] = r
+                elif "_" in o:
+                    if set(o.split("_")).issubset(self.overridesset):
+                        active[o] = r
+
+            mod = True
+            while mod:
+                mod = False
+                for o in self.overrides:
+                    for a in active.copy():
+                        if a.endswith("_" + o):
+                            t = active[a]
+                            del active[a]
+                            active[a.replace("_" + o, "")] = t
+                            mod = True
+                        elif a == o:
+                            match = active[a]
+                            del active[a]
+            if match:
+                value = self.getVar(match)
+
+        if local_var is not None and value is None:
             if flag in local_var:
                 value = copy.copy(local_var[flag])
             elif flag == "_content" and "_defaultval" in local_var and not noweakdefault:
                 value = copy.copy(local_var["_defaultval"])
+
+
+        if flag == "_content" and local_var is not None and "_append" in local_var and not parsing:
+            if not value:
+                value = ""
+            self.need_overrides()
+            for (r, o) in local_var["_append"]:
+                match = True
+                if o:
+                    for o2 in o.split("_"):
+                        if not o2 in self.overrides:
+                            match = False                            
+                if match:
+                    value = value + r
+
+        if flag == "_content" and local_var is not None and "_prepend" in local_var and not parsing:
+            if not value:
+                value = ""
+            self.need_overrides()
+            for (r, o) in local_var["_prepend"]:
+
+                match = True
+                if o:
+                    for o2 in o.split("_"):
+                        if not o2 in self.overrides:
+                            match = False                            
+                if match:
+                    value = r + value
+
         if expand and value:
             # Only getvar (flag == _content) hits the expand cache
             cachename = None
@@ -620,19 +683,30 @@ class DataSmart(MutableMapping):
             else:
                 cachename = var + "[" + flag + "]"
             value = self.expand(value, cachename)
-        if value and flag == "_content" and local_var is not None and "_removeactive" in local_var:
-            removes = [self.expand(r).split()  for r in local_var["_removeactive"]]
-            removes = reduce(lambda a, b: a+b, removes, [])
+
+        if value and flag == "_content" and local_var is not None and "_remove" in local_var:
+            removes = []
+            self.need_overrides()
+            for (r, o) in local_var["_remove"]:
+                match = True
+                if o:
+                    for o2 in o.split("_"):
+                        if not o2 in self.overrides:
+                            match = False                            
+                if match:
+                    removes.extend(self.expand(r).split())
+
             filtered = filter(lambda v: v not in removes,
                               value.split())
             value = " ".join(filtered)
-            if expand:
+            if expand and var in self.expand_cache:
                  # We need to ensure the expand cache has the correct value
                  # flag == "_content" here
                 self.expand_cache[var].value = value
         return value
 
     def delVarFlag(self, var, flag, **loginfo):
+        self.expand_cache = {}
         local_var = self._findVar(var)
         if not local_var:
             return
@@ -662,6 +736,7 @@ class DataSmart(MutableMapping):
         self.setVarFlag(var, flag, newvalue, ignore=True)
 
     def setVarFlags(self, var, flags, **loginfo):
+        self.expand_cache = {}
         infer_caller_details(loginfo)
         if not var in self.dict:
             self._makeShadowCopy(var)
@@ -691,6 +766,7 @@ class DataSmart(MutableMapping):
 
 
     def delVarFlags(self, var, **loginfo):
+        self.expand_cache = {}
         if not var in self.dict:
             self._makeShadowCopy(var)
 
@@ -708,13 +784,12 @@ class DataSmart(MutableMapping):
             else:
                 del self.dict[var]
 
-
     def createCopy(self):
         """
         Create a copy of self by setting _data to self
         """
         # we really want this to be a DataSmart...
-        data = DataSmart(seen=self._seen_overrides.copy(), special=self._special_values.copy())
+        data = DataSmart()
         data.dict["_data"] = self.dict
         data.varhistory = self.varhistory.copy()
         data.varhistory.datasmart = data
@@ -722,6 +797,12 @@ class DataSmart(MutableMapping):
 
         data._tracking = self._tracking
 
+        data.overrides = None
+        data.overridevars = copy.copy(self.overridevars)
+        # Should really be a deepcopy but has heavy overhead.
+        # Instead, we're careful with writes.
+        data.overridedata = copy.copy(self.overridedata)
+
         return data
 
     def expandVarref(self, variable, parents=False):
@@ -747,6 +828,7 @@ class DataSmart(MutableMapping):
 
     def __iter__(self):
         deleted = set()
+        overrides = set()
         def keylist(d):        
             klist = set()
             for key in d:
@@ -754,6 +836,8 @@ class DataSmart(MutableMapping):
                     continue
                 if key in deleted:
                     continue
+                if key in overrides:
+                    continue
                 if not d[key]:
                     deleted.add(key)
                     continue
@@ -764,9 +848,21 @@ class DataSmart(MutableMapping):
 
             return klist
 
+        self.need_overrides()
+        for var in self.overridedata:
+            for (r, o) in self.overridedata[var]:
+                if o in self.overridesset:
+                    overrides.add(var)
+                elif "_" in o:
+                    if set(o.split("_")).issubset(self.overridesset):
+                        overrides.add(var)
+
         for k in keylist(self.dict):
              yield k
 
+        for k in overrides:
+             yield k
+
     def __len__(self):
         return len(frozenset(self))
 

bitbake/lib/bb/event.py

@@ -72,6 +72,10 @@ _eventfilter = None
 
 def execute_handler(name, handler, event, d):
     event.data = d
+    addedd = False
+    if 'd' not in __builtins__:
+        __builtins__['d'] = d
+        addedd = True
     try:
         ret = handler(event)
     except (bb.parse.SkipRecipe, bb.BBHandledException):
@@ -87,6 +91,8 @@ def execute_handler(name, handler, event, d):
         raise
     finally:
         del event.data
+        if addedd:
+            del __builtins__['d']
 
 def fire_class_handlers(event, d):
     if isinstance(event, logging.LogRecord):
@@ -497,6 +503,16 @@ class TargetsTreeGenerated(Event):
         Event.__init__(self)
         self._model = model
 
+class ReachableStamps(Event):
+    """
+    An event listing all stamps reachable after parsing
+    which the metadata may use to clean up stale data
+    """
+
+    def __init__(self, stamps):
+        Event.__init__(self)
+        self.stamps = stamps
+
 class FilesMatchingFound(Event):
     """
     Event when a list of files matching the supplied pattern has

bitbake/lib/bb/fetch2/__init__.py

@@ -61,6 +61,17 @@ class BBFetchException(Exception):
     def __str__(self):
          return self.msg
 
+class UntrustedUrl(BBFetchException):
+    """Exception raised when encountering a host not listed in BB_ALLOWED_NETWORKS"""
+    def __init__(self, url, message=''):
+        if message:
+            msg = message
+        else:
+            msg = "The URL: '%s' is not trusted and cannot be used" % url
+        self.url = url
+        BBFetchException.__init__(self, msg)
+        self.args = (url,)
+
 class MalformedUrl(BBFetchException):
     """Exception raised when encountering an invalid url"""
     def __init__(self, url, message=''):
@@ -709,7 +720,7 @@ def get_autorev(d):
         d.setVar('__BB_DONT_CACHE', '1')
     return "AUTOINC"
 
-def get_srcrev(d):
+def get_srcrev(d, method_name='sortable_revision'):
     """
     Return the revsion string, usually for use in the version string (PV) of the current package
     Most packages usually only have one SCM so we just pass on the call.
@@ -718,6 +729,9 @@ def get_srcrev(d):
 
     The idea here is that we put the string "AUTOINC+" into return value if the revisions are not 
     incremental, other code is then responsible for turning that into an increasing value (if needed)
+
+    A method_name can be supplied to retrieve an alternatively formatted revision from a fetcher, if
+    that fetcher provides a method with the given name and the same signature as sortable_revision.
     """
 
     scms = []
@@ -731,7 +745,7 @@ def get_srcrev(d):
         raise FetchError("SRCREV was used yet no valid SCM was found in SRC_URI")
 
     if len(scms) == 1 and len(urldata[scms[0]].names) == 1:
-        autoinc, rev = urldata[scms[0]].method.sortable_revision(urldata[scms[0]], d, urldata[scms[0]].names[0])
+        autoinc, rev = getattr(urldata[scms[0]].method, method_name)(urldata[scms[0]], d, urldata[scms[0]].names[0])
         if len(rev) > 10:
             rev = rev[:10]
         if autoinc:
@@ -749,7 +763,7 @@ def get_srcrev(d):
     for scm in scms:
         ud = urldata[scm]
         for name in ud.names:
-            autoinc, rev = ud.method.sortable_revision(ud, d, name)
+            autoinc, rev = getattr(ud.method, method_name)(ud, d, name)
             seenautoinc = seenautoinc or autoinc
             if len(rev) > 10:
                 rev = rev[:10]
@@ -783,6 +797,8 @@ def runfetchcmd(cmd, d, quiet = False, cleanup = []):
                   'NO_PROXY', 'no_proxy',
                   'ALL_PROXY', 'all_proxy',
                   'GIT_PROXY_COMMAND',
+                  'GIT_SSL_CAINFO',
+                  'GIT_SMART_HTTP',
                   'SSH_AUTH_SOCK', 'SSH_AGENT_PID',
                   'SOCKS5_USER', 'SOCKS5_PASSWD']
 
@@ -851,6 +867,11 @@ def build_mirroruris(origud, mirrors, ld):
             newuri = uri_replace(ud, find, replace, replacements, ld)
             if not newuri or newuri in uris or newuri == origud.url:
                 continue
+
+            if not trusted_network(ld, newuri):
+                logger.debug(1, "Mirror %s not in the list of trusted networks, skipping" %  (newuri))
+                continue
+
             try:
                 newud = FetchData(newuri, ld)
                 newud.setup_localpath(ld)
@@ -858,7 +879,9 @@ def build_mirroruris(origud, mirrors, ld):
                 logger.debug(1, "Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url))
                 logger.debug(1, str(e))
                 try:
-                    ud.method.clean(ud, ld)
+                    # setup_localpath of file:// urls may fail, we should still see 
+                    # if mirrors of the url exist
+                    adduri(newud, uris, uds)
                 except UnboundLocalError:
                     pass
                 continue   
@@ -884,12 +907,12 @@ def rename_bad_checksum(ud, suffix):
     bb.utils.movefile(ud.localpath, new_localpath)
 
 
-def try_mirror_url(origud, ud, ld, check = False):
+def try_mirror_url(fetch, origud, ud, ld, check = False):
     # Return of None or a value means we're finished
     # False means try another url
     try:
         if check:
-            found = ud.method.checkstatus(ud, ld)
+            found = ud.method.checkstatus(fetch, ud, ld)
             if found:
                 return found
             return False
@@ -952,7 +975,7 @@ def try_mirror_url(origud, ud, ld, check = False):
             pass
         return False
 
-def try_mirrors(d, origud, mirrors, check = False):
+def try_mirrors(fetch, d, origud, mirrors, check = False):
     """
     Try to use a mirrored version of the sources.
     This method will be automatically called before the fetchers go.
@@ -966,11 +989,46 @@ def try_mirrors(d, origud, mirrors, check = False):
     uris, uds = build_mirroruris(origud, mirrors, ld)
 
     for index, uri in enumerate(uris):
-        ret = try_mirror_url(origud, uds[index], ld, check)
+        ret = try_mirror_url(fetch, origud, uds[index], ld, check)
         if ret != False:
             return ret
     return None
 
+def trusted_network(d, url):
+    """
+    Use a trusted url during download if networking is enabled and
+    BB_ALLOWED_NETWORKS is set globally or for a specific recipe.
+    Note: modifies SRC_URI & mirrors.
+    """
+    if d.getVar('BB_NO_NETWORK', True) == "1":
+        return True
+
+    pkgname = d.expand(d.getVar('PN', False))
+    trusted_hosts = d.getVarFlag('BB_ALLOWED_NETWORKS', pkgname)
+
+    if not trusted_hosts:
+        trusted_hosts = d.getVar('BB_ALLOWED_NETWORKS', True)
+
+    # Not enabled.
+    if not trusted_hosts:
+        return True
+
+    scheme, network, path, user, passwd, param = decodeurl(url)
+
+    if not network:
+        return True
+
+    network = network.lower()
+
+    for host in trusted_hosts.split(" "):
+        host = host.lower()
+        if host.startswith("*.") and ("." + network).endswith(host[1:]):
+            return True
+        if host == network:
+            return True
+
+    return False
+
 def srcrev_internal_helper(ud, d, name):
     """
     Return:
@@ -1415,7 +1473,7 @@ class FetchMethod(object):
         """
         return True
 
-    def checkstatus(self, urldata, d):
+    def checkstatus(self, fetch, urldata, d):
         """
         Check the status of a URL
         Assumes localpath was called first
@@ -1447,7 +1505,7 @@ class FetchMethod(object):
         return "%s-%s" % (key, d.getVar("PN", True) or "")
 
 class Fetch(object):
-    def __init__(self, urls, d, cache = True, localonly = False):
+    def __init__(self, urls, d, cache = True, localonly = False, connection_cache = None):
         if localonly and cache:
             raise Exception("bb.fetch2.Fetch.__init__: cannot set cache and localonly at same time")
 
@@ -1456,6 +1514,7 @@ class Fetch(object):
         self.urls = urls
         self.d = d
         self.ud = {}
+        self.connection_cache = connection_cache
 
         fn = d.getVar('FILE', True)
         if cache and fn and fn in urldata_cache:
@@ -1519,7 +1578,7 @@ class Fetch(object):
                 elif m.try_premirror(ud, self.d):
                     logger.debug(1, "Trying PREMIRRORS")
                     mirrors = mirror_from_string(self.d.getVar('PREMIRRORS', True))
-                    localpath = try_mirrors(self.d, ud, mirrors, False)
+                    localpath = try_mirrors(self, self.d, ud, mirrors, False)
 
                 if premirroronly:
                     self.d.setVar("BB_NO_NETWORK", "1")
@@ -1529,6 +1588,8 @@ class Fetch(object):
                 firsterr = None
                 if not localpath and ((not verify_donestamp(ud, self.d)) or m.need_update(ud, self.d)):
                     try:
+                        if not trusted_network(self.d, ud.url):
+                            raise UntrustedUrl(ud.url)
                         logger.debug(1, "Trying Upstream")
                         m.download(ud, self.d)
                         if hasattr(m, "build_mirror_data"):
@@ -1556,7 +1617,7 @@ class Fetch(object):
                         m.clean(ud, self.d)
                         logger.debug(1, "Trying MIRRORS")
                         mirrors = mirror_from_string(self.d.getVar('MIRRORS', True))
-                        localpath = try_mirrors (self.d, ud, mirrors)
+                        localpath = try_mirrors(self, self.d, ud, mirrors)
 
                 if not localpath or ((not os.path.exists(localpath)) and localpath.find("*") == -1):
                     if firsterr:
@@ -1588,15 +1649,15 @@ class Fetch(object):
             logger.debug(1, "Testing URL %s", u)
             # First try checking uri, u, from PREMIRRORS
             mirrors = mirror_from_string(self.d.getVar('PREMIRRORS', True))
-            ret = try_mirrors(self.d, ud, mirrors, True)
+            ret = try_mirrors(self, self.d, ud, mirrors, True)
             if not ret:
                 # Next try checking from the original uri, u
                 try:
-                    ret = m.checkstatus(ud, self.d)
+                    ret = m.checkstatus(self, ud, self.d)
                 except:
                     # Finally, try checking uri, u, from MIRRORS
                     mirrors = mirror_from_string(self.d.getVar('MIRRORS', True))
-                    ret = try_mirrors(self.d, ud, mirrors, True)
+                    ret = try_mirrors(self, self.d, ud, mirrors, True)
 
             if not ret:
                 raise FetchError("URL %s doesn't work" % u, u)
@@ -1651,6 +1712,42 @@ class Fetch(object):
             if ud.lockfile:
                 bb.utils.unlockfile(lf)
 
+class FetchConnectionCache(object):
+    """
+        A class which represents an container for socket connections.
+    """
+    def __init__(self):
+        self.cache = {}
+
+    def get_connection_name(self, host, port):
+        return host + ':' + str(port)
+
+    def add_connection(self, host, port, connection):
+        cn = self.get_connection_name(host, port)
+
+        if cn not in self.cache:
+            self.cache[cn] = connection
+
+    def get_connection(self, host, port):
+        connection = None
+
+        cn = self.get_connection_name(host, port)
+        if cn in self.cache:
+            connection = self.cache[cn]
+
+        return connection
+
+    def remove_connection(self, host, port):
+        cn = self.get_connection_name(host, port)
+        if cn in self.cache:
+            self.cache[cn].close()
+            del self.cache[cn]
+
+    def close_connections(self):
+        for cn in self.cache.keys():
+            self.cache[cn].close()
+            del self.cache[cn]
+
 from . import cvs
 from . import git
 from . import gitsm

bitbake/lib/bb/fetch2/clearcase.py

@@ -9,7 +9,7 @@ Usage in the recipe:
 
     SRC_URI = "ccrc://cc.example.org/ccrc;vob=/example_vob;module=/example_module"
     SRCREV = "EXAMPLE_CLEARCASE_TAG"
-    PV = "${@d.getVar("SRCREV").replace("/", "+")}"
+    PV = "${@d.getVar("SRCREV", False).replace("/", "+")}"
 
 The fetcher uses the rcleartool or cleartool remote client, depending on which one is available.
 
@@ -113,7 +113,7 @@ class ClearCase(FetchMethod):
         if data.getVar("SRCREV", d, True) == "INVALID":
           raise FetchError("Set a valid SRCREV for the clearcase fetcher in your recipe, e.g. SRCREV = \"/main/LATEST\" or any other label of your choice.")
 
-        ud.label = d.getVar("SRCREV")
+        ud.label = d.getVar("SRCREV", False)
         ud.customspec = d.getVar("CCASE_CUSTOM_CONFIG_SPEC", True)
 
         ud.server     = "%s://%s%s" % (ud.proto, ud.host, ud.path)

bitbake/lib/bb/fetch2/git.py

@@ -178,11 +178,6 @@ class Git(FetchMethod):
     def download(self, ud, d):
         """Fetch url"""
 
-        if ud.user:
-            username = ud.user + '@'
-        else:
-            username = ""
-
         ud.repochanged = not os.path.exists(ud.fullmirror)
 
         # If the checkout doesn't exist and the mirror tarball does, extract it
@@ -191,7 +186,7 @@ class Git(FetchMethod):
             os.chdir(ud.clonedir)
             runfetchcmd("tar -xzf %s" % (ud.fullmirror), d)
 
-        repourl = "%s://%s%s%s" % (ud.proto, username, ud.host, ud.path)
+        repourl = self._get_repo_url(ud)
 
         # If the repo still doesn't exist, fallback to cloning it
         if not os.path.exists(ud.clonedir):
@@ -246,7 +241,7 @@ class Git(FetchMethod):
         subdir = ud.parm.get("subpath", "")
         if subdir != "":
             readpathspec = ":%s" % (subdir)
-            def_destsuffix = "%s/" % os.path.basename(subdir)
+            def_destsuffix = "%s/" % os.path.basename(subdir.rstrip('/'))
         else:
             readpathspec = ""
             def_destsuffix = "git/"
@@ -277,13 +272,22 @@ class Git(FetchMethod):
             clonedir = indirectiondir
 
         runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, cloneflags, clonedir, destdir), d)
+        os.chdir(destdir)
+        repourl = self._get_repo_url(ud)
+        runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, repourl), d)
         if not ud.nocheckout:
-            os.chdir(destdir)
             if subdir != "":
                 runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revisions[ud.names[0]], readpathspec), d)
                 runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d)
+            elif not ud.nobranch:
+                branchname =  ud.branches[ud.names[0]]
+                runfetchcmd("%s checkout -B %s %s" % (ud.basecmd, branchname, \
+                            ud.revisions[ud.names[0]]), d)
+                runfetchcmd("%s branch --set-upstream %s origin/%s" % (ud.basecmd, branchname, \
+                            branchname), d)
             else:
                 runfetchcmd("%s checkout %s" % (ud.basecmd, ud.revisions[ud.names[0]]), d)
+
         return True
 
     def clean(self, ud, d):
@@ -312,6 +316,16 @@ class Git(FetchMethod):
             raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output))
         return output.split()[0] != "0"
 
+    def _get_repo_url(self, ud):
+        """
+        Return the repository URL
+        """
+        if ud.user:
+            username = ud.user + '@'
+        else:
+            username = ""
+        return "%s://%s%s%s" % (ud.proto, username, ud.host, ud.path)
+
     def _revision_key(self, ud, d, name):
         """
         Return a unique key for the url
@@ -322,13 +336,9 @@ class Git(FetchMethod):
         """
         Run git ls-remote with the specified search string
         """
-        if ud.user:
-            username = ud.user + '@'
-        else:
-            username = ""
-
-        cmd = "%s ls-remote %s://%s%s%s %s" % \
-              (ud.basecmd, ud.proto, username, ud.host, ud.path, search)
+        repourl = self._get_repo_url(ud)
+        cmd = "%s ls-remote %s %s" % \
+              (ud.basecmd, repourl, search)
         if ud.proto.lower() != 'file':
             bb.fetch2.check_network_access(d, cmd)
         output = runfetchcmd(cmd, d, True)
@@ -352,7 +362,8 @@ class Git(FetchMethod):
             for l in output.split('\n'):
                 if s in l:
                     return l.split()[0]
-        raise bb.fetch2.FetchError("Unable to resolve '%s' in upstream git repository in git ls-remote output" % ud.unresolvedrev[name])
+        raise bb.fetch2.FetchError("Unable to resolve '%s' in upstream git repository in git ls-remote output for %s" % \
+            (ud.unresolvedrev[name], ud.host+ud.path))
 
     def latest_versionstring(self, ud, d):
         """
@@ -360,25 +371,28 @@ class Git(FetchMethod):
         by searching through the tags output of ls-remote, comparing
         versions and returning the highest match.
         """
-        verstring = ""
+        pupver = ('', '')
+
         tagregex = re.compile(d.getVar('GITTAGREGEX', True) or "(?P<pver>([0-9][\.|_]?)+)")
         try:
-            output = self._lsremote(ud, d, "refs/tags/*^{}")
+            output = self._lsremote(ud, d, "refs/tags/*")
         except bb.fetch2.FetchError or bb.fetch2.NetworkAccess:
-            return ""
+            return pupver
 
+        verstring = ""
+        revision = ""
         for line in output.split("\n"):
             if not line:
                 break
 
-            line = line.split("/")[-1]
+            tag_head = line.split("/")[-1]
             # Ignore non-released branches
-            m = re.search("(alpha|beta|rc|final)+", line)
+            m = re.search("(alpha|beta|rc|final)+", tag_head)
             if m:
                 continue
 
             # search for version in the line
-            tag = tagregex.search(line)
+            tag = tagregex.search(tag_head)
             if tag == None:
                 continue
 
@@ -387,17 +401,44 @@ class Git(FetchMethod):
 
             if verstring and bb.utils.vercmp(("0", tag, ""), ("0", verstring, "")) < 0:
                 continue
-            verstring = tag
 
-        return verstring
+            verstring = tag
+            revision = line.split()[0]
+            pupver = (verstring, revision)
+
+        return pupver
 
     def _build_revision(self, ud, d, name):
         return ud.revisions[name]
 
-    def checkstatus(self, ud, d):
-        fetchcmd = "%s ls-remote %s" % (ud.basecmd, ud.url)
+    def gitpkgv_revision(self, ud, d, name):
+        """
+        Return a sortable revision number by counting commits in the history
+        Based on gitpkgv.bblass in meta-openembedded
+        """
+        rev = self._build_revision(ud, d, name)
+        localpath = ud.localpath
+        rev_file = os.path.join(localpath, "oe-gitpkgv_" + rev)
+        if not os.path.exists(localpath):
+            commits = None
+        else:
+            if not os.path.exists(rev_file) or not os.path.getsize(rev_file):
+                from pipes import quote
+                commits = bb.fetch2.runfetchcmd(
+                        "git rev-list %s -- | wc -l" % (quote(rev)),
+                        d, quiet=True).strip().lstrip('0')
+                if commits:
+                    open(rev_file, "w").write("%d\n" % int(commits))
+            else:
+                commits = open(rev_file, "r").readline(128).strip()
+        if commits:
+            return False, "%s+%s" % (commits, rev[:7])
+        else:
+            return True, str(rev)
+
+    def checkstatus(self, fetch, ud, d):
         try:
-            runfetchcmd(fetchcmd, d, quiet=True)
+            self._lsremote(ud, d, "")
             return True
         except FetchError:
             return False

bitbake/lib/bb/fetch2/gitsm.py

@@ -109,6 +109,7 @@ class GitSM(Git):
         runfetchcmd("sed " + gitdir + "/config -i -e 's/bare.*=.*true/bare = false/'", d)
         os.chdir(tmpclonedir)
         runfetchcmd(ud.basecmd + " reset --hard", d)
+        runfetchcmd(ud.basecmd + " checkout " + ud.revisions[ud.names[0]], d)
         runfetchcmd(ud.basecmd + " submodule init", d)
         runfetchcmd(ud.basecmd + " submodule update", d)
         self._set_relative_paths(tmpclonedir)

bitbake/lib/bb/fetch2/hg.py

@@ -43,6 +43,13 @@ class Hg(FetchMethod):
         """
         return ud.type in ['hg']
 
+    def supports_checksum(self, urldata):
+        """
+        Don't require checksums for local archives created from
+        repository checkouts.
+        """ 
+        return False
+
     def urldata_init(self, ud, d):
         """
         init hg specific variable within url data
@@ -52,10 +59,12 @@ class Hg(FetchMethod):
 
         ud.module = ud.parm["module"]
 
-        # Create paths to mercurial checkouts
-        relpath = self._strip_leading_slashes(ud.path)
-        ud.pkgdir = os.path.join(data.expand('${HGDIR}', d), ud.host, relpath)
-        ud.moddir = os.path.join(ud.pkgdir, ud.module)
+        if 'protocol' in ud.parm:
+            ud.proto = ud.parm['protocol']
+        elif not ud.host:
+            ud.proto = 'file'
+        else:
+            ud.proto = "hg"
 
         ud.setup_revisons(d)
 
@@ -64,7 +73,19 @@ class Hg(FetchMethod):
         elif not ud.revision:
             ud.revision = self.latest_revision(ud, d)
 
-        ud.localfile = data.expand('%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision), d)
+        # Create paths to mercurial checkouts
+        hgsrcname = '%s_%s_%s' % (ud.module.replace('/', '.'), \
+                            ud.host, ud.path.replace('/', '.'))
+        ud.mirrortarball = 'hg_%s.tar.gz' % hgsrcname
+        ud.fullmirror = os.path.join(d.getVar("DL_DIR", True), ud.mirrortarball)
+
+        hgdir = d.getVar("HGDIR", True) or (d.getVar("DL_DIR", True) + "/hg/")
+        ud.pkgdir = os.path.join(hgdir, hgsrcname)
+        ud.moddir = os.path.join(ud.pkgdir, ud.module)
+        ud.localfile = ud.moddir
+        ud.basecmd = data.getVar("FETCHCMD_hg", d, True) or "/usr/bin/env hg"
+
+        ud.write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS", True)
 
     def need_update(self, ud, d):
         revTag = ud.parm.get('rev', 'tip')
@@ -74,14 +95,21 @@ class Hg(FetchMethod):
             return True
         return False
 
+    def try_premirror(self, ud, d):
+        # If we don't do this, updating an existing checkout with only premirrors
+        # is not possible
+        if d.getVar("BB_FETCH_PREMIRRORONLY", True) is not None:
+            return True
+        if os.path.exists(ud.moddir):
+            return False
+        return True
+
     def _buildhgcommand(self, ud, d, command):
         """
         Build up an hg commandline based on ud
         command is "fetch", "update", "info"
         """
 
-        basecmd = data.expand('${FETCHCMD_hg}', d)
-
         proto = ud.parm.get('protocol', 'http')
 
         host = ud.host
@@ -98,7 +126,7 @@ class Hg(FetchMethod):
                 hgroot = ud.user + "@" + host + ud.path
 
         if command == "info":
-            return "%s identify -i %s://%s/%s" % (basecmd, proto, hgroot, ud.module)
+            return "%s identify -i %s://%s/%s" % (ud.basecmd, proto, hgroot, ud.module)
 
         options = [];
 
@@ -111,22 +139,22 @@ class Hg(FetchMethod):
 
         if command == "fetch":
             if ud.user and ud.pswd:
-                cmd = "%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" clone %s %s://%s/%s %s" % (basecmd, ud.user, ud.pswd, proto, " ".join(options), proto, hgroot, ud.module, ud.module)
+                cmd = "%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" clone %s %s://%s/%s %s" % (ud.basecmd, ud.user, ud.pswd, proto, " ".join(options), proto, hgroot, ud.module, ud.module)
             else:
-                cmd = "%s clone %s %s://%s/%s %s" % (basecmd, " ".join(options), proto, hgroot, ud.module, ud.module)	      
+                cmd = "%s clone %s %s://%s/%s %s" % (ud.basecmd, " ".join(options), proto, hgroot, ud.module, ud.module)
         elif command == "pull":
             # do not pass options list; limiting pull to rev causes the local
             # repo not to contain it and immediately following "update" command
             # will crash
             if ud.user and ud.pswd:
-                cmd = "%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" pull" % (basecmd, ud.user, ud.pswd, proto)
+                cmd = "%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" pull" % (ud.basecmd, ud.user, ud.pswd, proto)
             else:
-                cmd = "%s pull" % (basecmd)
+                cmd = "%s pull" % (ud.basecmd)
         elif command == "update":
             if ud.user and ud.pswd:
-                cmd = "%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" update -C %s" % (basecmd, ud.user, ud.pswd, proto, " ".join(options))
+                cmd = "%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" update -C %s" % (ud.basecmd, ud.user, ud.pswd, proto, " ".join(options))
             else:
-                cmd = "%s update -C %s" % (basecmd, " ".join(options))
+                cmd = "%s update -C %s" % (ud.basecmd, " ".join(options))
         else:
             raise FetchError("Invalid hg command %s" % command, ud.url)
 
@@ -135,18 +163,36 @@ class Hg(FetchMethod):
     def download(self, ud, d):
         """Fetch url"""
 
+        ud.repochanged = not os.path.exists(ud.fullmirror)
+
         logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'")
 
+        # If the checkout doesn't exist and the mirror tarball does, extract it
+        if not os.path.exists(ud.pkgdir) and os.path.exists(ud.fullmirror):
+            bb.utils.mkdirhier(ud.pkgdir)
+            os.chdir(ud.pkgdir)
+            runfetchcmd("tar -xzf %s" % (ud.fullmirror), d)
+
         if os.access(os.path.join(ud.moddir, '.hg'), os.R_OK):
-            updatecmd = self._buildhgcommand(ud, d, "pull")
-            logger.info("Update " + ud.url)
-            # update sources there
+            # Found the source, check whether need pull
+            updatecmd = self._buildhgcommand(ud, d, "update")
             os.chdir(ud.moddir)
             logger.debug(1, "Running %s", updatecmd)
-            bb.fetch2.check_network_access(d, updatecmd, ud.url)
-            runfetchcmd(updatecmd, d)
+            try:
+                runfetchcmd(updatecmd, d)
+            except bb.fetch2.FetchError:
+                # Runnning pull in the repo
+                pullcmd = self._buildhgcommand(ud, d, "pull")
+                logger.info("Pulling " + ud.url)
+                # update sources there
+                os.chdir(ud.moddir)
+                logger.debug(1, "Running %s", pullcmd)
+                bb.fetch2.check_network_access(d, pullcmd, ud.url)
+                runfetchcmd(pullcmd, d)
+                ud.repochanged = True
 
-        else:
+        # No source found, clone it.
+        if not os.path.exists(ud.moddir):
             fetchcmd = self._buildhgcommand(ud, d, "fetch")
             logger.info("Fetch " + ud.url)
             # check out sources there
@@ -163,14 +209,12 @@ class Hg(FetchMethod):
         logger.debug(1, "Running %s", updatecmd)
         runfetchcmd(updatecmd, d)
 
-        scmdata = ud.parm.get("scmdata", "")
-        if scmdata == "keep":
-            tar_flags = ""
-        else:
-            tar_flags = "--exclude '.hg' --exclude '.hgrags'"
+    def clean(self, ud, d):
+        """ Clean the hg dir """
 
-        os.chdir(ud.pkgdir)
-        runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, ud.module), d, cleanup = [ud.localpath])
+        bb.utils.remove(ud.localpath, True)
+        bb.utils.remove(ud.fullmirror)
+        bb.utils.remove(ud.fullmirror + ".done")
 
     def supports_srcrev(self):
         return True
@@ -191,3 +235,41 @@ class Hg(FetchMethod):
         Return a unique key for the url
         """
         return "hg:" + ud.moddir
+
+    def build_mirror_data(self, ud, d):
+        # Generate a mirror tarball if needed
+        if ud.write_tarballs == "1" and (ud.repochanged or not os.path.exists(ud.fullmirror)):
+            # it's possible that this symlink points to read-only filesystem with PREMIRROR
+            if os.path.islink(ud.fullmirror):
+                os.unlink(ud.fullmirror)
+
+            os.chdir(ud.pkgdir)
+            logger.info("Creating tarball of hg repository")
+            runfetchcmd("tar -czf %s %s" % (ud.fullmirror, ud.module), d)
+            runfetchcmd("touch %s.done" % (ud.fullmirror), d)
+
+    def localpath(self, ud, d):
+        return ud.pkgdir
+
+    def unpack(self, ud, destdir, d):
+        """
+        Make a local clone or export for the url
+        """
+
+        revflag = "-r %s" % ud.revision
+        subdir = ud.parm.get("destsuffix", ud.module)
+        codir = "%s/%s" % (destdir, subdir)
+
+        scmdata = ud.parm.get("scmdata", "")
+        if scmdata != "nokeep":
+            if not os.access(os.path.join(codir, '.hg'), os.R_OK):
+                logger.debug(2, "Unpack: creating new hg repository in '" + codir + "'")
+                runfetchcmd("%s init %s" % (ud.basecmd, codir), d)
+            logger.debug(2, "Unpack: updating source in '" + codir + "'")
+            os.chdir(codir)
+            runfetchcmd("%s pull %s" % (ud.basecmd, ud.moddir), d)
+            runfetchcmd("%s up -C %s" % (ud.basecmd, revflag), d)
+        else:
+            logger.debug(2, "Unpack: extracting source to '" + codir + "'")
+            os.chdir(ud.moddir)
+            runfetchcmd("%s archive -t files %s %s" % (ud.basecmd, revflag, codir), d)

bitbake/lib/bb/fetch2/local.py

@@ -112,7 +112,7 @@ class Local(FetchMethod):
 
         return True
 
-    def checkstatus(self, urldata, d):
+    def checkstatus(self, fetch, urldata, d):
         """
         Check the status of the url
         """

bitbake/lib/bb/fetch2/perforce.py

@@ -48,7 +48,7 @@ class Perforce(FetchMethod):
             (user, pswd, host, port) = path.split('@')[0].split(":")
             path = path.split('@')[1]
         else:
-            (host, port) = d.getVar('P4PORT').split(':')
+            (host, port) = d.getVar('P4PORT', False).split(':')
             user = ""
             pswd = ""
 
@@ -123,7 +123,7 @@ class Perforce(FetchMethod):
         if depot.find('/...') != -1:
             path = depot[:depot.find('/...')]
         else:
-            path = depot
+            path = depot[:depot.rfind('/')]
 
         module = parm.get('module', os.path.basename(path))
 

bitbake/lib/bb/fetch2/sftp.py

@@ -99,7 +99,7 @@ class SFTP(FetchMethod):
         """Fetch urls"""
 
         urlo = URI(ud.url)
-        basecmd = 'sftp -oPasswordAuthentication=no'
+        basecmd = 'sftp -oBatchMode=yes'
         port = ''
         if urlo.port:
             port = '-P %d' % urlo.port

bitbake/lib/bb/fetch2/wget.py

@@ -100,13 +100,173 @@ class Wget(FetchMethod):
 
         return True
 
-    def checkstatus(self, ud, d):
+    def checkstatus(self, fetch, ud, d):
+        import urllib2, socket, httplib
+        from urllib import addinfourl
+        from bb.fetch2 import FetchConnectionCache
+
+        class HTTPConnectionCache(httplib.HTTPConnection):
+            if fetch.connection_cache:
+                def connect(self):
+                    """Connect to the host and port specified in __init__."""
+
+                    sock = fetch.connection_cache.get_connection(self.host, self.port)
+                    if sock:
+                        self.sock = sock
+                    else:
+                        self.sock = socket.create_connection((self.host, self.port),
+                                    self.timeout, self.source_address)
+                        fetch.connection_cache.add_connection(self.host, self.port, self.sock)
+
+                    if self._tunnel_host:
+                        self._tunnel()
+
+        class CacheHTTPHandler(urllib2.HTTPHandler):
+            def http_open(self, req):
+                return self.do_open(HTTPConnectionCache, req)
+
+            def do_open(self, http_class, req):
+                """Return an addinfourl object for the request, using http_class.
+
+                http_class must implement the HTTPConnection API from httplib.
+                The addinfourl return value is a file-like object.  It also
+                has methods and attributes including:
+                    - info(): return a mimetools.Message object for the headers
+                    - geturl(): return the original request URL
+                    - code: HTTP status code
+                """
+                host = req.get_host()
+                if not host:
+                    raise urlllib2.URLError('no host given')
+
+                h = http_class(host, timeout=req.timeout) # will parse host:port
+                h.set_debuglevel(self._debuglevel)
+
+                headers = dict(req.unredirected_hdrs)
+                headers.update(dict((k, v) for k, v in req.headers.items()
+                            if k not in headers))
+
+                # We want to make an HTTP/1.1 request, but the addinfourl
+                # class isn't prepared to deal with a persistent connection.
+                # It will try to read all remaining data from the socket,
+                # which will block while the server waits for the next request.
+                # So make sure the connection gets closed after the (only)
+                # request.
+
+                # Don't close connection when connection_cache is enabled,
+                if fetch.connection_cache is None: 
+                    headers["Connection"] = "close"
+                else:
+                    headers["Connection"] = "Keep-Alive" # Works for HTTP/1.0
+
+                headers = dict(
+                    (name.title(), val) for name, val in headers.items())
+
+                if req._tunnel_host:
+                    tunnel_headers = {}
+                    proxy_auth_hdr = "Proxy-Authorization"
+                    if proxy_auth_hdr in headers:
+                        tunnel_headers[proxy_auth_hdr] = headers[proxy_auth_hdr]
+                        # Proxy-Authorization should not be sent to origin
+                        # server.
+                        del headers[proxy_auth_hdr]
+                    h.set_tunnel(req._tunnel_host, headers=tunnel_headers)
+
+                try:
+                    h.request(req.get_method(), req.get_selector(), req.data, headers)
+                except socket.error, err: # XXX what error?
+                    # Don't close connection when cache is enabled.
+                    if fetch.connection_cache is None:
+                        h.close()
+                    raise urllib2.URLError(err)
+                else:
+                    try:
+                        r = h.getresponse(buffering=True)
+                    except TypeError: # buffering kw not supported
+                        r = h.getresponse()
+
+                # Pick apart the HTTPResponse object to get the addinfourl
+                # object initialized properly.
+
+                # Wrap the HTTPResponse object in socket's file object adapter
+                # for Windows.  That adapter calls recv(), so delegate recv()
+                # to read().  This weird wrapping allows the returned object to
+                # have readline() and readlines() methods.
+
+                # XXX It might be better to extract the read buffering code
+                # out of socket._fileobject() and into a base class.
+                r.recv = r.read
+
+                # no data, just have to read
+                r.read()
+                class fp_dummy(object):
+                    def read(self):
+                        return ""
+                    def readline(self):
+                        return ""
+                    def close(self):
+                        pass
+
+                resp = addinfourl(fp_dummy(), r.msg, req.get_full_url())
+                resp.code = r.status
+                resp.msg = r.reason
+
+                # Close connection when server request it.
+                if fetch.connection_cache is not None:
+                    if 'Connection' in r.msg and r.msg['Connection'] == 'close':
+                        fetch.connection_cache.remove_connection(h.host, h.port)
+
+                return resp
+
+        def export_proxies(d):
+            variables = ['http_proxy', 'HTTP_PROXY', 'https_proxy', 'HTTPS_PROXY',
+                            'ftp_proxy', 'FTP_PROXY', 'no_proxy', 'NO_PROXY']
+            exported = False
+
+            for v in variables:
+                if v in os.environ.keys():
+                    exported = True
+                else:
+                    v_proxy = d.getVar(v, True)
+                    if v_proxy is not None:
+                        os.environ[v] = v_proxy
+                        exported = True
+
+            return exported
+
+        def head_method(self):
+            return "HEAD"
+
+        exported_proxies = export_proxies(d)
+
+        # XXX: Since Python 2.7.9 ssl cert validation is enabled by default
+        # see PEP-0476, this causes verification errors on some https servers
+        # so disable by default.
+        import ssl
+        ssl_context = None
+        if hasattr(ssl, '_create_unverified_context'):
+            ssl_context = ssl._create_unverified_context()
+
+        if exported_proxies == True and ssl_context is not None:
+            opener = urllib2.build_opener(urllib2.ProxyHandler, CacheHTTPHandler,
+                    urllib2.HTTPSHandler(context=ssl_context))
+        elif exported_proxies == False and ssl_context is not None:
+            opener = urllib2.build_opener(CacheHTTPHandler,
+                    urllib2.HTTPSHandler(context=ssl_context))
+        elif exported_proxies == True and ssl_context is None:
+            opener = urllib2.build_opener(urllib2.ProxyHandler, CacheHTTPHandler)
+        else:
+            opener = urllib2.build_opener(CacheHTTPHandler)
+
+        urllib2.Request.get_method = head_method
+        urllib2.install_opener(opener)
 
         uri = ud.url.split(";")[0]
-        fetchcmd = self.basecmd + " --spider '%s'" % uri
-
-        self._runwget(ud, d, fetchcmd, True)
 
+        try:
+            urllib2.urlopen(uri)
+        except:
+            return False
         return True
 
     def _parse_path(self, regex, s):
@@ -246,7 +406,7 @@ class Wget(FetchMethod):
         version_dir = ['', '', '']
         version = ['', '', '']
 
-        dirver_regex = re.compile("(\D*)((\d+[\.-_])+(\d+))")
+        dirver_regex = re.compile("(\D*)((\d+[\.\-_])+(\d+))")
         s = dirver_regex.search(dirver)
         if s:
             version_dir[1] = s.group(2)
@@ -305,7 +465,7 @@ class Wget(FetchMethod):
         pn_regex = "(%s|%s|%s)" % (pn_prefix1, pn_prefix2, pn_prefix3)
 
         # match version
-        pver_regex = "(([A-Z]*\d+[a-zA-Z]*[\.-_]*)+)"
+        pver_regex = "(([A-Z]*\d+[a-zA-Z]*[\.\-_]*)+)"
 
         # match arch
         parch_regex = "-source|_all_"
@@ -347,12 +507,12 @@ class Wget(FetchMethod):
         if not re.search("\d+", package):
             current_version[1] = re.sub('_', '.', current_version[1])
             current_version[1] = re.sub('-', '.', current_version[1])
-            return current_version[1]
+            return (current_version[1], '')
 
         package_regex = self._init_regexes(package, ud, d)
         if package_regex is None:
             bb.warn("latest_versionstring: package %s don't match pattern" % (package))
-            return ""
+            return ('', '')
         bb.debug(3, "latest_versionstring, regex: %s" % (package_regex.pattern))
 
         uri = ""
@@ -370,12 +530,12 @@ class Wget(FetchMethod):
 
                 dirver_pn_regex = re.compile("%s\d?" % (re.escape(pn)))
                 if not dirver_pn_regex.search(dirver):
-                    return self._check_latest_version_by_dir(dirver,
-                        package, package_regex, current_version, ud, d)
+                    return (self._check_latest_version_by_dir(dirver,
+                        package, package_regex, current_version, ud, d), '')
 
             uri = bb.fetch.encodeurl([ud.type, ud.host, path, ud.user, ud.pswd, {}])
         else:
             uri = regex_uri
 
-        return self._check_latest_version(uri, package, package_regex,
-                current_version, ud, d)
+        return (self._check_latest_version(uri, package, package_regex,
+                current_version, ud, d), '')

bitbake/lib/bb/main.py

@@ -36,10 +36,9 @@ from bb import ui
 from bb import server
 from bb import cookerdata
 
-__version__ = "1.25.0"
 logger = logging.getLogger("BitBake")
 
-class BBMainException(bb.BBHandledException):
+class BBMainException(Exception):
     pass
 
 def get_ui(config):
@@ -83,8 +82,8 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
 
     def parseCommandLine(self, argv=sys.argv):
         parser = optparse.OptionParser(
-            version = "BitBake Build Tool Core version %s, %%prog version %s" % (bb.__version__, __version__),
-            usage = """%prog [options] [recipename/target ...]
+            version = "BitBake Build Tool Core version %s" % bb.__version__,
+            usage = """%prog [options] [recipename/target recipe:do_task ...]
 
     Executes the specified task (default is 'build') for a given set of target recipes (.bb files).
     It is assumed there is a conf/bblayers.conf available in cwd or in BBPATH which
@@ -258,6 +257,7 @@ def start_server(servermodule, configParams, configuration, features):
                 logger.handle(event)
         raise exc_info[1], None, exc_info[2]
     server.detach()
+    cooker.lock.close()
     return server
 
 

bitbake/lib/bb/parse/__init__.py

@@ -26,9 +26,10 @@ File parsers for the BitBake build tools.
 
 handlers = []
 
+import errno
+import logging
 import os
 import stat
-import logging
 import bb
 import bb.utils
 import bb.siggen
@@ -81,7 +82,7 @@ def update_cache(f):
 def mark_dependency(d, f):
     if f.startswith('./'):
         f = "%s/%s" % (os.getcwd(), f[2:])
-    deps = (d.getVar('__depends') or [])
+    deps = (d.getVar('__depends', False) or [])
     s = (f, cached_mtime_noerror(f))
     if s not in deps:
         deps.append(s)
@@ -89,7 +90,7 @@ def mark_dependency(d, f):
 
 def check_dependency(d, f):
     s = (f, cached_mtime_noerror(f))
-    deps = (d.getVar('__depends') or [])
+    deps = (d.getVar('__depends', False) or [])
     return s in deps
    
 def supports(fn, data):
@@ -122,12 +123,12 @@ def resolve_file(fn, d):
         for af in attempts:
             mark_dependency(d, af)
         if not newfn:
-            raise IOError("file %s not found in %s" % (fn, bbpath))
+            raise IOError(errno.ENOENT, "file %s not found in %s" % (fn, bbpath))
         fn = newfn
 
     mark_dependency(d, fn)
     if not os.path.isfile(fn):
-        raise IOError("file %s not found" % fn)
+        raise IOError(errno.ENOENT, "file %s not found" % fn)
 
     return fn
 

bitbake/lib/bb/parse/ast.py

@@ -85,7 +85,7 @@ class DataNode(AstNode):
         if 'flag' in self.groupd and self.groupd['flag'] != None:
             return data.getVarFlag(key, self.groupd['flag'], noweakdefault=True)
         else:
-            return data.getVar(key, noweakdefault=True)
+            return data.getVar(key, False, noweakdefault=True, parsing=True)
 
     def eval(self, data):
         groupd = self.groupd
@@ -136,7 +136,7 @@ class DataNode(AstNode):
         if flag:
             data.setVarFlag(key, flag, val, **loginfo)
         else:
-            data.setVar(key, val, **loginfo)
+            data.setVar(key, val, parsing=True, **loginfo)
 
 class MethodNode(AstNode):
     tr_tbl = string.maketrans('/.+-@%&', '_______')
@@ -152,13 +152,13 @@ class MethodNode(AstNode):
             funcname = ("__anon_%s_%s" % (self.lineno, self.filename.translate(MethodNode.tr_tbl)))
             text = "def %s(d):\n" % (funcname) + text
             bb.methodpool.insert_method(funcname, text, self.filename)
-            anonfuncs = data.getVar('__BBANONFUNCS') or []
+            anonfuncs = data.getVar('__BBANONFUNCS', False) or []
             anonfuncs.append(funcname)
             data.setVar('__BBANONFUNCS', anonfuncs)
-            data.setVar(funcname, text)
+            data.setVar(funcname, text, parsing=True)
         else:
             data.setVarFlag(self.func_name, "func", 1)
-            data.setVar(self.func_name, text)
+            data.setVar(self.func_name, text, parsing=True)
 
 class PythonMethodNode(AstNode):
     def __init__(self, filename, lineno, function, modulename, body):
@@ -175,7 +175,7 @@ class PythonMethodNode(AstNode):
         bb.methodpool.insert_method(self.modulename, text, self.filename)
         data.setVarFlag(self.function, "func", 1)
         data.setVarFlag(self.function, "python", 1)
-        data.setVar(self.function, text)
+        data.setVar(self.function, text, parsing=True)
 
 class MethodFlagsNode(AstNode):
     def __init__(self, filename, lineno, key, m):
@@ -184,7 +184,7 @@ class MethodFlagsNode(AstNode):
         self.m = m
 
     def eval(self, data):
-        if data.getVar(self.key):
+        if data.getVar(self.key, False):
             # clean up old version of this piece of metadata, as its
             # flags could cause problems
             data.setVarFlag(self.key, 'python', None)
@@ -209,10 +209,10 @@ class ExportFuncsNode(AstNode):
         for func in self.n:
             calledfunc = self.classname + "_" + func
 
-            if data.getVar(func) and not data.getVarFlag(func, 'export_func'):
+            if data.getVar(func, False) and not data.getVarFlag(func, 'export_func'):
                 continue
 
-            if data.getVar(func):
+            if data.getVar(func, False):
                 data.setVarFlag(func, 'python', None)
                 data.setVarFlag(func, 'func', None)
 
@@ -224,11 +224,11 @@ class ExportFuncsNode(AstNode):
                     data.setVarFlag(calledfunc, flag, data.getVarFlag(func, flag))
 
             if data.getVarFlag(calledfunc, "python"):
-                data.setVar(func, "    bb.build.exec_func('" + calledfunc + "', d)\n")
+                data.setVar(func, "    bb.build.exec_func('" + calledfunc + "', d)\n", parsing=True)
             else:
                 if "-" in self.classname:
                    bb.fatal("The classname %s contains a dash character and is calling an sh function %s using EXPORT_FUNCTIONS. Since a dash is illegal in sh function names, this cannot work, please rename the class or don't use EXPORT_FUNCTIONS." % (self.classname, calledfunc))
-                data.setVar(func, "    " + calledfunc + "\n")
+                data.setVar(func, "    " + calledfunc + "\n", parsing=True)
             data.setVarFlag(func, 'export_func', '1')
 
 class AddTaskNode(AstNode):
@@ -255,7 +255,7 @@ class BBHandlerNode(AstNode):
         self.hs = fns.split()
 
     def eval(self, data):
-        bbhands = data.getVar('__BBHANDLERS') or []
+        bbhands = data.getVar('__BBHANDLERS', False) or []
         for h in self.hs:
             bbhands.append(h)
             data.setVarFlag(h, "handler", 1)
@@ -315,22 +315,22 @@ def handleInherit(statements, filename, lineno, m):
 
 def finalize(fn, d, variant = None):
     all_handlers = {}
-    for var in d.getVar('__BBHANDLERS') or []:
+    for var in d.getVar('__BBHANDLERS', False) or []:
         # try to add the handler
-        bb.event.register(var, d.getVar(var), (d.getVarFlag(var, "eventmask", True) or "").split())
+        bb.event.register(var, d.getVar(var, False), (d.getVarFlag(var, "eventmask", True) or "").split())
 
     bb.event.fire(bb.event.RecipePreFinalise(fn), d)
 
     bb.data.expandKeys(d)
     bb.data.update_data(d)
     code = []
-    for funcname in d.getVar("__BBANONFUNCS") or []:
+    for funcname in d.getVar("__BBANONFUNCS", False) or []:
         code.append("%s(d)" % funcname)
     bb.utils.better_exec("\n".join(code), {"d": d})
     bb.data.update_data(d)
 
-    tasklist = d.getVar('__BBTASKS') or []
-    deltasklist = d.getVar('__BBDELTASKS') or []
+    tasklist = d.getVar('__BBTASKS', False) or []
+    deltasklist = d.getVar('__BBDELTASKS', False) or []
     bb.build.add_tasks(tasklist, deltasklist, d)
 
     bb.parse.siggen.finalise(fn, d, variant)

bitbake/lib/bb/parse/parse_py/BBHandler.py

@@ -32,7 +32,7 @@ import bb.build, bb.utils
 from bb import data
 
 from . import ConfHandler
-from .. import resolve_file, ast, logger
+from .. import resolve_file, ast, logger, ParseError
 from .ConfHandler import include, init
 
 # For compatibility
@@ -48,7 +48,7 @@ __def_regexp__           = re.compile( r"def\s+(\w+).*:" )
 __python_func_regexp__   = re.compile( r"(\s+.*)|(^$)" )
 
 
-__infunc__ = ""
+__infunc__ = []
 __inpython__ = False
 __body__   = []
 __classname__ = ""
@@ -69,15 +69,14 @@ def supports(fn, d):
     return os.path.splitext(fn)[-1] in [".bb", ".bbclass", ".inc"]
 
 def inherit(files, fn, lineno, d):
-    __inherit_cache = d.getVar('__inherit_cache') or []
+    __inherit_cache = d.getVar('__inherit_cache', False) or []
     files = d.expand(files).split()
     for file in files:
         if not os.path.isabs(file) and not file.endswith(".bbclass"):
             file = os.path.join('classes', '%s.bbclass' % file)
 
         if not os.path.isabs(file):
-            dname = os.path.dirname(fn)
-            bbpath = "%s:%s" % (dname, d.getVar("BBPATH", True))
+            bbpath = d.getVar("BBPATH", True)
             abs_fn, attempts = bb.utils.which(bbpath, file, history=True)
             for af in attempts:
                 if af != abs_fn:
@@ -90,7 +89,7 @@ def inherit(files, fn, lineno, d):
             __inherit_cache.append( file )
             d.setVar('__inherit_cache', __inherit_cache)
             include(fn, file, lineno, d, "inherit")
-            __inherit_cache = d.getVar('__inherit_cache') or []
+            __inherit_cache = d.getVar('__inherit_cache', False) or []
 
 def get_statements(filename, absolute_filename, base_name):
     global cached_statements
@@ -120,7 +119,7 @@ def get_statements(filename, absolute_filename, base_name):
 def handle(fn, d, include):
     global __func_start_regexp__, __inherit_regexp__, __export_func_regexp__, __addtask_regexp__, __addhandler_regexp__, __infunc__, __body__, __residue__, __classname__
     __body__ = []
-    __infunc__ = ""
+    __infunc__ = []
     __classname__ = ""
     __residue__ = []
 
@@ -130,13 +129,13 @@ def handle(fn, d, include):
 
     if ext == ".bbclass":
         __classname__ = root
-        __inherit_cache = d.getVar('__inherit_cache') or []
+        __inherit_cache = d.getVar('__inherit_cache', False) or []
         if not fn in __inherit_cache:
             __inherit_cache.append(fn)
             d.setVar('__inherit_cache', __inherit_cache)
 
     if include != 0:
-        oldfile = d.getVar('FILE')
+        oldfile = d.getVar('FILE', False)
     else:
         oldfile = None
 
@@ -149,7 +148,7 @@ def handle(fn, d, include):
     statements = get_statements(fn, abs_fn, base_name)
 
     # DONE WITH PARSING... time to evaluate
-    if ext != ".bbclass":
+    if ext != ".bbclass" and abs_fn != oldfile:
         d.setVar('FILE', abs_fn)
 
     try:
@@ -159,10 +158,15 @@ def handle(fn, d, include):
         if include == 0:
             return { "" : d }
 
+    if __infunc__:
+        raise ParseError("Shell function %s is never closed" % __infunc__[0], __infunc__[1], __infunc__[2])
+    if __residue__:
+        raise ParseError("Leftover unparsed (incomplete?) data %s from %s" % __residue__, fn)
+
     if ext != ".bbclass" and include == 0:
         return ast.multi_finalize(fn, d)
 
-    if oldfile:
+    if ext != ".bbclass" and oldfile and abs_fn != oldfile:
         d.setVar("FILE", oldfile)
 
     return d
@@ -172,8 +176,8 @@ def feeder(lineno, s, fn, root, statements):
     if __infunc__:
         if s == '}':
             __body__.append('')
-            ast.handleMethod(statements, fn, lineno, __infunc__, __body__)
-            __infunc__ = ""
+            ast.handleMethod(statements, fn, lineno, __infunc__[0], __body__)
+            __infunc__ = []
             __body__ = []
         else:
             __body__.append(s)
@@ -217,8 +221,8 @@ def feeder(lineno, s, fn, root, statements):
 
     m = __func_start_regexp__.match(s)
     if m:
-        __infunc__ = m.group("func") or "__anonymous"
-        ast.handleMethodFlags(statements, fn, lineno, __infunc__, m)
+        __infunc__ = [m.group("func") or "__anonymous", fn, lineno]
+        ast.handleMethodFlags(statements, fn, lineno, __infunc__[0], m)
         return
 
     m = __def_regexp__.match(s)

bitbake/lib/bb/parse/parse_py/ConfHandler.py

@@ -24,8 +24,9 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-import re, os
-import logging
+import errno
+import re
+import os
 import bb.utils
 from bb.parse import ParseError, resolve_file, ast, logger, handle
 
@@ -58,7 +59,7 @@ __require_regexp__ = re.compile( r"require\s+(.+)" )
 __export_regexp__ = re.compile( r"export\s+([a-zA-Z0-9\-_+.${}/]+)$" )
 
 def init(data):
-    topdir = data.getVar('TOPDIR')
+    topdir = data.getVar('TOPDIR', False)
     if not topdir:
         data.setVar('TOPDIR', os.getcwd())
 
@@ -92,12 +93,17 @@ def include(parentfn, fn, lineno, data, error_out):
         logger.warn("Duplicate inclusion for %s in %s" % (fn, data.getVar('FILE', True)))
 
     try:
-        ret = bb.parse.handle(fn, data, True)
-    except (IOError, OSError):
-        if error_out:
-            raise ParseError("Could not %(error_out)s file %(fn)s" % vars(), parentfn, lineno)
-        logger.debug(2, "CONF file '%s' not found", fn)
-        bb.parse.mark_dependency(data, fn)
+        bb.parse.handle(fn, data, True)
+    except (IOError, OSError) as exc:
+        if exc.errno == errno.ENOENT:
+            if error_out:
+                raise ParseError("Could not %s file %s" % (error_out, fn), parentfn, lineno)
+            logger.debug(2, "CONF file '%s' not found", fn)
+        else:
+            if error_out:
+                raise ParseError("Could not %s file %s: %s" % (error_out, fn, exc.strerror), parentfn, lineno)
+            else:
+                raise ParseError("Error parsing %s: %s" % (fn, exc.strerror), parentfn, lineno)
 
 # We have an issue where a UI might want to enforce particular settings such as
 # an empty DISTRO variable. If configuration files do something like assigning
@@ -113,7 +119,7 @@ def handle(fn, data, include):
     if include == 0:
         oldfile = None
     else:
-        oldfile = data.getVar('FILE')
+        oldfile = data.getVar('FILE', False)
 
     abs_fn = resolve_file(fn, data)
     f = open(abs_fn, 'r')

bitbake/lib/bb/process.py

@@ -64,7 +64,7 @@ class Popen(subprocess.Popen):
         options.update(kwargs)
         subprocess.Popen.__init__(self, *args, **options)
 
-def _logged_communicate(pipe, log, input):
+def _logged_communicate(pipe, log, input, extrafiles):
     if pipe.stdin:
         if input is not None:
             pipe.stdin.write(input)
@@ -79,6 +79,20 @@ def _logged_communicate(pipe, log, input):
     if pipe.stderr is not None:
         bb.utils.nonblockingfd(pipe.stderr.fileno())
         rin.append(pipe.stderr)
+    for fobj, _ in extrafiles:
+        bb.utils.nonblockingfd(fobj.fileno())
+        rin.append(fobj)
+
+    def readextras(selected):
+        for fobj, func in extrafiles:
+            if fobj in selected:
+                try:
+                    data = fobj.read()
+                except IOError as err:
+                    if err.errno == errno.EAGAIN or err.errno == errno.EWOULDBLOCK:
+                        data = None
+                if data is not None:
+                    func(data)
 
     try:
         while pipe.poll() is None:
@@ -100,15 +114,21 @@ def _logged_communicate(pipe, log, input):
                 if data is not None:
                     errdata.append(data)
                     log.write(data)
+
+            readextras(r)
+
     finally:    
         log.flush()
+
+    readextras([fobj for fobj, _ in extrafiles])
+
     if pipe.stdout is not None:
         pipe.stdout.close()
     if pipe.stderr is not None:
         pipe.stderr.close()
     return ''.join(outdata), ''.join(errdata)
 
-def run(cmd, input=None, log=None, **options):
+def run(cmd, input=None, log=None, extrafiles=[], **options):
     """Convenience function to run a command and return its output, raising an
     exception when the command fails"""
 
@@ -124,7 +144,7 @@ def run(cmd, input=None, log=None, **options):
             raise CmdError(cmd, exc)
 
     if log:
-        stdout, stderr = _logged_communicate(pipe, log, input)
+        stdout, stderr = _logged_communicate(pipe, log, input, extrafiles)
     else:
         stdout, stderr = pipe.communicate(input)
 

bitbake/lib/bb/runqueue.py

@@ -1096,6 +1096,13 @@ class RunQueue:
             raise
         except SystemExit:
             raise
+        except bb.BBHandledException:
+            try:
+                self.teardown_workers()
+            except:
+                pass
+            self.state = runQueueComplete
+            raise
         except:
             logger.error("An uncaught exception occured in runqueue, please see the failure below:")
             try:
@@ -1154,9 +1161,14 @@ class RunQueue:
             sq_hash.append(self.rqdata.runq_hash[task])
             sq_taskname.append(taskname)
             sq_task.append(task)
-        call = self.hashvalidate + "(sq_fn, sq_task, sq_hash, sq_hashfn, d)"
-        locs = { "sq_fn" : sq_fn, "sq_task" : sq_taskname, "sq_hash" : sq_hash, "sq_hashfn" : sq_hashfn, "d" : self.cooker.data }
-        valid = bb.utils.better_eval(call, locs)
+        locs = { "sq_fn" : sq_fn, "sq_task" : sq_taskname, "sq_hash" : sq_hash, "sq_hashfn" : sq_hashfn, "d" : self.cooker.expanded_data }
+        try:
+            call = self.hashvalidate + "(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=True)"
+            valid = bb.utils.better_eval(call, locs)
+        # Handle version with no siginfo parameter
+        except TypeError:
+            call = self.hashvalidate + "(sq_fn, sq_task, sq_hash, sq_hashfn, d)"
+            valid = bb.utils.better_eval(call, locs)
         for v in valid:
             valid_new.add(sq_task[v])
 
@@ -1268,6 +1280,9 @@ class RunQueueExecute:
         if rq.fakeworkerpipe:
             rq.fakeworkerpipe.setrunqueueexec(self)
 
+        if self.number_tasks <= 0:
+             bb.fatal("Invalid BB_NUMBER_THREADS %s" % self.number_tasks)
+
     def runqueue_process_waitpid(self, task, status):
 
         # self.build_stamps[pid] may not exist when use shared work directory.
@@ -1331,7 +1346,7 @@ class RunQueueExecute:
             taskname = self.rqdata.runq_task[depid]
             taskdata[dep] = [pn, taskname, fn]
         call = self.rq.depvalidate + "(task, taskdata, notneeded, d)"
-        locs = { "task" : task, "taskdata" : taskdata, "notneeded" : self.scenequeue_notneeded, "d" : self.cooker.data }
+        locs = { "task" : task, "taskdata" : taskdata, "notneeded" : self.scenequeue_notneeded, "d" : self.cooker.expanded_data }
         valid = bb.utils.better_eval(call, locs)
         return valid
 
@@ -1400,7 +1415,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
 
             call = self.rq.setsceneverify + "(covered, tasknames, fnids, fns, d, invalidtasks=invalidtasks)"
             call2 = self.rq.setsceneverify + "(covered, tasknames, fnids, fns, d)"
-            locs = { "covered" : self.rq.scenequeue_covered, "tasknames" : self.rqdata.runq_task, "fnids" : self.rqdata.runq_fnid, "fns" : self.rqdata.taskData.fn_index, "d" : self.cooker.data, "invalidtasks" : invalidtasks }
+            locs = { "covered" : self.rq.scenequeue_covered, "tasknames" : self.rqdata.runq_task, "fnids" : self.rqdata.runq_fnid, "fns" : self.rqdata.taskData.fn_index, "d" : self.cooker.expanded_data, "invalidtasks" : invalidtasks }
             # Backwards compatibility with older versions without invalidtasks
             try:
                 covered_remove = bb.utils.better_eval(call, locs)
@@ -1565,7 +1580,12 @@ class RunQueueExecuteTasks(RunQueueExecute):
             taskdep = self.rqdata.dataCache.task_deps[fn]
             if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run:
                 if not self.rq.fakeworker:
-                    self.rq.start_fakeworker(self)
+                    try:
+                        self.rq.start_fakeworker(self)
+                    except OSError as exc:
+                        logger.critical("Failed to spawn fakeroot worker to run %s:%s: %s" % (fn, taskname, str(exc)))
+                        self.rq.state = runQueueFailed
+                        return True
                 self.rq.fakeworker.stdin.write("<runtask>" + pickle.dumps((fn, task, taskname, False, self.cooker.collection.get_file_appends(fn), taskdepdata)) + "</runtask>")
                 self.rq.fakeworker.stdin.flush()
             else:
@@ -1820,7 +1840,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
                 sq_taskname.append(taskname)
                 sq_task.append(task)
             call = self.rq.hashvalidate + "(sq_fn, sq_task, sq_hash, sq_hashfn, d)"
-            locs = { "sq_fn" : sq_fn, "sq_task" : sq_taskname, "sq_hash" : sq_hash, "sq_hashfn" : sq_hashfn, "d" : self.cooker.data }
+            locs = { "sq_fn" : sq_fn, "sq_task" : sq_taskname, "sq_hash" : sq_hash, "sq_hashfn" : sq_hashfn, "d" : self.cooker.expanded_data }
             valid = bb.utils.better_eval(call, locs)
 
             valid_new = stamppresent

bitbake/lib/bb/server/process.py

@@ -144,8 +144,9 @@ class ProcessServer(Process, BaseImplServer):
                     fds = fds + retval
             except SystemExit:
                 raise
-            except Exception:
-                logger.exception('Running idle function')
+            except Exception as exc:
+                if not isinstance(exc, bb.BBHandledException):
+                    logger.exception('Running idle function')
                 del self._idlefuns[function]
                 self.quit = True
 

bitbake/lib/bb/siggen.py

@@ -419,12 +419,16 @@ def compare_sigfiles(a, b, recursecb = None):
         for f in removed:
             output.append("Dependency on checksum of file %s was removed" % (f))
 
-    changed = []
-    for idx, task in enumerate(a_data['runtaskdeps']):
-        a = a_data['runtaskdeps'][idx]
-        b = b_data['runtaskdeps'][idx]
-        if a_data['runtaskhashes'][a] != b_data['runtaskhashes'][b]:
-            changed.append("%s with hash %s\n changed to\n%s with hash %s" % (a, a_data['runtaskhashes'][a], b, b_data['runtaskhashes'][b]))
+
+    if len(a_data['runtaskdeps']) != len(b_data['runtaskdeps']):
+        changed = ["Number of task dependencies changed"]
+    else:
+        changed = []
+        for idx, task in enumerate(a_data['runtaskdeps']):
+            a = a_data['runtaskdeps'][idx]
+            b = b_data['runtaskdeps'][idx]
+            if a_data['runtaskhashes'][a] != b_data['runtaskhashes'][b]:
+                changed.append("%s with hash %s\n changed to\n%s with hash %s" % (a, a_data['runtaskhashes'][a], b, b_data['runtaskhashes'][b]))
 
     if changed:
         output.append("runtaskdeps changed from %s to %s" % (clean_basepaths_list(a_data['runtaskdeps']), clean_basepaths_list(b_data['runtaskdeps'])))

bitbake/lib/bb/taskdata.py

@@ -41,7 +41,7 @@ class TaskData:
     """
     BitBake Task Data implementation
     """
-    def __init__(self, abort = True, tryaltconfigs = False, skiplist = None):
+    def __init__(self, abort = True, tryaltconfigs = False, skiplist = None, allowincomplete = False):
         self.build_names_index = []
         self.run_names_index = []
         self.fn_index = []
@@ -70,6 +70,7 @@ class TaskData:
 
         self.abort = abort
         self.tryaltconfigs = tryaltconfigs
+        self.allowincomplete = allowincomplete
 
         self.skiplist = skiplist
 
@@ -594,9 +595,10 @@ class TaskData:
                     added = added + 1
                 except bb.providers.NoProvider:
                     targetid = self.getbuild_id(target)
-                    if self.abort and targetid in self.external_targets:
+                    if self.abort and targetid in self.external_targets and not self.allowincomplete:
                         raise
-                    self.remove_buildtarget(targetid)
+                    if not self.allowincomplete:
+                        self.remove_buildtarget(targetid)
             for target in self.get_unresolved_run_targets(dataCache):
                 try:
                     self.add_rprovider(cfgData, dataCache, target)

bitbake/lib/bb/tests/data.py

@@ -24,6 +24,30 @@ import unittest
 import bb
 import bb.data
 import bb.parse
+import logging
+
+class LogRecord():
+    def __enter__(self):
+        logs = []
+        class LogHandler(logging.Handler):
+            def emit(self, record):
+                logs.append(record)
+        logger = logging.getLogger("BitBake")
+        handler = LogHandler()
+        self.handler = handler
+        logger.addHandler(handler)
+        return logs
+    def __exit__(self, type, value, traceback):
+        logger = logging.getLogger("BitBake")
+        logger.removeHandler(self.handler)
+        return
+
+def logContains(item, logs):
+    for l in logs:
+        m = l.getMessage()
+        if item in m:
+            return True
+    return False
 
 class DataExpansions(unittest.TestCase):
     def setUp(self):
@@ -110,12 +134,12 @@ class DataExpansions(unittest.TestCase):
 
     def test_rename(self):
         self.d.renameVar("foo", "newfoo")
-        self.assertEqual(self.d.getVar("newfoo"), "value_of_foo")
-        self.assertEqual(self.d.getVar("foo"), None)
+        self.assertEqual(self.d.getVar("newfoo", False), "value_of_foo")
+        self.assertEqual(self.d.getVar("foo", False), None)
 
     def test_deletion(self):
         self.d.delVar("foo")
-        self.assertEqual(self.d.getVar("foo"), None)
+        self.assertEqual(self.d.getVar("foo", False), None)
 
     def test_keys(self):
         keys = self.d.keys()
@@ -172,28 +196,28 @@ class TestMemoize(unittest.TestCase):
     def test_memoized(self):
         d = bb.data.init()
         d.setVar("FOO", "bar")
-        self.assertTrue(d.getVar("FOO") is d.getVar("FOO"))
+        self.assertTrue(d.getVar("FOO", False) is d.getVar("FOO", False))
 
     def test_not_memoized(self):
         d1 = bb.data.init()
         d2 = bb.data.init()
         d1.setVar("FOO", "bar")
         d2.setVar("FOO", "bar2")
-        self.assertTrue(d1.getVar("FOO") is not d2.getVar("FOO"))
+        self.assertTrue(d1.getVar("FOO", False) is not d2.getVar("FOO", False))
 
     def test_changed_after_memoized(self):
         d = bb.data.init()
         d.setVar("foo", "value of foo")
-        self.assertEqual(str(d.getVar("foo")), "value of foo")
+        self.assertEqual(str(d.getVar("foo", False)), "value of foo")
         d.setVar("foo", "second value of foo")
-        self.assertEqual(str(d.getVar("foo")), "second value of foo")
+        self.assertEqual(str(d.getVar("foo", False)), "second value of foo")
 
     def test_same_value(self):
         d = bb.data.init()
         d.setVar("foo", "value of")
         d.setVar("bar", "value of")
-        self.assertEqual(d.getVar("foo"),
-                         d.getVar("bar"))
+        self.assertEqual(d.getVar("foo", False),
+                         d.getVar("bar", False))
 
 class TestConcat(unittest.TestCase):
     def setUp(self):
@@ -246,6 +270,13 @@ class TestConcatOverride(unittest.TestCase):
         bb.data.update_data(self.d)
         self.assertEqual(self.d.getVar("TEST", True), "foo:val:val2:bar")
 
+    def test_append_unset(self):
+        self.d.setVar("TEST_prepend", "${FOO}:")
+        self.d.setVar("TEST_append", ":val2")
+        self.d.setVar("TEST_append", ":${BAR}")
+        bb.data.update_data(self.d)
+        self.assertEqual(self.d.getVar("TEST", True), "foo::val2:bar")
+
     def test_remove(self):
         self.d.setVar("TEST", "${VAL} ${BAR}")
         self.d.setVar("TEST_remove", "val")
@@ -294,13 +325,66 @@ class TestOverrides(unittest.TestCase):
         bb.data.update_data(self.d)
         self.assertEqual(self.d.getVar("TEST", True), "testvalue2")
 
+    def test_one_override_unset(self):
+        self.d.setVar("TEST2_bar", "testvalue2")
+        bb.data.update_data(self.d)
+        self.assertEqual(self.d.getVar("TEST2", True), "testvalue2")
+        self.assertItemsEqual(self.d.keys(), ['TEST', 'TEST2', 'OVERRIDES', 'TEST2_bar'])
+
     def test_multiple_override(self):
         self.d.setVar("TEST_bar", "testvalue2")
         self.d.setVar("TEST_local", "testvalue3")
         self.d.setVar("TEST_foo", "testvalue4")
         bb.data.update_data(self.d)
         self.assertEqual(self.d.getVar("TEST", True), "testvalue3")
+        self.assertItemsEqual(self.d.keys(), ['TEST', 'TEST_foo', 'OVERRIDES', 'TEST_bar', 'TEST_local'])
 
+    def test_multiple_combined_overrides(self):
+        self.d.setVar("TEST_local_foo_bar", "testvalue3")
+        bb.data.update_data(self.d)
+        self.assertEqual(self.d.getVar("TEST", True), "testvalue3")
+
+    def test_multiple_overrides_unset(self):
+        self.d.setVar("TEST2_local_foo_bar", "testvalue3")
+        bb.data.update_data(self.d)
+        self.assertEqual(self.d.getVar("TEST2", True), "testvalue3")
+
+    def test_keyexpansion_override(self):
+        self.d.setVar("LOCAL", "local")
+        self.d.setVar("TEST_bar", "testvalue2")
+        self.d.setVar("TEST_${LOCAL}", "testvalue3")
+        self.d.setVar("TEST_foo", "testvalue4")
+        bb.data.update_data(self.d)
+        bb.data.expandKeys(self.d)
+        self.assertEqual(self.d.getVar("TEST", True), "testvalue3")
+
+    def test_rename_override(self):
+        self.d.setVar("ALTERNATIVE_ncurses-tools_class-target", "a")
+        self.d.setVar("OVERRIDES", "class-target")
+        bb.data.update_data(self.d)
+        self.d.renameVar("ALTERNATIVE_ncurses-tools", "ALTERNATIVE_lib32-ncurses-tools")
+        self.assertEqual(self.d.getVar("ALTERNATIVE_lib32-ncurses-tools", True), "a")
+
+    def test_underscore_override(self):
+        self.d.setVar("TEST_bar", "testvalue2")
+        self.d.setVar("TEST_some_val", "testvalue3")
+        self.d.setVar("TEST_foo", "testvalue4")
+        self.d.setVar("OVERRIDES", "foo:bar:some_val")
+        self.assertEqual(self.d.getVar("TEST", True), "testvalue3")
+
+class TestKeyExpansion(unittest.TestCase):
+    def setUp(self):
+        self.d = bb.data.init()
+        self.d.setVar("FOO", "foo")
+        self.d.setVar("BAR", "foo")
+
+    def test_keyexpand(self):
+        self.d.setVar("VAL_${FOO}", "A")
+        self.d.setVar("VAL_${BAR}", "B")
+        with LogRecord() as logs:
+            bb.data.expandKeys(self.d)
+            self.assertTrue(logContains("Variable key VAL_${FOO} (A) replaces original key VAL_foo (B)", logs))
+        self.assertEqual(self.d.getVar("VAL_foo", True), "A")
 
 class TestFlags(unittest.TestCase):
     def setUp(self):

bitbake/lib/bb/tests/fetch.py

@@ -315,6 +315,7 @@ class URITest(unittest.TestCase):
 class FetcherTest(unittest.TestCase):
 
     def setUp(self):
+        self.origdir = os.getcwd()
         self.d = bb.data.init()
         self.tempdir = tempfile.mkdtemp()
         self.dldir = os.path.join(self.tempdir, "download")
@@ -326,6 +327,7 @@ class FetcherTest(unittest.TestCase):
         self.d.setVar("PERSISTENT_DIR", persistdir)
 
     def tearDown(self):
+        os.chdir(self.origdir)
         bb.utils.prunedir(self.tempdir)
 
 class MirrorUriTest(FetcherTest):
@@ -391,6 +393,18 @@ class MirrorUriTest(FetcherTest):
         uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
         self.assertEqual(uris, ['file:///someotherpath/downloads/bitbake-1.0.tar.gz'])
 
+    def test_mirror_of_mirror(self):
+        # Test if mirror of a mirror works
+        mirrorvar = self.mirrorvar + " http://.*/.* http://otherdownloads.yoctoproject.org/downloads/ \n"
+        mirrorvar = mirrorvar + " http://otherdownloads.yoctoproject.org/.* http://downloads2.yoctoproject.org/downloads/ \n"
+        fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
+        mirrors = bb.fetch2.mirror_from_string(mirrorvar)
+        uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
+        self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz', 
+                                'file:///someotherpath/downloads/bitbake-1.0.tar.gz', 
+                                'http://otherdownloads.yoctoproject.org/downloads/bitbake-1.0.tar.gz',
+                                'http://downloads2.yoctoproject.org/downloads/bitbake-1.0.tar.gz'])
+
 
 class FetcherLocalTest(FetcherTest):
     def setUp(self):
@@ -476,6 +490,19 @@ class FetcherNetworkTest(FetcherTest):
             fetcher.download()
             self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
 
+        def test_fetch_mirror_of_mirror(self):
+            self.d.setVar("MIRRORS", "http://.*/.* http://invalid2.yoctoproject.org/ \n http://invalid2.yoctoproject.org/.* http://downloads.yoctoproject.org/releases/bitbake")
+            fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
+            fetcher.download()
+            self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
+
+        def test_fetch_file_mirror_of_mirror(self):
+            self.d.setVar("MIRRORS", "http://.*/.* file:///some1where/ \n file:///some1where/.* file://some2where/ \n file://some2where/.* http://downloads.yoctoproject.org/releases/bitbake")
+            fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
+            os.mkdir(self.dldir + "/some2where")
+            fetcher.download()
+            self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
+
         def test_fetch_premirror(self):
             self.d.setVar("PREMIRRORS", "http://.*/.* http://downloads.yoctoproject.org/releases/bitbake")
             fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
@@ -547,6 +574,43 @@ class FetcherNetworkTest(FetcherTest):
             os.chdir(os.path.dirname(self.unpackdir))
             fetcher.unpack(self.unpackdir)
 
+        def test_trusted_network(self):
+            # Ensure trusted_network returns False when the host IS in the list.
+            url = "git://Someserver.org/foo;rev=1"
+            self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org someserver.org server2.org server3.org")
+            self.assertTrue(bb.fetch.trusted_network(self.d, url))
+
+        def test_wild_trusted_network(self):
+            # Ensure trusted_network returns true when the *.host IS in the list.
+            url = "git://Someserver.org/foo;rev=1"
+            self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
+            self.assertTrue(bb.fetch.trusted_network(self.d, url))
+
+        def test_prefix_wild_trusted_network(self):
+            # Ensure trusted_network returns true when the prefix matches *.host.
+            url = "git://git.Someserver.org/foo;rev=1"
+            self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
+            self.assertTrue(bb.fetch.trusted_network(self.d, url))
+
+        def test_two_prefix_wild_trusted_network(self):
+            # Ensure trusted_network returns true when the prefix matches *.host.
+            url = "git://something.git.Someserver.org/foo;rev=1"
+            self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
+            self.assertTrue(bb.fetch.trusted_network(self.d, url))
+
+        def test_untrusted_network(self):
+            # Ensure trusted_network returns False when the host is NOT in the list.
+            url = "git://someserver.org/foo;rev=1"
+            self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org")
+            self.assertFalse(bb.fetch.trusted_network(self.d, url))
+
+        def test_wild_untrusted_network(self):
+            # Ensure trusted_network returns False when the host is NOT in the list.
+            url = "git://*.someserver.org/foo;rev=1"
+            self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org")
+            self.assertFalse(bb.fetch.trusted_network(self.d, url))
+
+
 class URLHandle(unittest.TestCase):
 
     datatable = {
@@ -566,7 +630,7 @@ class URLHandle(unittest.TestCase):
             result = bb.fetch.encodeurl(v)
             self.assertEqual(result, k)
 
-class FetchMethodTest(FetcherTest):
+class FetchLatestVersionTest(FetcherTest):
 
     test_git_uris = {
         # version pattern "X.Y.Z"
@@ -633,7 +697,8 @@ class FetchMethodTest(FetcherTest):
                 self.d.setVar("SRCREV", k[2])
                 self.d.setVar("GITTAGREGEX", k[3])
                 ud = bb.fetch2.FetchData(k[1], self.d)
-                verstring = ud.method.latest_versionstring(ud, self.d)
+                pupver= ud.method.latest_versionstring(ud, self.d)
+                verstring = pupver[0]
                 r = bb.utils.vercmp_string(v, verstring)
                 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring))
 
@@ -643,6 +708,52 @@ class FetchMethodTest(FetcherTest):
                 self.d.setVar("REGEX_URI", k[2])
                 self.d.setVar("REGEX", k[3])
                 ud = bb.fetch2.FetchData(k[1], self.d)
-                verstring = ud.method.latest_versionstring(ud, self.d)
+                pupver = ud.method.latest_versionstring(ud, self.d)
+                verstring = pupver[0]
                 r = bb.utils.vercmp_string(v, verstring)
                 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring))
+
+
+class FetchCheckStatusTest(FetcherTest):
+    test_wget_uris = ["http://www.cups.org/software/1.7.2/cups-1.7.2-source.tar.bz2",
+                      "http://www.cups.org/software/ipptool/ipptool-20130731-linux-ubuntu-i686.tar.gz",
+                      "http://www.cups.org/",
+                      "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.1.tar.gz",
+                      "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.2.tar.gz",
+                      "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.3.tar.gz",
+                      "https://yoctoproject.org/",
+                      "https://yoctoproject.org/documentation",
+                      "http://downloads.yoctoproject.org/releases/opkg/opkg-0.1.7.tar.gz",
+                      "http://downloads.yoctoproject.org/releases/opkg/opkg-0.3.0.tar.gz",
+                      "ftp://ftp.gnu.org/gnu/autoconf/autoconf-2.60.tar.gz",
+                      "ftp://ftp.gnu.org/gnu/chess/gnuchess-5.08.tar.gz",
+                      "ftp://ftp.gnu.org/gnu/gmp/gmp-4.0.tar.gz",
+                      ]
+
+    if os.environ.get("BB_SKIP_NETTESTS") == "yes":
+        print("Unset BB_SKIP_NETTESTS to run network tests")
+    else:
+
+        def test_wget_checkstatus(self):
+            fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d)
+            for u in self.test_wget_uris:
+                ud = fetch.ud[u]
+                m = ud.method
+                ret = m.checkstatus(fetch, ud, self.d)
+                self.assertTrue(ret, msg="URI %s, can't check status" % (u))
+
+
+        def test_wget_checkstatus_connection_cache(self):
+            from bb.fetch2 import FetchConnectionCache
+
+            connection_cache = FetchConnectionCache()
+            fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d,
+                        connection_cache = connection_cache)
+
+            for u in self.test_wget_uris:
+                ud = fetch.ud[u]
+                m = ud.method
+                ret = m.checkstatus(fetch, ud, self.d)
+                self.assertTrue(ret, msg="URI %s, can't check status" % (u))
+
+            connection_cache.close_connections()

bitbake/lib/bb/tests/parse.py

@@ -0,0 +1,147 @@
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Test for lib/bb/parse/
+#
+# Copyright (C) 2015 Richard Purdie
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+
+import unittest
+import tempfile
+import logging
+import bb
+import os
+
+logger = logging.getLogger('BitBake.TestParse')
+
+import bb.parse
+import bb.data
+import bb.siggen
+
+class ParseTest(unittest.TestCase):
+
+    testfile = """
+A = "1"
+B = "2"
+do_install() {
+	echo "hello"
+}
+
+C = "3"
+"""
+
+    def setUp(self):
+        self.d = bb.data.init()
+        bb.parse.siggen = bb.siggen.init(self.d)
+
+    def parsehelper(self, content, suffix = ".bb"):
+
+        f = tempfile.NamedTemporaryFile(suffix = suffix)
+        f.write(content)
+        f.flush()
+        os.chdir(os.path.dirname(f.name))
+        return f
+
+    def test_parse_simple(self):
+        f = self.parsehelper(self.testfile)
+        d = bb.parse.handle(f.name, self.d)['']
+        self.assertEqual(d.getVar("A", True), "1")
+        self.assertEqual(d.getVar("B", True), "2")
+        self.assertEqual(d.getVar("C", True), "3")
+
+    def test_parse_incomplete_function(self):
+        testfileB = self.testfile.replace("}", "")
+        f = self.parsehelper(testfileB)
+        with self.assertRaises(bb.parse.ParseError):
+            d = bb.parse.handle(f.name, self.d)['']
+
+    overridetest = """
+RRECOMMENDS_${PN} = "a"
+RRECOMMENDS_${PN}_libc = "b"
+OVERRIDES = "libc:${PN}"
+PN = "gtk+"
+"""
+
+    def test_parse_overrides(self):
+        f = self.parsehelper(self.overridetest)
+        d = bb.parse.handle(f.name, self.d)['']
+        self.assertEqual(d.getVar("RRECOMMENDS", True), "b")
+        bb.data.expandKeys(d)
+        self.assertEqual(d.getVar("RRECOMMENDS", True), "b")
+        d.setVar("RRECOMMENDS_gtk+", "c")
+        self.assertEqual(d.getVar("RRECOMMENDS", True), "c")
+
+    overridetest2 = """
+EXTRA_OECONF = ""
+EXTRA_OECONF_class-target = "b"
+EXTRA_OECONF_append = " c"
+"""
+
+    def test_parse_overrides(self):
+        f = self.parsehelper(self.overridetest2)
+        d = bb.parse.handle(f.name, self.d)['']
+        d.appendVar("EXTRA_OECONF", " d")
+        d.setVar("OVERRIDES", "class-target")
+        self.assertEqual(d.getVar("EXTRA_OECONF", True), "b c d")
+
+    overridetest3 = """
+DESCRIPTION = "A"
+DESCRIPTION_${PN}-dev = "${DESCRIPTION} B"
+PN = "bc"
+"""
+
+    def test_parse_combinations(self):
+        f = self.parsehelper(self.overridetest3)
+        d = bb.parse.handle(f.name, self.d)['']
+        bb.data.expandKeys(d)
+        self.assertEqual(d.getVar("DESCRIPTION_bc-dev", True), "A B")
+        d.setVar("DESCRIPTION", "E")
+        d.setVar("DESCRIPTION_bc-dev", "C D")
+        d.setVar("OVERRIDES", "bc-dev")
+        self.assertEqual(d.getVar("DESCRIPTION", True), "C D")
+
+
+    classextend = """
+VAR_var_override1 = "B"
+EXTRA = ":override1"
+OVERRIDES = "nothing${EXTRA}"
+
+BBCLASSEXTEND = "###CLASS###"
+"""
+    classextend_bbclass = """
+EXTRA = ""
+python () {
+    d.renameVar("VAR_var", "VAR_var2")
+}
+"""
+
+    #
+    # Test based upon a real world data corruption issue. One
+    # data store changing a variable poked through into a different data
+    # store. This test case replicates that issue where the value 'B' would 
+    # become unset/disappear.
+    #
+    def test_parse_classextend_contamination(self):
+        cls = self.parsehelper(self.classextend_bbclass, suffix=".bbclass")
+        #clsname = os.path.basename(cls.name).replace(".bbclass", "")
+        self.classextend = self.classextend.replace("###CLASS###", cls.name)
+        f = self.parsehelper(self.classextend)
+        alldata = bb.parse.handle(f.name, self.d)
+        d1 = alldata['']
+        d2 = alldata[cls.name]
+        self.assertEqual(d1.getVar("VAR_var", True), "B")
+        self.assertEqual(d2.getVar("VAR_var", True), None)
+

bitbake/lib/bb/tests/utils.py

@@ -21,6 +21,8 @@
 
 import unittest
 import bb
+import os
+import tempfile
 
 class VerCmpString(unittest.TestCase):
 
@@ -88,3 +90,289 @@ class VerCmpString(unittest.TestCase):
 
         # Check that clearly invalid operator raises an exception
         self.assertRaises(bb.utils.VersionStringException, bb.utils.vercmp_string_op, '0', '0', '$')
+
+
+class Path(unittest.TestCase):
+    def test_unsafe_delete_path(self):
+        checkitems = [('/', True),
+                      ('//', True),
+                      ('///', True),
+                      (os.getcwd().count(os.sep) * ('..' + os.sep), True),
+                      (os.environ.get('HOME', '/home/test'), True),
+                      ('/home/someone', True),
+                      ('/home/other/', True),
+                      ('/home/other/subdir', False),
+                      ('', False)]
+        for arg1, correctresult in checkitems:
+            result = bb.utils._check_unsafe_delete_path(arg1)
+            self.assertEqual(result, correctresult, '_check_unsafe_delete_path("%s") != %s' % (arg1, correctresult))
+
+
+class EditMetadataFile(unittest.TestCase):
+    _origfile = """
+# A comment
+HELLO = "oldvalue"
+
+THIS = "that"
+
+# Another comment
+NOCHANGE = "samevalue"
+OTHER = 'anothervalue'
+
+MULTILINE = "a1 \\
+             a2 \\
+             a3"
+
+MULTILINE2 := " \\
+               b1 \\
+               b2 \\
+               b3 \\
+               "
+
+
+MULTILINE3 = " \\
+              c1 \\
+              c2 \\
+              c3 \\
+"
+
+do_functionname() {
+    command1 ${VAL1} ${VAL2}
+    command2 ${VAL3} ${VAL4}
+}
+"""
+    def _testeditfile(self, varvalues, compareto, dummyvars=None):
+        if dummyvars is None:
+            dummyvars = []
+        with tempfile.NamedTemporaryFile('w', delete=False) as tf:
+            tf.write(self._origfile)
+            tf.close()
+            try:
+                varcalls = []
+                def handle_file(varname, origvalue, op, newlines):
+                    self.assertIn(varname, varvalues, 'Callback called for variable %s not in the list!' % varname)
+                    self.assertNotIn(varname, dummyvars, 'Callback called for variable %s in dummy list!' % varname)
+                    varcalls.append(varname)
+                    return varvalues[varname]
+
+                bb.utils.edit_metadata_file(tf.name, varvalues.keys(), handle_file)
+                with open(tf.name) as f:
+                    modfile = f.readlines()
+                # Ensure the output matches the expected output
+                self.assertEqual(compareto.splitlines(True), modfile)
+                # Ensure the callback function was called for every variable we asked for
+                # (plus allow testing behaviour when a requested variable is not present)
+                self.assertEqual(sorted(varvalues.keys()), sorted(varcalls + dummyvars))
+            finally:
+                os.remove(tf.name)
+
+
+    def test_edit_metadata_file_nochange(self):
+        # Test file doesn't get modified with nothing to do
+        self._testeditfile({}, self._origfile)
+        # Test file doesn't get modified with only dummy variables
+        self._testeditfile({'DUMMY1': ('should_not_set', None, 0, True),
+                        'DUMMY2': ('should_not_set_again', None, 0, True)}, self._origfile, dummyvars=['DUMMY1', 'DUMMY2'])
+        # Test file doesn't get modified with some the same values
+        self._testeditfile({'THIS': ('that', None, 0, True),
+                        'OTHER': ('anothervalue', None, 0, True),
+                        'MULTILINE3': ('               c1               c2               c3', None, 4, False)}, self._origfile)
+
+    def test_edit_metadata_file_1(self):
+
+        newfile1 = """
+# A comment
+HELLO = "newvalue"
+
+THIS = "that"
+
+# Another comment
+NOCHANGE = "samevalue"
+OTHER = 'anothervalue'
+
+MULTILINE = "a1 \\
+             a2 \\
+             a3"
+
+MULTILINE2 := " \\
+               b1 \\
+               b2 \\
+               b3 \\
+               "
+
+
+MULTILINE3 = " \\
+              c1 \\
+              c2 \\
+              c3 \\
+"
+
+do_functionname() {
+    command1 ${VAL1} ${VAL2}
+    command2 ${VAL3} ${VAL4}
+}
+"""
+        self._testeditfile({'HELLO': ('newvalue', None, 4, True)}, newfile1)
+
+
+    def test_edit_metadata_file_2(self):
+
+        newfile2 = """
+# A comment
+HELLO = "oldvalue"
+
+THIS = "that"
+
+# Another comment
+NOCHANGE = "samevalue"
+OTHER = 'anothervalue'
+
+MULTILINE = " \\
+    d1 \\
+    d2 \\
+    d3 \\
+    "
+
+MULTILINE2 := " \\
+               b1 \\
+               b2 \\
+               b3 \\
+               "
+
+
+MULTILINE3 = "nowsingle"
+
+do_functionname() {
+    command1 ${VAL1} ${VAL2}
+    command2 ${VAL3} ${VAL4}
+}
+"""
+        self._testeditfile({'MULTILINE': (['d1','d2','d3'], None, 4, False),
+                        'MULTILINE3': ('nowsingle', None, 4, True),
+                        'NOTPRESENT': (['a', 'b'], None, 4, False)}, newfile2, dummyvars=['NOTPRESENT'])
+
+
+    def test_edit_metadata_file_3(self):
+
+        newfile3 = """
+# A comment
+HELLO = "oldvalue"
+
+# Another comment
+NOCHANGE = "samevalue"
+OTHER = "yetanothervalue"
+
+MULTILINE = "e1 \\
+             e2 \\
+             e3 \\
+             "
+
+MULTILINE2 := "f1 \\
+\tf2 \\
+\t"
+
+
+MULTILINE3 = " \\
+              c1 \\
+              c2 \\
+              c3 \\
+"
+
+do_functionname() {
+    othercommand_one a b c
+    othercommand_two d e f
+}
+"""
+
+        self._testeditfile({'do_functionname()': (['othercommand_one a b c', 'othercommand_two d e f'], None, 4, False),
+                        'MULTILINE2': (['f1', 'f2'], None, '\t', True),
+                        'MULTILINE': (['e1', 'e2', 'e3'], None, -1, True),
+                        'THIS': (None, None, 0, False),
+                        'OTHER': ('yetanothervalue', None, 0, True)}, newfile3)
+
+
+    def test_edit_metadata_file_4(self):
+
+        newfile4 = """
+# A comment
+HELLO = "oldvalue"
+
+THIS = "that"
+
+# Another comment
+OTHER = 'anothervalue'
+
+MULTILINE = "a1 \\
+             a2 \\
+             a3"
+
+MULTILINE2 := " \\
+               b1 \\
+               b2 \\
+               b3 \\
+               "
+
+
+"""
+
+        self._testeditfile({'NOCHANGE': (None, None, 0, False),
+                        'MULTILINE3': (None, None, 0, False),
+                        'THIS': ('that', None, 0, False),
+                        'do_functionname()': (None, None, 0, False)}, newfile4)
+
+
+    def test_edit_metadata(self):
+        newfile5 = """
+# A comment
+HELLO = "hithere"
+
+# A new comment
+THIS += "that"
+
+# Another comment
+NOCHANGE = "samevalue"
+OTHER = 'anothervalue'
+
+MULTILINE = "a1 \\
+             a2 \\
+             a3"
+
+MULTILINE2 := " \\
+               b1 \\
+               b2 \\
+               b3 \\
+               "
+
+
+MULTILINE3 = " \\
+              c1 \\
+              c2 \\
+              c3 \\
+"
+
+NEWVAR = "value"
+
+do_functionname() {
+    command1 ${VAL1} ${VAL2}
+    command2 ${VAL3} ${VAL4}
+}
+"""
+
+
+        def handle_var(varname, origvalue, op, newlines):
+            if varname == 'THIS':
+                newlines.append('# A new comment\n')
+            elif varname == 'do_functionname()':
+                newlines.append('NEWVAR = "value"\n')
+                newlines.append('\n')
+            valueitem = varvalues.get(varname, None)
+            if valueitem:
+                return valueitem
+            else:
+                return (origvalue, op, 0, True)
+
+        varvalues = {'HELLO': ('hithere', None, 0, True), 'THIS': ('that', '+=', 0, True)}
+        varlist = ['HELLO', 'THIS', 'do_functionname()']
+        (updated, newlines) = bb.utils.edit_metadata(self._origfile.splitlines(True), varlist, handle_var)
+        self.assertTrue(updated, 'List should be updated but isn\'t')
+        self.assertEqual(newlines, newfile5.splitlines(True))

bitbake/lib/bb/tinfoil.py

@@ -84,6 +84,11 @@ class Tinfoil:
             else:
                 self.parseRecipes()
 
+    def shutdown(self):
+        self.cooker.shutdown(force=True)
+        self.cooker.post_serve()
+        self.cooker.unlockBitbake()
+
 class TinfoilConfigParameters(ConfigParameters):
 
     def __init__(self, **options):

bitbake/lib/bb/ui/buildinfohelper.py

@@ -16,7 +16,6 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-import datetime
 import sys
 import bb
 import re
@@ -24,6 +23,7 @@ import ast
 
 os.environ["DJANGO_SETTINGS_MODULE"] = "toaster.toastermain.settings"
 
+from django.utils import timezone
 import toaster.toastermain.settings as toaster_django_settings
 from toaster.orm.models import Build, Task, Recipe, Layer_Version, Layer, Target, LogMessage, HelpText
 from toaster.orm.models import Target_Image_File, BuildArtifact
@@ -31,6 +31,9 @@ from toaster.orm.models import Variable, VariableHistory
 from toaster.orm.models import Package, Package_File, Target_Installed_Package, Target_File
 from toaster.orm.models import Task_Dependency, Package_Dependency
 from toaster.orm.models import Recipe_Dependency
+
+from toaster.orm.models import Project
+
 from bb.msg import BBLogFormatter as format
 from django.db import models
 from pprint import pformat
@@ -38,7 +41,7 @@ import logging
 
 from django.db import transaction, connection
 
-logger = logging.getLogger("BitBake")
+logger = logging.getLogger("ToasterLogger")
 
 
 class NotExisting(Exception):
@@ -103,7 +106,7 @@ class ORMWrapper(object):
 
         return vars(self)[dictname][key]
 
-    def create_build_object(self, build_info, brbe):
+    def create_build_object(self, build_info, brbe, project_id):
         assert 'machine' in build_info
         assert 'distro' in build_info
         assert 'distro_version' in build_info
@@ -112,7 +115,42 @@ class ORMWrapper(object):
         assert 'build_name' in build_info
         assert 'bitbake_version' in build_info
 
-        build = Build.objects.create(
+        prj = None
+        buildrequest = None
+        if brbe is not None:            # this build was triggered by a request from a user
+            logger.debug(1, "buildinfohelper: brbe is %s" % brbe)
+            from bldcontrol.models import BuildEnvironment, BuildRequest
+            br, be = brbe.split(":")
+            buildrequest = BuildRequest.objects.get(pk = br)
+            prj = buildrequest.project
+
+        elif project_id is not None:    # this build was triggered by an external system for a specific project
+            logger.debug(1, "buildinfohelper: project is %s" % prj)
+            prj = Project.objects.get(pk = project_id)
+
+        else:                           # this build was triggered by a legacy system, or command line interactive mode
+            prj, created = Project.objects.get_or_create(pk=0, name="Default Project")
+            logger.debug(1, "buildinfohelper: project is not specified, defaulting to %s" % prj)
+
+
+        if buildrequest is not None:
+            build = buildrequest.build
+            logger.info("Updating existing build, with %s" % build_info)
+            build.machine=build_info['machine']
+            build.distro=build_info['distro']
+            build.distro_version=build_info['distro_version']
+            started_on=build_info['started_on']
+            completed_on=build_info['started_on']
+            build.cooker_log_path=build_info['cooker_log_path']
+            build.build_name=build_info['build_name']
+            build.bitbake_version=build_info['bitbake_version']
+            build.save()
+
+            Target.objects.filter(build = build).delete()
+
+        else:
+            build = Build.objects.create(
+                                    project = prj,
                                     machine=build_info['machine'],
                                     distro=build_info['distro'],
                                     distro_version=build_info['distro_version'],
@@ -123,17 +161,11 @@ class ORMWrapper(object):
                                     bitbake_version=build_info['bitbake_version'])
 
         logger.debug(1, "buildinfohelper: build is created %s" % build)
-        if brbe is not None:
-            logger.debug(1, "buildinfohelper: brbe is %s" % brbe)
-            from bldcontrol.models import BuildEnvironment, BuildRequest
-            br, be = brbe.split(":")
 
-            buildrequest = BuildRequest.objects.get(pk = br)
+        if buildrequest is not None:
             buildrequest.build = build
             buildrequest.save()
 
-            build.project_id = buildrequest.project_id
-            build.save()
         return build
 
     def create_target_objects(self, target_info):
@@ -158,10 +190,7 @@ class ORMWrapper(object):
         if errors or taskfailures:
             outcome = Build.FAILED
 
-        build.completed_on = datetime.datetime.now()
-        build.timespent = int((build.completed_on - build.started_on).total_seconds())
-        build.errors_no = errors
-        build.warnings_no = warnings
+        build.completed_on = timezone.now()
         build.outcome = outcome
         build.save()
 
@@ -221,12 +250,12 @@ class ORMWrapper(object):
     def get_update_recipe_object(self, recipe_information, must_exist = False):
         assert 'layer_version' in recipe_information
         assert 'file_path' in recipe_information
+        assert 'pathflags' in recipe_information
 
-        if recipe_information['file_path'].startswith(recipe_information['layer_version'].layer.local_path):
-            recipe_information['file_path'] = recipe_information['file_path'][len(recipe_information['layer_version'].layer.local_path):].lstrip("/")
+        assert not recipe_information['file_path'].startswith("/")      # we should have layer-relative paths at all times
 
         recipe_object, created = self._cached_get_or_create(Recipe, layer_version=recipe_information['layer_version'],
-                                     file_path=recipe_information['file_path'])
+                                     file_path=recipe_information['file_path'], pathflags = recipe_information['pathflags'])
         if created and must_exist:
             raise NotExisting("Recipe object created when expected to exist", recipe_information)
 
@@ -247,13 +276,15 @@ class ORMWrapper(object):
         assert 'branch' in layer_version_information
         assert 'commit' in layer_version_information
         assert 'priority' in layer_version_information
+        assert 'local_path' in layer_version_information
 
         layer_version_object, created = Layer_Version.objects.get_or_create(
                                     build = build_obj,
                                     layer = layer_obj,
                                     branch = layer_version_information['branch'],
                                     commit = layer_version_information['commit'],
-                                    priority = layer_version_information['priority']
+                                    priority = layer_version_information['priority'],
+                                    local_path = layer_version_information['local_path'],
                                     )
 
         self.layer_version_objects.append(layer_version_object)
@@ -262,13 +293,11 @@ class ORMWrapper(object):
 
     def get_update_layer_object(self, layer_information, brbe):
         assert 'name' in layer_information
-        assert 'local_path' in layer_information
         assert 'layer_index_url' in layer_information
 
         if brbe is None:
             layer_object, created = Layer.objects.get_or_create(
                                 name=layer_information['name'],
-                                local_path=layer_information['local_path'],
                                 layer_index_url=layer_information['layer_index_url'])
             return layer_object
         else:
@@ -297,7 +326,6 @@ class ORMWrapper(object):
                     for pl in buildrequest.project.projectlayer_set.filter(layercommit__layer__name = brl.name):
                         if pl.layercommit.layer.vcs_url == brl.giturl :
                             layer = pl.layercommit.layer
-                            layer.local_path = layer_information['local_path']
                             layer.save()
                             return layer
 
@@ -639,6 +667,7 @@ class BuildInfoHelper(object):
         self.has_build_history = has_build_history
         self.tmp_dir = self.server.runCommand(["getVariable", "TMPDIR"])[0]
         self.brbe    = self.server.runCommand(["getVariable", "TOASTER_BRBE"])[0]
+        self.project = self.server.runCommand(["getVariable", "TOASTER_PROJECT"])[0]
         logger.debug(1, "buildinfohelper: Build info helper inited %s" % vars(self))
 
 
@@ -657,8 +686,8 @@ class BuildInfoHelper(object):
         build_info['machine'] = self.server.runCommand(["getVariable", "MACHINE"])[0]
         build_info['distro'] = self.server.runCommand(["getVariable", "DISTRO"])[0]
         build_info['distro_version'] = self.server.runCommand(["getVariable", "DISTRO_VERSION"])[0]
-        build_info['started_on'] = datetime.datetime.now()
-        build_info['completed_on'] = datetime.datetime.now()
+        build_info['started_on'] = timezone.now()
+        build_info['completed_on'] = timezone.now()
         build_info['cooker_log_path'] = self.server.runCommand(["getVariable", "BB_CONSOLELOG"])[0]
         build_info['build_name'] = self.server.runCommand(["getVariable", "BUILDNAME"])[0]
         build_info['bitbake_version'] = self.server.runCommand(["getVariable", "BB_VERSION"])[0]
@@ -687,12 +716,12 @@ class BuildInfoHelper(object):
         if self.brbe is None:
             def _slkey_interactive(layer_version):
                 assert isinstance(layer_version, Layer_Version)
-                return len(layer_version.layer.local_path)
+                return len(layer_version.local_path)
 
             # Heuristics: we always match recipe to the deepest layer path in the discovered layers
             for lvo in sorted(self.orm_wrapper.layer_version_objects, reverse=True, key=_slkey_interactive):
                 # we can match to the recipe file path
-                if path.startswith(lvo.layer.local_path):
+                if path.startswith(lvo.local_path):
                     return lvo
 
         else:
@@ -718,21 +747,33 @@ class BuildInfoHelper(object):
                             return lvo
 
         #if we get here, we didn't read layers correctly; dump whatever information we have on the error log
-        logger.error("Could not match layer version for recipe path %s : %s" % (path, self.orm_wrapper.layer_version_objects))
+        logger.warn("Could not match layer version for recipe path %s : %s" % (path, self.orm_wrapper.layer_version_objects))
 
         #mockup the new layer
-        unknown_layer, created = Layer.objects.get_or_create(name="__FIXME__unidentified_layer", local_path="/", layer_index_url="")
+        unknown_layer, created = Layer.objects.get_or_create(name="__FIXME__unidentified_layer", layer_index_url="")
         unknown_layer_version_obj, created = Layer_Version.objects.get_or_create(layer = unknown_layer, build = self.internal_state['build'])
 
+        # append it so we don't run into this error again and again
+        self.orm_wrapper.layer_version_objects.append(unknown_layer_version_obj)
+
         return unknown_layer_version_obj
 
     def _get_recipe_information_from_taskfile(self, taskfile):
         localfilepath = taskfile.split(":")[-1]
+        filepath_flags = ":".join(sorted(taskfile.split(":")[:-1]))
         layer_version_obj = self._get_layer_version_for_path(localfilepath)
 
+
+
         recipe_info = {}
         recipe_info['layer_version'] = layer_version_obj
-        recipe_info['file_path'] = taskfile
+        recipe_info['file_path'] = localfilepath
+        recipe_info['pathflags'] = filepath_flags
+
+        if recipe_info['file_path'].startswith(recipe_info['layer_version'].local_path):
+            recipe_info['file_path'] = recipe_info['file_path'][len(recipe_info['layer_version'].local_path):].lstrip("/")
+        else:
+            raise RuntimeError("Recipe file path %s is not under layer version at %s" % (recipe_info['file_path'], recipe_info['layer_version'].local_path))
 
         return recipe_info
 
@@ -784,6 +825,7 @@ class BuildInfoHelper(object):
         for layer in layerinfos:
             try:
                 self.internal_state['lvs'][self.orm_wrapper.get_update_layer_object(layerinfos[layer], self.brbe)] = layerinfos[layer]['version']
+                self.internal_state['lvs'][self.orm_wrapper.get_update_layer_object(layerinfos[layer], self.brbe)]['local_path'] = layerinfos[layer]['local_path']
             except NotExisting as nee:
                 logger.warn("buildinfohelper: cannot identify layer exception:%s " % nee)
 
@@ -792,7 +834,7 @@ class BuildInfoHelper(object):
         assert '_pkgs' in vars(event)
         build_information = self._get_build_information()
 
-        build_obj = self.orm_wrapper.create_build_object(build_information, self.brbe)
+        build_obj = self.orm_wrapper.create_build_object(build_information, self.brbe, self.project)
 
         self.internal_state['build'] = build_obj
 
@@ -814,6 +856,31 @@ class BuildInfoHelper(object):
 
         # Save build configuration
         data = self.server.runCommand(["getAllKeysWithFlags", ["doc", "func"]])[0]
+
+        # convert the paths from absolute to relative to either the build directory or layer checkouts
+        path_prefixes = []
+
+        if self.brbe is not None:
+            br_id, be_id = self.brbe.split(":")
+            from bldcontrol.models import BuildEnvironment, BuildRequest
+            be = BuildEnvironment.objects.get(pk = be_id)
+            path_prefixes.append(be.builddir)
+
+        for layer in sorted(self.orm_wrapper.layer_version_objects, key = lambda x:len(x.local_path), reverse=True):
+            path_prefixes.append(layer.local_path)
+
+        # we strip the prefixes
+        for k in data:
+            if not bool(data[k]['func']):
+                for vh in data[k]['history']:
+                    if not 'documentation.conf' in vh['file']:
+                        abs_file_name = vh['file']
+                        for pp in path_prefixes:
+                            if abs_file_name.startswith(pp + "/"):
+                                vh['file']=abs_file_name[len(pp + "/"):]
+                                break
+
+        # save the variables
         self.orm_wrapper.save_build_variables(build_obj, data)
 
         return self.brbe
@@ -896,8 +963,8 @@ class BuildInfoHelper(object):
 
             recipe_information = self._get_recipe_information_from_taskfile(taskfile)
             try:
-                if recipe_information['file_path'].startswith(recipe_information['layer_version'].layer.local_path):
-                    recipe_information['file_path'] = recipe_information['file_path'][len(recipe_information['layer_version'].layer.local_path):].lstrip("/")
+                if recipe_information['file_path'].startswith(recipe_information['layer_version'].local_path):
+                    recipe_information['file_path'] = recipe_information['file_path'][len(recipe_information['layer_version'].local_path):].lstrip("/")
 
                 recipe_object = Recipe.objects.get(layer_version = recipe_information['layer_version'],
                             file_path__endswith = recipe_information['file_path'],
@@ -988,7 +1055,7 @@ class BuildInfoHelper(object):
             mevent.taskhash = taskhash
             task_information = self._get_task_information(mevent,recipe)
 
-            task_information['start_time'] = datetime.datetime.now()
+            task_information['start_time'] = timezone.now()
             task_information['outcome'] = Task.OUTCOME_NA
             task_information['sstate_checksum'] = taskhash
             task_information['sstate_result'] = Task.SSTATE_MISS
@@ -1048,8 +1115,9 @@ class BuildInfoHelper(object):
         self.internal_state['recipes'] = {}
         for pn in event._depgraph['pn']:
 
-            file_name = event._depgraph['pn'][pn]['filename']
-            layer_version_obj = self._get_layer_version_for_path(file_name.split(":")[-1])
+            file_name = event._depgraph['pn'][pn]['filename'].split(":")[-1]
+            pathflags = ":".join(sorted(event._depgraph['pn'][pn]['filename'].split(":")[:-1]))
+            layer_version_obj = self._get_layer_version_for_path(file_name)
 
             assert layer_version_obj is not None
 
@@ -1079,6 +1147,13 @@ class BuildInfoHelper(object):
                 recipe_info['bugtracker'] = event._depgraph['pn'][pn]['bugtracker']
 
             recipe_info['file_path'] = file_name
+            recipe_info['pathflags'] = pathflags
+
+            if recipe_info['file_path'].startswith(recipe_info['layer_version'].local_path):
+                recipe_info['file_path'] = recipe_info['file_path'][len(recipe_info['layer_version'].local_path):].lstrip("/")
+            else:
+                raise RuntimeError("Recipe file path %s is not under layer version at %s" % (recipe_info['file_path'], recipe_info['layer_version'].local_path))
+
             recipe = self.orm_wrapper.get_update_recipe_object(recipe_info)
             recipe.is_image = False
             if 'inherits' in event._depgraph['pn'][pn].keys():
@@ -1155,6 +1230,7 @@ class BuildInfoHelper(object):
                             )
 
     def _store_build_done(self, errorcode):
+        logger.info("Build exited with errorcode %d", errorcode)
         br_id, be_id = self.brbe.split(":")
         from bldcontrol.models import BuildEnvironment, BuildRequest
         be = BuildEnvironment.objects.get(pk = be_id)
@@ -1174,7 +1250,7 @@ class BuildInfoHelper(object):
         mockevent.levelno = format.ERROR
         mockevent.msg = text
         mockevent.pathname = '-- None'
-        mockevent.lineno = -1
+        mockevent.lineno = LogMessage.ERROR
         self.store_log_event(mockevent)
 
     def store_log_exception(self, text, backtrace = ""):
@@ -1198,13 +1274,12 @@ class BuildInfoHelper(object):
                 if not 'backlog' in self.internal_state:
                     self.internal_state['backlog'] = []
                 self.internal_state['backlog'].append(event)
-            else:   # we're under Toaster control, post the errors to the build request
+                return
+            else:   # we're under Toaster control, the build is already created
                 from bldcontrol.models import BuildRequest, BRError
                 br, be = self.brbe.split(":")
                 buildrequest = BuildRequest.objects.get(pk = br)
-                brerror = BRError.objects.create(req = buildrequest, errtype="build", errmsg = event.msg)
-
-            return
+                self.internal_state['build'] = buildrequest.build
 
         if 'build' in self.internal_state and 'backlog' in self.internal_state:
             # if we have a backlog of events, do our best to save them here
@@ -1213,7 +1288,7 @@ class BuildInfoHelper(object):
                 logger.debug(1, "buildinfohelper: Saving stored event %s " % tempevent)
                 self.store_log_event(tempevent)
             else:
-                logger.error("buildinfohelper: Events not saved: %s" % self.internal_state['backlog'])
+                logger.info("buildinfohelper: All events saved")
                 del self.internal_state['backlog']
 
         log_information = {}
@@ -1222,14 +1297,15 @@ class BuildInfoHelper(object):
             log_information['level'] = LogMessage.ERROR
         elif event.levelno == format.WARNING:
             log_information['level'] = LogMessage.WARNING
-        elif event.levelno == -1:   # toaster self-logging
-            log_information['level'] = -1
+        elif event.levelno == -2:   # toaster self-logging
+            log_information['level'] = -2
         else:
             log_information['level'] = LogMessage.INFO
 
         log_information['message'] = event.msg
         log_information['pathname'] = event.pathname
         log_information['lineno'] = event.lineno
+        logger.info("Logging error 2: %s" % log_information)
         self.orm_wrapper.create_logmessage(log_information)
 
     def close(self, errorcode):

bitbake/lib/bb/ui/crumbs/builder.py

@@ -309,7 +309,7 @@ class Parameters:
 
 def hob_conf_filter(fn, data):
     if fn.endswith("/local.conf"):
-        distro = data.getVar("DISTRO_HOB")
+        distro = data.getVar("DISTRO_HOB", False)
         if distro:
             if distro != "defaultsetup":
                 data.setVar("DISTRO", distro)
@@ -320,13 +320,13 @@ def hob_conf_filter(fn, data):
                 "BB_NUMBER_THREADS_HOB", "PARALLEL_MAKE_HOB", "DL_DIR_HOB", \
                 "SSTATE_DIR_HOB", "SSTATE_MIRRORS_HOB", "INCOMPATIBLE_LICENSE_HOB"]
         for key in keys:
-            var_hob = data.getVar(key)
+            var_hob = data.getVar(key, False)
             if var_hob:
                 data.setVar(key.split("_HOB")[0], var_hob)
         return
 
     if fn.endswith("/bblayers.conf"):
-        layers = data.getVar("BBLAYERS_HOB")
+        layers = data.getVar("BBLAYERS_HOB", False)
         if layers:
             data.setVar("BBLAYERS", layers)
         return

bitbake/lib/bb/ui/toasterui.py

@@ -43,7 +43,7 @@ import xmlrpclib
 
 featureSet = [bb.cooker.CookerFeatures.HOB_EXTRA_CACHES, bb.cooker.CookerFeatures.SEND_DEPENDS_TREE, bb.cooker.CookerFeatures.BASEDATASTORE_TRACKING, bb.cooker.CookerFeatures.SEND_SANITYEVENTS]
 
-logger = logging.getLogger("BitBake")
+logger = logging.getLogger("ToasterLogger")
 interactive = sys.stdout.isatty()
 
 
@@ -66,7 +66,6 @@ def _log_settings_from_server(server):
 
 
 def main(server, eventHandler, params ):
-
     helper = uihelper.BBUIHelper()
 
     console = logging.StreamHandler(sys.stdout)
@@ -75,6 +74,7 @@ def main(server, eventHandler, params ):
     bb.msg.addDefaultlogFilter(console)
     console.setFormatter(format)
     logger.addHandler(console)
+    logger.setLevel(logging.INFO)
 
     includelogs, loglines, consolelogfile = _log_settings_from_server(server)
 
@@ -88,7 +88,7 @@ def main(server, eventHandler, params ):
 
     if not params.observe_only:
         logger.error("ToasterUI can only work in observer mode")
-        return
+        return 1
 
 
     main.shutdown = 0
@@ -141,10 +141,12 @@ def main(server, eventHandler, params ):
                 continue
 
             if isinstance(event, logging.LogRecord):
+                if event.levelno == -1:
+                    event.levelno = format.ERROR
+
                 buildinfohelper.store_log_event(event)
                 if event.levelno >= format.ERROR:
                     errors = errors + 1
-                    return_value = 1
                 elif event.levelno == format.WARNING:
                     warnings = warnings + 1
                 # For "normal" logging conditions, don't show note logs from tasks
@@ -158,7 +160,6 @@ def main(server, eventHandler, params ):
 
             if isinstance(event, bb.build.TaskFailed):
                 buildinfohelper.update_and_store_task(event)
-                return_value = 1
                 logfile = event.logfile
                 if logfile and os.path.exists(logfile):
                     bb.error("Logfile of failure stored in: %s" % logfile)
@@ -188,7 +189,6 @@ def main(server, eventHandler, params ):
                 continue
 
             if isinstance(event, bb.event.NoProvider):
-                return_value = 1
                 errors = errors + 1
                 if event._runtime:
                     r = "R"
@@ -238,27 +238,19 @@ def main(server, eventHandler, params ):
             if isinstance(event, (bb.event.TreeDataPreparationStarted, bb.event.TreeDataPreparationCompleted)):
                 continue
 
-            if isinstance(event, (bb.event.BuildCompleted)):
-                continue
+            if isinstance(event, (bb.event.BuildCompleted, bb.command.CommandFailed)):
 
-            if isinstance(event, (bb.command.CommandCompleted,
-                                  bb.command.CommandFailed,
-                                  bb.command.CommandExit)):
-                errorcode = 0
+		errorcode = 0
                 if (isinstance(event, bb.command.CommandFailed)):
-                    event.levelno = format.ERROR
-                    event.msg = "Command Failed " + event.error
-                    event.pathname = ""
-                    event.lineno = 0
-                    buildinfohelper.store_log_event(event)
                     errors += 1
                     errorcode = 1
                     logger.error("Command execution failed: %s", event.error)
 
+                # update the build info helper on BuildCompleted, not on CommandXXX
                 buildinfohelper.update_build_information(event, errors, warnings, taskfailures)
                 buildinfohelper.close(errorcode)
                 # mark the log output; controllers may kill the toasterUI after seeing this log
-                logger.info("ToasterUI build done")
+                logger.info("ToasterUI build done 1, brbe: %s" % buildinfohelper.brbe )
 
                 # we start a new build info
                 if buildinfohelper.brbe is not None:
@@ -272,6 +264,14 @@ def main(server, eventHandler, params ):
                     taskfailures = []
                     buildinfohelper = BuildInfoHelper(server, build_history_enabled)
 
+                logger.info("ToasterUI build done 2")
+                continue
+
+            if isinstance(event, (bb.command.CommandCompleted,
+                                  bb.command.CommandFailed,
+                                  bb.command.CommandExit)):
+                errorcode = 0
+
                 continue
 
             if isinstance(event, bb.event.MetadataEvent):
@@ -316,6 +316,7 @@ def main(server, eventHandler, params ):
                 continue
 
             logger.error("Unknown event: %s", event)
+            return_value += 1
 
         except EnvironmentError as ioerror:
             # ignore interrupted io
@@ -344,10 +345,13 @@ def main(server, eventHandler, params ):
             except Exception as ce:
                 logger.error("CRITICAL - Failed to to save toaster exception to the database: %s" % str(ce))
 
+            # make sure we return with an error
+            return_value += 1
             pass
 
     if interrupted:
         if return_value == 0:
-            return_value = 1
+            return_value += 1
 
+    logger.warn("Return value is %d", return_value)
     return return_value

bitbake/lib/bb/utils.py

@@ -31,6 +31,7 @@ import subprocess
 import glob
 import traceback
 import errno
+import signal
 from commands import getstatusoutput
 from contextlib import contextmanager
 
@@ -412,10 +413,30 @@ def fileslocked(files):
     for lock in locks:
         bb.utils.unlockfile(lock)
 
-def lockfile(name, shared=False, retry=True):
+@contextmanager
+def timeout(seconds):
+    def timeout_handler(signum, frame):
+        pass
+
+    original_handler = signal.signal(signal.SIGALRM, timeout_handler)
+
+    try:
+        signal.alarm(seconds)
+        yield
+    finally:
+        signal.alarm(0)
+        signal.signal(signal.SIGALRM, original_handler)
+
+def lockfile(name, shared=False, retry=True, block=False):
     """
-    Use the file fn as a lock file, return when the lock has been acquired.
-    Returns a variable to pass to unlockfile().
+    Use the specified file as a lock file, return when the lock has
+    been acquired. Returns a variable to pass to unlockfile().
+    Parameters:
+        retry: True to re-try locking if it fails, False otherwise
+        block: True to block until the lock succeeds, False otherwise
+    The retry and block parameters are kind of equivalent unless you
+    consider the possibility of sending a signal to the process to break
+    out - at which point you want block=True rather than retry=True.
     """
     dirname = os.path.dirname(name)
     mkdirhier(dirname)
@@ -428,7 +449,7 @@ def lockfile(name, shared=False, retry=True):
     op = fcntl.LOCK_EX
     if shared:
         op = fcntl.LOCK_SH
-    if not retry:
+    if not retry and not block:
         op = op | fcntl.LOCK_NB
 
     while True:
@@ -601,11 +622,30 @@ def build_environment(d):
         if export:
             os.environ[var] = d.getVar(var, True) or ""
 
+def _check_unsafe_delete_path(path):
+    """
+    Basic safeguard against recursively deleting something we shouldn't. If it returns True,
+    the caller should raise an exception with an appropriate message.
+    NOTE: This is NOT meant to be a security mechanism - just a guard against silly mistakes
+    with potentially disastrous results.
+    """
+    extra = ''
+    # HOME might not be /home/something, so in case we can get it, check against it
+    homedir = os.environ.get('HOME', '')
+    if homedir:
+        extra = '|%s' % homedir
+    if re.match('(/|//|/home|/home/[^/]*%s)$' % extra, os.path.abspath(path)):
+        return True
+    return False
+
 def remove(path, recurse=False):
     """Equivalent to rm -f or rm -rf"""
     if not path:
         return
     if recurse:
+        for name in glob.glob(path):
+            if _check_unsafe_delete_path(path):
+                raise Exception('bb.utils.remove: called with dangerous path "%s" and recurse=True, refusing to delete!' % path)
         # shutil.rmtree(name) would be ideal but its too slow
         subprocess.call(['rm', '-rf'] + glob.glob(path))
         return
@@ -619,6 +659,8 @@ def remove(path, recurse=False):
 def prunedir(topdir):
     # Delete everything reachable from the directory named in 'topdir'.
     # CAUTION:  This is dangerous!
+    if _check_unsafe_delete_path(topdir):
+        raise Exception('bb.utils.prunedir: called with dangerous path "%s", refusing to delete!' % topdir)
     for root, dirs, files in os.walk(topdir, topdown = False):
         for name in files:
             os.remove(os.path.join(root, name))
@@ -887,11 +929,17 @@ def cpu_count():
 def nonblockingfd(fd):
     fcntl.fcntl(fd, fcntl.F_SETFL, fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK)
 
-def process_profilelog(fn):
-    pout = open(fn + '.processed', 'w')
+def process_profilelog(fn, pout = None):
+    # Either call with a list of filenames and set pout or a filename and optionally pout.
+    if not pout:
+        pout = fn + '.processed'
+    pout = open(pout, 'w')
    
     import pstats
-    p = pstats.Stats(fn, stream=pout)
+    if isinstance(fn, list):
+        p = pstats.Stats(*fn, stream=pout)
+    else:
+        p = pstats.Stats(fn, stream=pout)
     p.sort_stats('time')
     p.print_stats()
     p.print_callers()
@@ -942,77 +990,207 @@ def exec_flat_python_func(func, *args, **kwargs):
     bb.utils.better_exec(comp, context, code, '<string>')
     return context['retval']
 
-def edit_metadata_file(meta_file, variables, func):
-    """Edit a recipe or config file and modify one or more specified
-    variable values set in the file using a specified callback function.
-    The file is only written to if the value(s) actually change.
+def edit_metadata(meta_lines, variables, varfunc, match_overrides=False):
+    """Edit lines from a recipe or config file and modify one or more
+    specified variable values set in the file using a specified callback
+    function. Lines are expected to have trailing newlines.
+    Parameters:
+        meta_lines: lines from the file; can be a list or an iterable
+            (e.g. file pointer)
+        variables: a list of variable names to look for. Functions
+            may also be specified, but must be specified with '()' at
+            the end of the name. Note that the function doesn't have
+            any intrinsic understanding of _append, _prepend, _remove,
+            or overrides, so these are considered as part of the name.
+            These values go into a regular expression, so regular
+            expression syntax is allowed.
+        varfunc: callback function called for every variable matching
+            one of the entries in the variables parameter. The function
+            should take four arguments:
+                varname: name of variable matched
+                origvalue: current value in file
+                op: the operator (e.g. '+=')
+                newlines: list of lines up to this point. You can use
+                    this to prepend lines before this variable setting
+                    if you wish.
+            and should return a three-element tuple:
+                newvalue: new value to substitute in, or None to drop
+                    the variable setting entirely. (If the removal
+                    results in two consecutive blank lines, one of the
+                    blank lines will also be dropped).
+                newop: the operator to use - if you specify None here,
+                    the original operation will be used.
+                indent: number of spaces to indent multi-line entries,
+                    or -1 to indent up to the level of the assignment
+                    and opening quote, or a string to use as the indent.
+                minbreak: True to allow the first element of a
+                    multi-line value to continue on the same line as
+                    the assignment, False to indent before the first
+                    element.
+        match_overrides: True to match items with _overrides on the end,
+            False otherwise
+    Returns a tuple:
+        updated:
+            True if changes were made, False otherwise.
+        newlines:
+            Lines after processing
     """
+
     var_res = {}
+    if match_overrides:
+        override_re = '(_[a-zA-Z0-9-_$(){}]+)?'
+    else:
+        override_re = ''
     for var in variables:
-        var_res[var] = re.compile(r'^%s[ \t]*[?+]*=' % var)
+        if var.endswith('()'):
+            var_res[var] = re.compile('^(%s%s)[ \\t]*\([ \\t]*\)[ \\t]*{' % (var[:-2].rstrip(), override_re))
+        else:
+            var_res[var] = re.compile('^(%s%s)[ \\t]*[?+:.]*=[+.]*[ \\t]*(["\'])' % (var, override_re))
 
     updated = False
     varset_start = ''
+    varlines = []
     newlines = []
     in_var = None
     full_value = ''
+    var_end = ''
 
     def handle_var_end():
-        (newvalue, indent, minbreak) = func(in_var, full_value)
-        if newvalue != full_value:
-            if isinstance(newvalue, list):
-                intentspc = ' ' * indent
-                if minbreak:
-                    # First item on first line
-                    if len(newvalue) == 1:
-                        newlines.append('%s "%s"\n' % (varset_start, newvalue[0]))
+        prerun_newlines = newlines[:]
+        op = varset_start[len(in_var):].strip()
+        (newvalue, newop, indent, minbreak) = varfunc(in_var, full_value, op, newlines)
+        changed = (prerun_newlines != newlines)
+
+        if newvalue is None:
+            # Drop the value
+            return True
+        elif newvalue != full_value or (newop not in [None, op]):
+            if newop not in [None, op]:
+                # Callback changed the operator
+                varset_new = "%s %s" % (in_var, newop)
+            else:
+                varset_new = varset_start
+
+            if isinstance(indent, (int, long)):
+                if indent == -1:
+                    indentspc = ' ' * (len(varset_new) + 2)
+                else:
+                    indentspc = ' ' * indent
+            else:
+                indentspc = indent
+            if in_var.endswith('()'):
+                # A function definition
+                if isinstance(newvalue, list):
+                    newlines.append('%s {\n%s%s\n}\n' % (varset_new, indentspc, ('\n%s' % indentspc).join(newvalue)))
+                else:
+                    if not newvalue.startswith('\n'):
+                        newvalue = '\n' + newvalue
+                    if not newvalue.endswith('\n'):
+                        newvalue = newvalue + '\n'
+                    newlines.append('%s {%s}\n' % (varset_new, newvalue))
+            else:
+                # Normal variable
+                if isinstance(newvalue, list):
+                    if not newvalue:
+                        # Empty list -> empty string
+                        newlines.append('%s ""\n' % varset_new)
+                    elif minbreak:
+                        # First item on first line
+                        if len(newvalue) == 1:
+                            newlines.append('%s "%s"\n' % (varset_new, newvalue[0]))
+                        else:
+                            newlines.append('%s "%s \\\n' % (varset_new, newvalue[0]))
+                            for item in newvalue[1:]:
+                                newlines.append('%s%s \\\n' % (indentspc, item))
+                            newlines.append('%s"\n' % indentspc)
                     else:
-                        newlines.append('%s "%s\\\n' % (varset_start, newvalue[0]))
-                        for item in newvalue[1:]:
-                            newlines.append('%s%s \\\n' % (intentspc, item))
+                        # No item on first line
+                        newlines.append('%s " \\\n' % varset_new)
+                        for item in newvalue:
+                            newlines.append('%s%s \\\n' % (indentspc, item))
                         newlines.append('%s"\n' % indentspc)
                 else:
-                    # No item on first line
-                    newlines.append('%s " \\\n' % varset_start)
-                    for item in newvalue:
-                        newlines.append('%s%s \\\n' % (intentspc, item))
-                    newlines.append('%s"\n' % intentspc)
-            else:
-                newlines.append('%s "%s"\n' % (varset_start, newvalue))
+                    newlines.append('%s "%s"\n' % (varset_new, newvalue))
             return True
-        return False
+        else:
+            # Put the old lines back where they were
+            newlines.extend(varlines)
+            # If newlines was touched by the function, we'll need to return True
+            return changed
 
-    with open(meta_file, 'r') as f:
-        for line in f:
-            if in_var:
-                value = line.rstrip()
-                full_value += value[:-1]
-                if value.endswith('"') or value.endswith("'"):
-                    if handle_var_end():
-                        updated = True
-                    in_var = None
+    checkspc = False
+
+    for line in meta_lines:
+        if in_var:
+            value = line.rstrip()
+            varlines.append(line)
+            if in_var.endswith('()'):
+                full_value += '\n' + value
             else:
-                matched = False
-                for (varname, var_re) in var_res.iteritems():
-                    if var_re.match(line):
-                        splitvalue = line.split('"', 1)
-                        varset_start = splitvalue[0].rstrip()
-                        value = splitvalue[1].rstrip()
-                        if value.endswith('\\'):
-                            value = value[:-1]
-                        full_value = value
-                        if value.endswith('"') or value.endswith("'"):
-                            if handle_var_end():
-                                updated = True
-                        else:
-                            in_var = varname
-                        matched = True
-                        break
-                if not matched:
-                    newlines.append(line)
+                full_value += value[:-1]
+            if value.endswith(var_end):
+                if in_var.endswith('()'):
+                    if full_value.count('{') - full_value.count('}') >= 0:
+                        continue
+                full_value = full_value[:-1]
+                if handle_var_end():
+                    updated = True
+                    checkspc = True
+                in_var = None
+        else:
+            skip = False
+            for (varname, var_re) in var_res.iteritems():
+                res = var_re.match(line)
+                if res:
+                    isfunc = varname.endswith('()')
+                    if isfunc:
+                        splitvalue = line.split('{', 1)
+                        var_end = '}'
+                    else:
+                        var_end = res.groups()[-1]
+                        splitvalue = line.split(var_end, 1)
+                    varset_start = splitvalue[0].rstrip()
+                    value = splitvalue[1].rstrip()
+                    if not isfunc and value.endswith('\\'):
+                        value = value[:-1]
+                    full_value = value
+                    varlines = [line]
+                    in_var = res.group(1)
+                    if isfunc:
+                        in_var += '()'
+                    if value.endswith(var_end):
+                        full_value = full_value[:-1]
+                        if handle_var_end():
+                            updated = True
+                            checkspc = True
+                        in_var = None
+                    skip = True
+                    break
+            if not skip:
+                if checkspc:
+                    checkspc = False
+                    if newlines[-1] == '\n' and line == '\n':
+                        # Squash blank line if there are two consecutive blanks after a removal
+                        continue
+                newlines.append(line)
+    return (updated, newlines)
+
+
+def edit_metadata_file(meta_file, variables, varfunc):
+    """Edit a recipe or config file and modify one or more specified
+    variable values set in the file using a specified callback function.
+    The file is only written to if the value(s) actually change.
+    This is basically the file version of edit_metadata(), see that
+    function's description for parameter/usage information.
+    Returns True if the file was written to, False otherwise.
+    """
+    with open(meta_file, 'r') as f:
+        (updated, newlines) = edit_metadata(f, variables, varfunc)
     if updated:
         with open(meta_file, 'w') as f:
             f.writelines(newlines)
+    return updated
+
 
 def edit_bblayers_conf(bblayers_conf, add, remove):
     """Edit bblayers.conf, adding and/or removing layers"""
@@ -1041,7 +1219,7 @@ def edit_bblayers_conf(bblayers_conf, add, remove):
     # Need to use a list here because we can't set non-local variables from a callback in python 2.x
     bblayercalls = []
 
-    def handle_bblayers(varname, origvalue):
+    def handle_bblayers(varname, origvalue, op, newlines):
         bblayercalls.append(varname)
         updated = False
         bblayers = [remove_trailing_sep(x) for x in origvalue.split()]
@@ -1065,9 +1243,9 @@ def edit_bblayers_conf(bblayers_conf, add, remove):
                     notadded.append(addlayer)
 
         if updated:
-            return (bblayers, 2, False)
+            return (bblayers, None, 2, False)
         else:
-            return (origvalue, 2, False)
+            return (origvalue, None, 2, False)
 
     edit_metadata_file(bblayers_conf, ['BBLAYERS'], handle_bblayers)
 
@@ -1076,3 +1254,19 @@ def edit_bblayers_conf(bblayers_conf, add, remove):
 
     return (notadded, notremoved)
 
+
+def get_file_layer(filename, d):
+    """Determine the collection (as defined by a layer's layer.conf file) containing the specified file"""
+    collections = (d.getVar('BBFILE_COLLECTIONS', True) or '').split()
+    collection_res = {}
+    for collection in collections:
+        collection_res[collection] = d.getVar('BBFILE_PATTERN_%s' % collection, True) or ''
+
+    # Use longest path so we handle nested layers
+    matchlen = 0
+    match = None
+    for collection, regex in collection_res.iteritems():
+        if len(regex) > matchlen and re.match(regex, filename):
+            matchlen = len(regex)
+            match = collection
+    return match

bitbake/lib/prserv/db.py

@@ -15,6 +15,16 @@ sqlversion = sqlite3.sqlite_version_info
 if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
     raise Exception("sqlite3 version 3.3.0 or later is required.")
 
+#
+# "No History" mode - for a given query tuple (version, pkgarch, checksum),
+# the returned value will be the largest among all the values of the same
+# (version, pkgarch). This means the PR value returned can NOT be decremented.
+#
+# "History" mode - Return a new higher value for previously unseen query
+# tuple (version, pkgarch, checksum), otherwise return historical value.
+# Value can decrement if returning to a previous build.
+#
+
 class PRTable(object):
     def __init__(self, conn, table, nohist):
         self.conn = conn

bitbake/lib/prserv/serv.py

@@ -1,7 +1,6 @@
 import os,sys,logging
-import signal, time, atexit, threading
+import signal, time
 from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
-import xmlrpclib
 import threading
 import Queue
 

bitbake/lib/toaster/bldcollector/admin.py

@@ -1,6 +1,6 @@
 from django.contrib import admin
 from django.contrib.admin.filters import RelatedFieldListFilter
-from .models import BitbakeVersion, Release, LayerSource, ToasterSetting
+from orm.models import BitbakeVersion, Release, LayerSource, ToasterSetting
 from django.forms.widgets import Textarea
 from django import forms
 import django.db.models as models
@@ -31,4 +31,3 @@ admin.site.register(LayerSource, LayerSourceAdmin)
 admin.site.register(BitbakeVersion, BitbakeVersionAdmin)
 admin.site.register(Release, ReleaseAdmin)
 admin.site.register(ToasterSetting, ToasterSettingAdmin)
-

bitbake/lib/toaster/bldcollector/urls.py

@@ -20,8 +20,7 @@
 from django.conf.urls import patterns, include, url
 from django.views.generic import RedirectView
 
-urlpatterns = patterns('orm.views',
+urlpatterns = patterns('bldcollector.views',
         # landing point for pushing a bitbake_eventlog.json file to this toaster instace
         url(r'^eventfile$', 'eventfile', name='eventfile'),
         )
-

bitbake/lib/toaster/bldcollector/views.py

@@ -37,7 +37,7 @@ from django.views.decorators.csrf import csrf_exempt
 def eventfile(request):
     """ Receives a file by POST, and runs toaster-eventreply on this file """
     if request.method != "POST":
-        return HttpResponseBadRequest("This API only accepts POST requests. Post a file with:\n\ncurl -F eventlog=@bitbake_eventlog.json http[s]://[server-address]/orm/eventfile\n", content_type="text/plain;utf8")
+        return HttpResponseBadRequest("This API only accepts POST requests. Post a file with:\n\ncurl -F eventlog=@bitbake_eventlog.json %s\n" % request.build_absolute_uri(reverse('eventfile')), content_type="text/plain;utf8")
 
     # write temporary file
     (handle, abstemppath) = tempfile.mkstemp(dir="/tmp/")
@@ -55,6 +55,8 @@ def eventfile(request):
     scriptenv["DATABASE_URL"] = toastermain.settings.getDATABASE_URL()
 
     # run the data loading process and return the results
-    (out, err) = subprocess.Popen([import_script, abstemppath], stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=scriptenv).communicate()
-    os.remove(abstemppath)
-    return HttpResponse("%s\n%s" % (out, err), content_type="text/plain;utf8")
+    importer = subprocess.Popen([import_script, abstemppath], stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=scriptenv)
+    (out, err) = importer.communicate()
+    if importer.returncode == 0:
+        os.remove(abstemppath)
+    return HttpResponse("== Retval %d\n== STDOUT\n%s\n\n== STDERR\n%s" % (importer.returncode, out, err), content_type="text/plain;utf8")

bitbake/lib/toaster/bldcontrol/bbcontroller.py

@@ -190,6 +190,9 @@ class BuildEnvironmentController(object):
         """
         raise Exception("Must override BE release")
 
+    def triggerBuild(self, bitbake, layers, variables, targets):
+        raise Exception("Must override BE release")
+
 class ShellCmdException(Exception):
     pass
 

bitbake/lib/toaster/bldcontrol/localhostbecontroller.py

@@ -54,7 +54,7 @@ class LocalhostBEController(BuildEnvironmentController):
         if cwd is None:
             cwd = self.be.sourcedir
 
-        #logger.debug("lbc_shellcmmd: (%s) %s" % (cwd, command))
+        logger.debug("lbc_shellcmmd: (%s) %s" % (cwd, command))
         p = subprocess.Popen(command, cwd = cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         (out,err) = p.communicate()
         p.wait()
@@ -63,27 +63,22 @@ class LocalhostBEController(BuildEnvironmentController):
                 err = "command: %s \n%s" % (command, out)
             else:
                 err = "command: %s \n%s" % (command, err)
-            #logger.warn("localhostbecontroller: shellcmd error %s" % err)
+            logger.warn("localhostbecontroller: shellcmd error %s" % err)
             raise ShellCmdException(err)
         else:
-            #logger.debug("localhostbecontroller: shellcmd success")
+            logger.debug("localhostbecontroller: shellcmd success")
             return out
 
-    def _createdirpath(self, path):
-        from os.path import dirname as DN
-        if path == "":
-            raise Exception("Invalid path creation specified.")
-        if not os.path.exists(DN(path)):
-            self._createdirpath(DN(path))
-        if not os.path.exists(path):
-            os.mkdir(path, 0755)
-
     def _setupBE(self):
         assert self.pokydirname and os.path.exists(self.pokydirname)
-        self._createdirpath(self.be.builddir)
-        self._shellcmd("bash -c \"source %s/oe-init-build-env %s\"" % (self.pokydirname, self.be.builddir))
+        path = self.be.builddir
+        if not path:
+            raise Exception("Invalid path creation specified.")
+        if not os.path.exists(path):
+            os.makedirs(path, 0755)
+        self._shellcmd("bash -c \"source %s/oe-init-build-env %s\"" % (self.pokydirname, path))
         # delete the templateconf.cfg; it may come from an unsupported layer configuration
-        os.remove(os.path.join(self.be.builddir, "conf/templateconf.cfg"))
+        os.remove(os.path.join(path, "conf/templateconf.cfg"))
 
 
     def writeConfFile(self, file_name, variable_list = None, raw = None):
@@ -113,19 +108,25 @@ class LocalhostBEController(BuildEnvironmentController):
         # get the file length; we need to detect the _last_ start of the toaster UI, not the first
         toaster_ui_log_filelength = 0
         if os.path.exists(toaster_ui_log_filepath):
-            with open(toaster_ui_log_filepath, "r") as f:
+            with open(toaster_ui_log_filepath, "w") as f:
                 f.seek(0, 2)    # jump to the end
                 toaster_ui_log_filelength = f.tell()
 
-        cmd = "bash -c \"source %s/oe-init-build-env %s 2>&1 >toaster_server.log && bitbake --read conf/toaster-pre.conf --postread conf/toaster.conf --server-only -t xmlrpc -B 0.0.0.0:0 2>&1 >toaster_server.log && DATABASE_URL=%s BBSERVER=0.0.0.0:-1 daemon -d -i -D %s -o toaster_ui.log -- %s --observe-only -u toasterui &\"" % (self.pokydirname, self.be.builddir,
-                self.dburl, self.be.builddir, own_bitbake)
+        cmd = "bash -c \"source %s/oe-init-build-env %s 2>&1 >toaster_server.log && bitbake --read %s/conf/toaster-pre.conf --postread %s/conf/toaster.conf --server-only -t xmlrpc -B 0.0.0.0:0 2>&1 >>toaster_server.log \"" % (self.pokydirname, self.be.builddir, self.be.builddir, self.be.builddir)
+
         port = "-1"
         logger.debug("localhostbecontroller: starting builder \n%s\n" % cmd)
+
         cmdoutput = self._shellcmd(cmd)
-        for i in cmdoutput.split("\n"):
-            if i.startswith("Bitbake server address"):
-                port = i.split(" ")[-1]
-                logger.debug("localhostbecontroller: Found bitbake server port %s" % port)
+        with open(self.be.builddir + "/toaster_server.log", "r") as f:
+            for i in f.readlines():
+                if i.startswith("Bitbake server address"):
+                    port = i.split(" ")[-1]
+                    logger.debug("localhostbecontroller: Found bitbake server port %s" % port)
+
+        cmd = "bash -c \"source %s/oe-init-build-env-memres -1 %s && DATABASE_URL=%s %s --observe-only -u toasterui --remote-server=0.0.0.0:-1 -t xmlrpc\"" % (self.pokydirname, self.be.builddir, self.dburl, own_bitbake)
+        with open(toaster_ui_log_filepath, "a+") as f:
+            p = subprocess.Popen(cmd, cwd = self.be.builddir, shell=True, stdout=f, stderr=f)
 
         def _toaster_ui_started(filepath, filepos = 0):
             if not os.path.exists(filepath):
@@ -147,8 +148,9 @@ class LocalhostBEController(BuildEnvironmentController):
             retries += 1
 
         if not started:
+            toaster_ui_log = open(os.path.join(self.be.builddir, "toaster_ui.log"), "r").read()
             toaster_server_log = open(os.path.join(self.be.builddir, "toaster_server.log"), "r").read()
-            raise BuildSetupException("localhostbecontroller: Bitbake server did not start in 5 seconds, aborting (Error: '%s' '%s')" % (cmdoutput, toaster_server_log))
+            raise BuildSetupException("localhostbecontroller: Bitbake server did not start in 5 seconds, aborting (Error: '%s' '%s')" % (toaster_ui_log, toaster_server_log))
 
         logger.debug("localhostbecontroller: Started bitbake server")
 
@@ -310,3 +312,25 @@ class LocalhostBEController(BuildEnvironmentController):
         import shutil
         shutil.rmtree(os.path.join(self.be.sourcedir, "build"))
         assert not os.path.exists(self.be.builddir)
+
+
+    def triggerBuild(self, bitbake, layers, variables, targets):
+        # set up the buid environment with the needed layers
+        self.setLayers(bitbake, layers)
+        self.writeConfFile("conf/toaster-pre.conf", variables)
+        self.writeConfFile("conf/toaster.conf", raw = "INHERIT+=\"toaster buildhistory\"")
+
+        # get the bb server running with the build req id and build env id
+        bbctrl = self.getBBController()
+
+        # trigger the build command
+        task = reduce(lambda x, y: x if len(y)== 0 else y, map(lambda y: y.task, targets))
+        if len(task) == 0:
+            task = None
+
+        bbctrl.build(list(map(lambda x:x.target, targets)), task)
+
+        logger.debug("localhostbecontroller: Build launched, exiting. Follow build logs at %s/toaster_ui.log" % self.be.builddir)
+
+        # disconnect from the server
+        bbctrl.disconnect()

bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py

@@ -2,7 +2,7 @@ from django.core.management.base import NoArgsCommand, CommandError
 from django.db import transaction
 from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException
 from bldcontrol.models import BuildRequest, BuildEnvironment, BRError
-from orm.models import ToasterSetting
+from orm.models import ToasterSetting, Build
 import os
 
 def DN(path):
@@ -61,7 +61,7 @@ class Command(NoArgsCommand):
         return DN(self._find_first_path_for_file(DN(self.guesspath), "bblayers.conf", 4))
 
 
-    def handle(self, **options):
+    def _verify_artifact_storage_dir(self):
         # verify that we have a settings for downloading artifacts
         while ToasterSetting.objects.filter(name="ARTIFACTS_STORAGE_DIR").count() == 0:
             guessedpath = os.getcwd() + "/toaster_build_artifacts/"
@@ -78,7 +78,10 @@ class Command(NoArgsCommand):
                     else:
                         raise ose
                 ToasterSetting.objects.create(name="ARTIFACTS_STORAGE_DIR", value=artifacts_storage_dir)
+        return 0
 
+
+    def _verify_build_environment(self):
         self.guesspath = DN(DN(DN(DN(DN(DN(DN(__file__)))))))
         # refuse to start if we have no build environments
         while BuildEnvironment.objects.count() == 0:
@@ -163,6 +166,8 @@ class Command(NoArgsCommand):
                             conffilepath, error = subprocess.Popen('bash -c ". '+os.path.join(dirname, ".templateconf")+'; echo \"\$TEMPLATECONF\""', shell=True, stdout=subprocess.PIPE).communicate()
                             conffilepath = os.path.join(conffilepath.strip(), "toasterconf.json")
                             candidatefilepath = os.path.join(dirname, conffilepath)
+                            if "toaster_cloned" in candidatefilepath:
+                                continue
                             if os.path.exists(candidatefilepath):
                                 config_files.append(candidatefilepath)
 
@@ -197,12 +202,16 @@ class Command(NoArgsCommand):
 
             while (_verify_be()):
                 pass
+        return 0
 
+    def _verify_default_settings(self):
         # verify that default settings are there
         if ToasterSetting.objects.filter(name = 'DEFAULT_RELEASE').count() != 1:
             ToasterSetting.objects.filter(name = 'DEFAULT_RELEASE').delete()
             ToasterSetting.objects.get_or_create(name = 'DEFAULT_RELEASE', value = '')
+        return 0
 
+    def _verify_builds_in_progress(self):
         # we are just starting up. we must not have any builds in progress, or build environments taken
         for b in BuildRequest.objects.filter(state = BuildRequest.REQ_INPROGRESS):
             BRError.objects.create(req = b, errtype = "toaster", errmsg = "Toaster found this build IN PROGRESS while Toaster started up. This is an inconsistent state, and the build was marked as failed")
@@ -211,4 +220,19 @@ class Command(NoArgsCommand):
 
         BuildEnvironment.objects.update(lock = BuildEnvironment.LOCK_FREE)
 
+        # also mark "In Progress builds as failures"
+        from django.utils import timezone
+        Build.objects.filter(outcome = Build.IN_PROGRESS).update(outcome = Build.FAILED, completed_on = timezone.now())
+
         return 0
+
+
+
+    def handle(self, **options):
+        retval = 0
+        retval += self._verify_artifact_storage_dir()
+        retval += self._verify_build_environment()
+        retval += self._verify_default_settings()
+        retval += self._verify_builds_in_progress()
+
+        return retval

bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py

@@ -1,12 +1,12 @@
 from django.core.management.base import NoArgsCommand, CommandError
 from django.db import transaction
-from orm.models import Build, ToasterSetting
+from orm.models import Build, ToasterSetting, LogMessage, Target
 from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException, BuildSetupException
 from bldcontrol.models import BuildRequest, BuildEnvironment, BRError, BRVariable
 import os
 import logging
 
-logger = logging.getLogger("toaster")
+logger = logging.getLogger("ToasterScheduler")
 
 class Command(NoArgsCommand):
     args    = ""
@@ -35,7 +35,7 @@ class Command(NoArgsCommand):
                 # select the build environment and the request to build
                 br = self._selectBuildRequest()
             except IndexError as e:
-                # logger.debug("runbuilds: No build request")
+                #logger.debug("runbuilds: No build request")
                 return
             try:
                 bec = self._selectBuildEnvironment()
@@ -50,33 +50,16 @@ class Command(NoArgsCommand):
 
             # write the build identification variable
             BRVariable.objects.create(req = br, name="TOASTER_BRBE", value="%d:%d" % (br.pk, bec.be.pk))
+
             # let the build request know where it is being executed
             br.environment = bec.be
             br.save()
 
-            # set up the buid environment with the needed layers
-            bec.setLayers(br.brbitbake_set.all(), br.brlayer_set.all())
-            bec.writeConfFile("conf/toaster-pre.conf", br.brvariable_set.all())
-            bec.writeConfFile("conf/toaster.conf", raw = "INHERIT+=\"toaster buildhistory\"")
-
-            # get the bb server running with the build req id and build env id
-            bbctrl = bec.getBBController()
-
-            # trigger the build command
-            task = reduce(lambda x, y: x if len(y)== 0 else y, map(lambda y: y.task, br.brtarget_set.all()))
-            if len(task) == 0:
-                task = None
-            bbctrl.build(list(map(lambda x:x.target, br.brtarget_set.all())), task)
-
-            logger.debug("runbuilds: Build launched, exiting. Follow build logs at %s/toaster_ui.log" % bec.be.builddir)
-            # disconnect from the server
-            bbctrl.disconnect()
-
-            # cleanup to be performed by toaster when the deed is done
-
+            # this triggers an async build
+            bec.triggerBuild(br.brbitbake_set.all(), br.brlayer_set.all(), br.brvariable_set.all(), br.brtarget_set.all())
 
         except Exception as e:
-            logger.error("runbuilds: Error executing shell command %s" % e)
+            logger.error("runbuilds: Error launching build %s" % e)
             traceback.print_exc(e)
             if "[Errno 111] Connection refused" in str(e):
                 # Connection refused, read toaster_server.out
@@ -125,7 +108,43 @@ class Command(NoArgsCommand):
         from django.utils import timezone
         from datetime import timedelta
         # environments locked for more than 30 seconds - they should be unlocked
-        BuildEnvironment.objects.filter(lock=BuildEnvironment.LOCK_LOCK).filter(updated__lt = timezone.now() - timedelta(seconds = 30)).update(lock = BuildEnvironment.LOCK_FREE)
+        BuildEnvironment.objects.filter(buildrequest__state__in=[BuildRequest.REQ_FAILED, BuildRequest.REQ_COMPLETED]).filter(lock=BuildEnvironment.LOCK_LOCK).filter(updated__lt = timezone.now() - timedelta(seconds = 30)).update(lock = BuildEnvironment.LOCK_FREE)
+
+
+        # update all Builds that failed to start
+
+        for br in BuildRequest.objects.filter(state = BuildRequest.REQ_FAILED, build__outcome = Build.IN_PROGRESS):
+            # transpose the launch errors in ToasterExceptions
+            br.build.outcome = Build.FAILED
+            for brerror in br.brerror_set.all():
+                logger.debug("Saving error %s" % brerror)
+                LogMessage.objects.create(build = br.build, level = LogMessage.EXCEPTION, message = brerror.errmsg)
+            br.build.save()
+
+            # we don't have a true build object here; hence, toasterui didn't have a change to release the BE lock
+            br.environment.lock = BuildEnvironment.LOCK_FREE
+            br.environment.save()
+
+
+
+        # update all BuildRequests without a build created
+        for br in BuildRequest.objects.filter(build = None):
+            br.build = Build.objects.create(project = br.project, completed_on = br.updated, started_on = br.created)
+            br.build.outcome = BuildRequest.REQ_FAILED
+            try:
+                br.build.machine = br.brvariable_set.get(name='MACHINE').value
+            except BRVariable.DoesNotExist:
+                pass
+            br.save()
+            # transpose target information
+            for brtarget in br.brtarget_set.all():
+                Target.objects.create(build = br.build, target= brtarget.target)
+            # transpose the launch errors in ToasterExceptions
+            for brerror in br.brerror_set.all():
+                LogMessage.objects.create(build = br.build, level = LogMessage.EXCEPTION, message = brerror.errmsg)
+
+            br.build.save()
+        pass
 
 
     def handle_noargs(self, **options):

bitbake/lib/toaster/bldcontrol/models.py

@@ -70,19 +70,19 @@ class BuildEnvironment(models.Model):
                 return "binary/octet-stream"
             except ImportError:
                 return "binary/octet-stream"
-        raise Exception("FIXME: artifact type not implemented for build environment type %s" % be.get_betype_display())
+        raise Exception("FIXME: artifact type not implemented for build environment type %s" % self.get_betype_display())
 
 
     def get_artifact(self, path):
         if self.betype == BuildEnvironment.TYPE_LOCAL:
             return open(path, "r")
-        raise Exception("FIXME: artifact download not implemented for build environment type %s" % be.get_betype_display())
+        raise Exception("FIXME: artifact download not implemented for build environment type %s" % self.get_betype_display())
 
     def has_artifact(self, path):
         import os
-        if self.betype == BuildRequest.TYPE_LOCAL:
+        if self.betype == BuildEnvironment.TYPE_LOCAL:
             return os.path.exists(path)
-        raise Exception("FIXME: has artifact not implemented for build environment type %s" % be.get_betype_display())
+        raise Exception("FIXME: has artifact not implemented for build environment type %s" % self.get_betype_display())
 
 # a BuildRequest is a request that the scheduler will build using a BuildEnvironment
 # the build request queue is the table itself, ordered by state
@@ -106,7 +106,7 @@ class BuildRequest(models.Model):
         (REQ_ARCHIVE, "archive"),
     )
 
-    search_allowed_fields = ("brtarget__target", "build__project__name", "build__machine")
+    search_allowed_fields = ("brtarget__target", "build__project__name")
 
     project     = models.ForeignKey(Project)
     build       = models.OneToOneField(Build, null = True)     # TODO: toasterui should set this when Build is created
@@ -125,6 +125,9 @@ class BuildRequest(models.Model):
     def get_machine(self):
         return self.brvariable_set.get(name="MACHINE").value
 
+    def __str__(self):
+        return "%s %s" % (self.project, self.get_state_display())
+
 # These tables specify the settings for running an actual build.
 # They MUST be kept in sync with the tables in orm.models.Project*
 
@@ -156,3 +159,6 @@ class BRError(models.Model):
     errtype     = models.CharField(max_length=100)
     errmsg      = models.TextField()
     traceback   = models.TextField()
+
+    def __str__(self):
+        return "%s (%s)" % (self.errmsg, self.req)

bitbake/lib/toaster/bldcontrol/sshbecontroller.py

@@ -31,6 +31,9 @@ from toastermain import settings
 
 from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException
 
+class NotImplementedException(Exception):
+    pass
+
 def DN(path):
     return "/".join(path.split("/")[0:-1])
 
@@ -125,7 +128,7 @@ class SSHBEController(BuildEnvironmentController):
         # set layers in the layersource
 
 
-        raise Exception("Not implemented: SSH setLayers")
+        raise NotImplementedException("Not implemented: SSH setLayers")
         # 3. configure the build environment, so we have a conf/bblayers.conf
         assert self.pokydirname is not None
         self._setupBE()
@@ -153,3 +156,24 @@ class SSHBEController(BuildEnvironmentController):
         import shutil
         shutil.rmtree(os.path.join(self.be.sourcedir, "build"))
         assert not self._pathexists(self.be.builddir)
+
+    def triggerBuild(self, bitbake, layers, variables, targets):
+        # set up the buid environment with the needed layers
+        self.setLayers(bitbake, layers)
+        self.writeConfFile("conf/toaster-pre.conf", )
+        self.writeConfFile("conf/toaster.conf", raw = "INHERIT+=\"toaster buildhistory\"")
+
+        # get the bb server running with the build req id and build env id
+        bbctrl = self.getBBController()
+
+        # trigger the build command
+        task = reduce(lambda x, y: x if len(y)== 0 else y, map(lambda y: y.task, targets))
+        if len(task) == 0:
+            task = None
+
+        bbctrl.build(list(map(lambda x:x.target, targets)), task)
+
+        logger.debug("localhostbecontroller: Build launched, exiting. Follow build logs at %s/toaster_ui.log" % self.be.builddir)
+
+        # disconnect from the server
+        bbctrl.disconnect()

bitbake/lib/toaster/bldcontrol/tests.py

@@ -7,7 +7,7 @@ Replace this with more appropriate tests for your application.
 
 from django.test import TestCase
 
-from bldcontrol.bbcontroller import BitbakeController
+from bldcontrol.bbcontroller import BitbakeController, BuildSetupException
 from bldcontrol.localhostbecontroller import LocalhostBEController
 from bldcontrol.sshbecontroller import SSHBEController
 from bldcontrol.models import BuildEnvironment, BuildRequest
@@ -15,6 +15,7 @@ from bldcontrol.management.commands.runbuilds import Command
 
 import socket
 import subprocess
+import os
 
 # standard poky data hardcoded for testing
 BITBAKE_LAYERS = [type('bitbake_info', (object,), { "giturl": "git://git.yoctoproject.org/poky.git", "dirpath": "", "commit": "HEAD"})]
@@ -29,6 +30,17 @@ POKY_LAYERS = [
 # we have an abstract test class designed to ensure that the controllers use a single interface
 # specific controller tests only need to override the _getBuildEnvironment() method
 
+test_sourcedir = os.getenv("TTS_SOURCE_DIR")
+test_builddir = os.getenv("TTS_BUILD_DIR")
+test_address = os.getenv("TTS_TEST_ADDRESS", "localhost")
+
+if test_sourcedir == None or test_builddir == None or test_address == None:
+    raise Exception("Please set TTTS_SOURCE_DIR, TTS_BUILD_DIR and TTS_TEST_ADDRESS")
+
+# The bb server will expect a toaster-pre.conf file to exist. If it doesn't exit then we make
+# an empty one here.
+open(test_builddir + 'conf/toaster-pre.conf', 'a').close()
+
 class BEControllerTests(object):
 
     def _serverForceStop(self, bc):
@@ -36,28 +48,53 @@ class BEControllerTests(object):
         self.assertTrue(err == '', "bitbake server pid %s not stopped" % err)
 
     def test_serverStartAndStop(self):
+        from bldcontrol.sshbecontroller import NotImplementedException
         obe =  self._getBuildEnvironment()
         bc = self._getBEController(obe)
-        bc.setLayers(BITBAKE_LAYERS, POKY_LAYERS) # setting layers, skip any layer info
+        try:
+            # setting layers, skip any layer info
+            bc.setLayers(BITBAKE_LAYERS, POKY_LAYERS)
+        except NotImplementedException,  e:
+            print "Test skipped due to command not implemented yet"
+            return True
+        # We are ok with the exception as we're handling the git already exists
+        except BuildSetupException:
+            pass
 
-        hostname = self.test_address.split("@")[-1]
+        bc.pokydirname = test_sourcedir
+        bc.islayerset = True
+
+        hostname = test_address.split("@")[-1]
 
         # test start server and stop
-        self.assertTrue(socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex((hostname, 8200)), "Port already occupied")
-        bc.startBBServer("0:0")
-        self.assertFalse(socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex((hostname, 8200)), "Server not answering")
+        bc.startBBServer()
+
+        self.assertFalse(socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex((hostname, int(bc.be.bbport))), "Server not answering")
 
         bc.stopBBServer()
-        self.assertTrue(socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex((hostname, 8200)), "Server not stopped")
+        self.assertTrue(socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex((hostname, int(bc.be.bbport))), "Server not stopped")
 
         self._serverForceStop(bc)
 
     def test_getBBController(self):
+        from bldcontrol.sshbecontroller import NotImplementedException
         obe = self._getBuildEnvironment()
         bc = self._getBEController(obe)
-        bc.setLayers(BITBAKE_LAYERS, POKY_LAYERS) # setting layers, skip any layer info
+        layerSet = False
+        try:
+            # setting layers, skip any layer info
+            layerSet = bc.setLayers(BITBAKE_LAYERS, POKY_LAYERS)
+        except NotImplementedException:
+            print "Test skipped due to command not implemented yet"
+            return True
+        # We are ok with the exception as we're handling the git already exists
+        except BuildSetupException:
+            pass
 
-        bbc = bc.getBBController("%d:%d" % (-1, obe.pk))
+        bc.pokydirname = test_sourcedir
+        bc.islayerset = True
+
+        bbc = bc.getBBController()
         self.assertTrue(isinstance(bbc, BitbakeController))
         bc.stopBBServer()
 
@@ -66,19 +103,15 @@ class BEControllerTests(object):
 class LocalhostBEControllerTests(TestCase, BEControllerTests):
     def __init__(self, *args):
         super(LocalhostBEControllerTests, self).__init__(*args)
-        # hardcoded for Alex's machine; since the localhost BE is machine-dependent,
-        # I found no good way to abstractize this
-        self.test_sourcedir = "/home/ddalex/ssd/yocto"
-        self.test_builddir = "/home/ddalex/ssd/yocto/build"
-        self.test_address = "localhost"
+
 
     def _getBuildEnvironment(self):
         return BuildEnvironment.objects.create(
                 lock = BuildEnvironment.LOCK_FREE,
                 betype = BuildEnvironment.TYPE_LOCAL,
-                address = self.test_address,
-                sourcedir = self.test_sourcedir,
-                builddir = self.test_builddir )
+                address = test_address,
+                sourcedir = test_sourcedir,
+                builddir = test_builddir )
 
     def _getBEController(self, obe):
         return LocalhostBEController(obe)
@@ -86,25 +119,20 @@ class LocalhostBEControllerTests(TestCase, BEControllerTests):
 class SSHBEControllerTests(TestCase, BEControllerTests):
     def __init__(self, *args):
         super(SSHBEControllerTests, self).__init__(*args)
-        self.test_address = "ddalex-desktop.local"
-        # hardcoded for ddalex-desktop.local machine; since the localhost BE is machine-dependent,
-        # I found no good way to abstractize this
-        self.test_sourcedir = "/home/ddalex/ssd/yocto"
-        self.test_builddir = "/home/ddalex/ssd/yocto/build"
 
     def _getBuildEnvironment(self):
         return BuildEnvironment.objects.create(
                 lock = BuildEnvironment.LOCK_FREE,
                 betype = BuildEnvironment.TYPE_SSH,
-                address = self.test_address,
-                sourcedir = self.test_sourcedir,
-                builddir = self.test_builddir )
+                address = test_address,
+                sourcedir = test_sourcedir,
+                builddir = test_builddir )
 
     def _getBEController(self, obe):
         return SSHBEController(obe)
 
     def test_pathExists(self):
-        obe = BuildEnvironment.objects.create(betype = BuildEnvironment.TYPE_SSH, address= self.test_address)
+        obe = BuildEnvironment.objects.create(betype = BuildEnvironment.TYPE_SSH, address= test_address)
         sbc = SSHBEController(obe)
         self.assertTrue(sbc._pathexists("/"))
         self.assertFalse(sbc._pathexists("/.deadbeef"))
@@ -129,7 +157,7 @@ class RunBuildsCommandTests(TestCase):
         self.assertRaises(IndexError, command._selectBuildEnvironment)
 
     def test_br_select(self):
-        from orm.models import Project, Release, BitbakeVersion
+        from orm.models import Project, Release, BitbakeVersion, Branch
         p = Project.objects.create_project("test", Release.objects.get_or_create(name = "HEAD", bitbake_version = BitbakeVersion.objects.get_or_create(name="HEAD", branch=Branch.objects.get_or_create(name="HEAD"))[0])[0])
         obr = BuildRequest.objects.create(state = BuildRequest.REQ_QUEUED, project = p)
         command = Command()

bitbake/lib/toaster/bldviewer/api.py

@@ -1,44 +0,0 @@
-#
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-#
-# BitBake Toaster Implementation
-#
-# Copyright (C) 2013        Intel Corporation
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-from django.conf.urls import patterns, include, url
-
-
-urlpatterns = patterns('bldviewer.views',
-        url(r'^builds$', 'model_explorer',  {'model_name':'build'}, name='builds'),
-        url(r'^targets$', 'model_explorer',  {'model_name':'target'}, name='targets'),
-        url(r'^target_files$', 'model_explorer',  {'model_name':'target_file'}, name='target_file'),
-        url(r'^target_image_file$', 'model_explorer',  {'model_name':'target_image_file'}, name='target_image_file'),
-        url(r'^tasks$', 'model_explorer', {'model_name':'task'}, name='task'),
-        url(r'^task_dependencies$', 'model_explorer',  {'model_name':'task_dependency'}, name='task_dependencies'),
-        url(r'^packages$', 'model_explorer',  {'model_name':'package'}, name='package'),
-        url(r'^package_dependencies$', 'model_explorer',  {'model_name':'package_dependency'}, name='package_dependency'),
-        url(r'^target_packages$', 'model_explorer',  {'model_name':'target_installed_package'}, name='target_packages'),
-        url(r'^target_installed_packages$', 'model_explorer',  {'model_name':'target_installed_package'}, name='target_installed_package'),
-        url(r'^package_files$', 'model_explorer',  {'model_name':'build_file'}, name='build_file'),
-        url(r'^layers$', 'model_explorer', {'model_name':'layer'}, name='layer'),
-        url(r'^layerversions$', 'model_explorer', {'model_name':'layerversion'}, name='layerversion'),
-        url(r'^recipes$', 'model_explorer', {'model_name':'recipe'}, name='recipe'),
-        url(r'^recipe_dependencies$', 'model_explorer',  {'model_name':'recipe_dependency'}, name='recipe_dependencies'),
-        url(r'^variables$', 'model_explorer',  {'model_name':'variable'}, name='variables'),
-        url(r'^variableshistory$', 'model_explorer',  {'model_name':'variablehistory'}, name='variablehistory'),
-        url(r'^logmessages$', 'model_explorer',  {'model_name':'logmessage'}, name='logmessages'),
-)

bitbake/lib/toaster/bldviewer/templates/simple_base.html

@@ -1,30 +0,0 @@
-<!DOCTYPE html>
-{% load static %}
-<html>
-	<head>
-		<title>Toaster Simple Explorer</title>
-<script src="{% static 'js/jquery-2.0.3.js' %}">
-</script>
-<script src="{% static 'js/bootstrap.js' %}">
-</script>
-<link href="{% static 'css/bootstrap.css' %}" rel="stylesheet" type="text/css">
-	</head>
-
-<body style="height: 100%">
-<div style="width:100%; height: 100%; position:absolute">
-<div style="width: 100%; height: 3em" class="nav">
-    <ul class="nav nav-tabs">
-        <li><a href="{% url "simple-all-builds" %}">All Builds</a></li>
-        <li><a href="{% url "simple-all-layers" %}">All Layers</a></li>
-    </ul>
-</div>
-
-<div style="overflow-y:scroll; width: 100%; position: absolute; top: 3em; bottom:70px ">
-{% block pagecontent %}
-{% endblock %}
-</div>
-<div class="navbar" style="position: absolute; bottom: 0; width:100%"><br/>About Toaster | Yocto Project </div>
-</div>
-</body>
-</html>
-

bitbake/lib/toaster/bldviewer/templates/simple_basebuildpage.html

@@ -1,17 +0,0 @@
-{% extends "simple_basetable.html" %}
-
-{% block pagename %}
-<ul class="nav nav-tabs" style="display: inline-block">
-  <li><a>Build {{build.target_set.all|join:"&nbsp;"}} at {{build.started_on}} : </a></li>
-  <li><a href="{% url "simple-task" build.id %}"> Tasks </a></li>
-  <li><a href="{% url "simple-bpackage" build.id %}"> Build Packages </a></li>
-    {% for t in build.target_set.all %}
-        {% if t.is_image %}
-  <li><a href="{% url "simple-tpackage" build.id t.pk %}"> Packages for {{t.target}} </a> </li>
-        {% endif %}
-    {% endfor %}
-  <li><a href="{% url "simple-configuration" build.id %}"> Configuration </a> </li>
-</ul>
-     <h1>Toaster - Build {% block pagetitle %} {% endblock %}</h1>
-{% endblock %}
-

bitbake/lib/toaster/bldviewer/templates/simple_basetable.html

@@ -1,64 +0,0 @@
-{% extends "simple_base.html" %}
-
-{% block pagecontent %}
-<script>
-function showhideTableColumn(i, sh) {
-    if (sh)
-        $('td:nth-child('+i+'),th:nth-child('+i+')').show();
-    else
-        $('td:nth-child('+i+'),th:nth-child('+i+')').hide();
-}
-
-
-function filterTableRows(test) {
-    if (test.length > 0) {
-        var r = test.split(/[ ,]+/).map(function (e) { return new RegExp(e, 'i') });
-        $('tr.data').map( function (i, el) {
-            (! r.map(function (j) { return j.test($(el).html())}).reduce(function (c, p) { return c && p;} )) ? $(el).hide() : $(el).show();
-        });
-    } else
-    {
-        $('tr.data').show();
-    }
-}
-</script>
-<div style="margin-bottom: 0.5em">
-
-    {% block pagename %}
-    {% endblock %}
-    <div align="left" style="display:inline-block; width: 40%; margin-left: 2em"> Filter: <input type="search" id="filterstring" style="width: 80%" onkeyup="filterTableRows($('#filterstring').val())" autocomplete="off">
-    </div>
-    {% if hideshowcols %}
-    <div align="right" style="display: inline-block; width: 40%">Show/Hide columns:
-    {% for i in hideshowcols %}
-        <span>{{i.name}} <input type="checkbox" id="ct{{i.name}}" onchange="showhideTableColumn({{i.order}}, $('#ct{{i.name}}').is(':checked'))" checked autocomplete="off"></span> |
-    {% endfor %}
-    </div>
-    {% endif %}
-</div>
-
-   <div style="display: block; float:right; margin-left: auto; margin-right:5em"><span class="pagination" style="vertical-align: top; margin-right: 3em">Showing {{objects.start_index}} to {{objects.end_index}} out of {{objects.paginator.count}} entries.&nbsp;</span>
-   <ul class="pagination" style="display: block-inline">
-{%if objects.has_previous %}
-  <li><a href="?page={{objects.previous_page_number}}">&laquo;</a></li>
-{%else%}
-  <li class="disabled"><a href="#">&laquo;</a></li>
-{%endif%}
-{% for i in objects.page_range %}
-  <li{%if i == objects.number %} class="active" {%endif%}><a href="?page={{i}}">{{i}}</a></li>
-{% endfor %}
-{%if objects.has_next%}
-  <li><a href="?page={{objects.next_page_number}}">&raquo;</a></li>
-{%else%}
-  <li class="disabled"><a href="#">&raquo;</a></li>
-{%endif%}
-  </ul>
-</div>
-
-    <table class="table table-striped table-condensed" style="width:95%">
-{% block pagetable %}
-{% endblock %}
-    </table>
-</div>
-
-{% endblock %}

bitbake/lib/toaster/bldviewer/templates/simple_bfile.html

@@ -1,24 +0,0 @@
-{% extends "simple_basebuildpage.html" %}
-
-{% block pagetitle %}Files for package {{objects.0.bpackage.name}} {% endblock %}
-{% block pagetable %}
-    {% if not objects %}
-        <p>No files were recorded for this package!</p>
-    {% else %}
-
-            <tr>
-            <th>Name</th>
-            <th>Size (Bytes)</th>
-            </tr>
-
-            {% for file in objects %}
-
-            <tr class="data">
-            <td>{{file.path}}</td>
-            <td>{{file.size}}</td>
-
-            {% endfor %}
-
-    {% endif %}
-
-{% endblock %}

bitbake/lib/toaster/bldviewer/templates/simple_bpackage.html

@@ -1,44 +0,0 @@
-{% extends "simple_basebuildpage.html" %}
-
-{% block pagetitle %}Packages{% endblock %}
-{% block pagetable %}
-    {% if not objects %}
-        <p>No packages were recorded for this target!</p>
-    {% else %}
-
-            <tr>
-            <th>Name</th>
-            <th>Version</th>
-            <th>Recipe</th>
-            <th>Summary</th>
-            <th>Section</th>
-            <th>Description</th>
-            <th>Size on host disk (Bytes)</th>
-            <th>License</th>
-            <th>Dependencies List (all)</th>
-            </tr>
-
-            {% for package in objects %}
-
-            <tr class="data">
-                <td><a name="#{{package.name}}" href="{% url "simple-bfile" build.pk package.pk %}">{{package.name}} ({{package.filelist_bpackage.count}} files)</a></td>
-                <td>{{package.version}}-{{package.revision}}</td>
-                <td>{%if package.recipe%}<a href="{% url "simple-layer_versions_recipes" package.recipe.layer_version_id %}#{{package.recipe.name}}">{{package.recipe.name}}</a>{{package.package_name}}</a>{%endif%}</td>
-
-            <td>{{package.summary}}</td>
-            <td>{{package.section}}</td>
-            <td>{{package.description}}</td>
-            <td>{{package.size}}</td>
-            <td>{{package.license}}</td>
-            <td>
-        <div style="height: 3em; overflow:auto">
-            {% for bpd in package.package_dependencies_source.all %}
-                {{bpd.dep_type}}: {{bpd.depends_on.name}} <br/>
-            {% endfor %}
-        </div>
-            </td>
-            {% endfor %}
-
-    {% endif %}
-
-{% endblock %}

bitbake/lib/toaster/bldviewer/templates/simple_build.html

@@ -1,43 +0,0 @@
-{% extends "simple_basetable.html" %}
-
-{% block pagename %}
-    <h1>Toaster - Builds</h1>
-{% endblock %}
-
-{% block pagetable %}
-
-    {% load simple_projecttags %}
-        <tr>
-            <th>Outcome</th>
-            <th>Started On</th>
-            <th>Completed On</th>
-            <th>Target</th>
-            <th>Machine</th>
-            <th>Time</th>
-            <th>Errors</th>
-            <th>Warnings</th>
-            <th>Output</th>
-            <th>Log</th>
-            <th>Bitbake Version</th>
-            <th>Build Name</th>
-        </tr>
-        {% for build in objects %}
-        <tr class="data">
-            <td><a href="{% url "simple-configuration" build.id %}">{{build.get_outcome_display}}</a></td>
-            <td>{{build.started_on}}</td>
-            <td>{{build.completed_on}}</td>
-            <td>{% for t in build.target_set.all %}{%if t.is_image %}<a href="{% url "simple-tpackage" build.id t.id %}">{% endif %}{{t.target}}{% if t.is_image %}</a>{% endif %}<br/>{% endfor %}</td>
-            <td>{{build.machine}}</td>
-            <td>{% time_difference build.started_on build.completed_on %}</td>
-            <td>{{build.errors_no}}:{% if  build.errors_no %}{% for error in logs %}{% if error.build == build %}{% if error.level == 2 %}<p>{{error.message}}</p>{% endif %}{% endif %}{% endfor %}{% else %}None{% endif %}</td>
-            <td>{{build.warnings_no}}:{% if  build.warnings_no %}{% for warning in logs %}{% if warning.build == build %}{% if warning.level == 1 %}<p>{{warning.message}}</p>{% endif %}{% endif %}{% endfor %}{% else %}None{% endif %}</td>
-            <td>TBD: determine image file list</td>
-            <td>{{build.cooker_log_path}}</td>
-            <td>{{build.bitbake_version}}</td>
-            <td>{{build.build_name}}</td>
-        </tr>
-
-        {% endfor %}
-{% endblock %}
-
-

bitbake/lib/toaster/bldviewer/templates/simple_configuration.html

@@ -1,22 +0,0 @@
-{% extends "simple_basebuildpage.html" %}
-
-{% block pagetitle %}Configuration{% endblock %}
-{% block pagetable %}
-
-            <tr>
-            <th>Name</th>
-            <th>Description</th>
-            <th>Definition history</th>
-            <th>Value</th>
-            </tr>
-
-            {% for variable in objects %}
-
-            <tr class="data">
-                <td>{{variable.variable_name}}</td>
-                <td>{% if variable.description %}{{variable.description}}{% endif %}</td>
-                <td>{% for vh in variable.variablehistory_set.all %}{{vh.operation}} in {{vh.file_name}}:{{vh.line_number}}<br/>{%endfor%}</td>
-                <td>{{variable.variable_value}}</td>
-            {% endfor %}
-
-{% endblock %}

bitbake/lib/toaster/bldviewer/templates/simple_layer.html

@@ -1,34 +0,0 @@
-{% extends "simple_basetable.html" %}
-
-{% block pagename %}
-    <h1>Toaster - Layers</h1>
-{% endblock %}
-
-{% block pagetable %}
-    {% load simple_projecttags %}
-
-        <tr>
-            <th>Name</th>
-            <th>Local Path</th>
-            <th>Layer Index URL</th>
-            <th>Known Versions</th>
-        </tr>
-
-        {% for layer in objects %}
-
-        <tr class="data">
-            <td>{{layer.name}}</td>
-            <td>{{layer.local_path}}</td>
-            <td><a href='{{layer.layer_index_url}}'>{{layer.layer_index_url}}</a></td>
-            <td><table>
-            {% for lv in layer.versions %}
-                <tr><td>
-        <a href="{% url "simple-layer_versions_recipes" lv.id %}">({{lv.priority}}){{lv.branch}}:{{lv.commit}} ({{lv.count}} recipes)</a>
-                </td></tr>
-            {% endfor %}
-            </table></td>
-        </tr>
-
-        {% endfor %}
-
-{% endblock %}

bitbake/lib/toaster/bldviewer/templates/simple_package.html

@@ -1,36 +0,0 @@
-{% extends "simple_basebuildpage.html" %}
-
-{% block pagetable %}
-    {% if not objects %}
-        <p>No packages were recorded for this target!</p>
-    {% else %}
-
-            <tr>
-            <th>Name</th>
-            <th>Version</th>
-            <th>Size (Bytes)</th>
-            <th>Recipe</th>
-            <th>Depends on</th>
-            </tr>
-
-            {% for package in objects %}
-
-            <tr class="data">
-                <td><a name="#{{package.name}}">{{package.name}}</a></td>
-                <td>{{package.version}}</td>
-                <td>{{package.size}}</td>
-                <td>{%if package.recipe %}<a name="{{package.recipe.name}}.{{package.package_name}}">
-                <a href="{% url "simple-layer_versions_recipes" package.recipe.layer_version_id %}#{{package.recipe.name}}">{{package.recipe.name}}</a>{{package.package_name}}</a>{%endif%}</td>
-                <td>
-            <div style="height: 4em; overflow:auto">
-                    {% for d in package.package_dependencies_source.all %}
-                    <a href="#{{d.name}}">{{d.depends_on.name}}</a><br/>
-                    {% endfor %}
-            </div>
-                </td>
-
-            {% endfor %}
-
-    {% endif %}
-
-{% endblock %}

bitbake/lib/toaster/bldviewer/templates/simple_recipe.html

@@ -1,50 +0,0 @@
-{% extends "simple_basetable.html" %}
-
-{% block pagename %}
-<ul class="nav nav-tabs" style="display: inline-block">
-  <li><a>Layer  {{layer_version.layer.name}}&nbsp;:&nbsp;{{layer_version.branch}}&nbsp;:&nbsp;{{layer_version.commit}}&nbsp;:&nbsp;{{layer_version.priority}}</a></li>
-</ul>
-    <h1>Toaster - Recipes for a Layer</h1>
-{% endblock %}
-
-{% block pagetable %}
-    {% load simple_projecttags %}
-
-        <tr>
-        </tr>
-            <th>Name</th>
-            <th>Version</th>
-            <th>Summary</th>
-            <th>Description</th>
-            <th>Section</th>
-            <th>License</th>
-            <th>Homepage</th>
-            <th>Bugtracker</th>
-            <th>File_path</th>
-            <th style="width: 30em">Recipe Dependency</th>
-
-
-        {% for recipe in objects %}
-
-        <tr class="data">
-            <td><a name="{{recipe.name}}">{{recipe.name}}</a></td>
-            <td>{{recipe.version}}</td>
-            <td>{{recipe.summary}}</td>
-            <td>{{recipe.description}}</td>
-            <td>{{recipe.section}}</td>
-            <td>{{recipe.license}}</td>
-            <td>{{recipe.homepage}}</td>
-            <td>{{recipe.bugtracker}}</td>
-            <td>{{recipe.file_path}}</td>
-            <td>
-        <div style="height: 5em; overflow:auto">
-            {% for rr in recipe.r_dependencies_recipe.all %}
-                <a href="#{{rr.depends_on.name}}">{{rr.depends_on.name}}</a><br/>
-            {% endfor %}
-        </div>
-            </td>
-        </tr>
-
-        {% endfor %}
-
-{% endblock %}

bitbake/lib/toaster/bldviewer/templates/simple_task.html

@@ -1,71 +0,0 @@
-{% extends "simple_basebuildpage.html" %}
-
-{% block pagetitle %}Tasks{% endblock %}
-{% block pagetable %}
-    {% if not objects %}
-        <p>No tasks were executed in this build!</p>
-    {% else %}
-
-            <tr>
-            <th>Order</th>
-            <th>Task</th>
-            <th>Recipe Version</th>
-            <th>Task Type</th>
-            <th>Checksum</th>
-            <th>Outcome</th>
-            <th>Message</th>
-            <th>Time</th>
-            <th>CPU usage</th>
-            <th>Disk I/O</th>
-            <th>Script type</th>
-            <th>Filesystem</th>
-            <th>Depends</th>
-            </tr>
-
-            {% for task in objects %}
-
-            <tr class="data">
-                <td>{{task.order}}</td>
-                <td><a name="{{task.recipe.name}}.{{task.task_name}}">
-                <a href="{% url "simple-layer_versions_recipes" task.recipe.layer_version_id %}#{{task.recipe.name}}">{{task.recipe.name}}</a>.{{task.task_name}}</a></td>
-                <td>{{task.recipe.version}}</td>
-
-                {% if task.task_executed %}
-                <td>Executed</td>
-                {% else %}
-                <td>Not Executed</td>
-                {% endif %}
-
-                <td>{{task.sstate_checksum}}</td>
-                <td>{{task.get_outcome_display}}{% if task.provider %}</br>(by <a href="#{{task.provider.recipe.name}}.{{task.provider.task_name}}">{{task.provider.recipe.name}}.{{task.provider.task_name}}</a>){% endif %}
-			{% if task.outcome == task.OUTCOME_CACHED %}{% for t in task.get_related_setscene %}
-			<br/>({{t.task_name}} {{t.get_outcome_display}})
-			{% endfor %}{%endif%}
-		</td>
-                <td><p>{{task.message}}</td>
-                <td>{{task.elapsed_time}}</td>
-                <td>{{task.cpu_usage}}</td>
-                <td>{{task.disk_io}}</td>
-                <td>{{task.get_script_type_display}}</td>
-                <td> <table>
-<tr><td>                Recipe</td><td><a target="_fileview" href="file:///{{task.recipe.file_path}}">{{task.recipe.file_path}}</a></td></tr>
-<tr><td>                Source</td><td><a target="_fileview" href="file:///{{task.file_name}}">{{task.file_name}}:{{task.line_number}}</a></td></tr>
-<tr><td>                Workdir</td><td><a target="_fileview" href="file:///{{task.work_directory}}">{{task.work_directory}}</a></td></tr>
-<tr><td>                Log</td><td><a target="_fileview" href="file:///{{task.logfile}}">{{task.logfile}}</a><br/></td></tr>
-</table>
-                </td>
-                <td>
-            <div style="height: 3em; overflow:auto">
-                {% for tt in task.task_dependencies_task.all %}
-                    <a href="#{{tt.depends_on.recipe.name}}.{{tt.depends_on.task_name}}">
-                    {{tt.depends_on.recipe.name}}.{{tt.depends_on.task_name}}</a><br/>
-                {% endfor %}
-            </div>
-                </td>
-                </tr>
-
-            {% endfor %}
-
-    {% endif %}
-
-{% endblock %}

bitbake/lib/toaster/bldviewer/templatetags/simple_projecttags.py

@@ -1,29 +0,0 @@
-#
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-#
-# BitBake Toaster Implementation
-#
-# Copyright (C) 2013        Intel Corporation
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-from datetime import datetime
-from django import template
-
-register = template.Library()
-
-@register.simple_tag
-def time_difference(start_time, end_time):
-    return end_time - start_time

bitbake/lib/toaster/bldviewer/tests.py

@@ -1,345 +0,0 @@
-"""
-This file demonstrates writing tests using the unittest module. These will pass
-when you run "manage.py test".
-
-Replace this with more appropriate tests for your application.
-"""
-from django.test import TestCase
-from django.test.client import Client
-from django.db.models import Count, Q
-from orm.models import Target, Recipe, Recipe_Dependency, Layer_Version, Target_Installed_Package
-from orm.models import Build, Task, Layer, Package, Package_File, LogMessage, Variable, VariableHistory
-import json, os, re, urllib, shlex
-
-
-class Tests(TestCase):
-    # fixtures = ['orm_views_testdata.json']
-
-    def setUp(self):
-        raise Exception("The %s test data is not longer valid, tests disabled" % __name__)
-
-    def test_builds(self):
-        client = Client()
-        resp = client.get('http://localhost:8000/api/1.0/builds')
-        self.assertEqual(resp.status_code, 200)
-        response = json.loads(resp.content)
-        self.assertTrue(len(json.loads(response['list'])) > 0)
-        for item in json.loads(response['list']):
-            fields = item['fields']
-            self.assertTrue(fields["machine"] == "qemux86")
-            self.assertTrue(fields["distro"] == "poky")
-            self.assertTrue(fields["image_fstypes"] == "tar.bz2 ext3")
-            self.assertTrue(fields["bitbake_version"] == "1.21.1")
-            self.assertTrue("1.5+snapshot-" in fields["distro_version"])
-            self.assertEqual(fields["outcome"], 0)
-            self.assertEqual(fields["errors_no"], 0)
-            log_path = "/tmp/log/cooker/qemux86/"
-            self.assertTrue(log_path in fields["cooker_log_path"])
-            self.assertTrue(".log" in fields["cooker_log_path"])
-
-    def test_targets(self):
-        client = Client()
-        resp = client.get('http://localhost:8000/api/1.0/targets')
-        self.assertEqual(resp.status_code, 200)
-        response = json.loads(resp.content)
-        self.assertTrue(len(json.loads(response['list'])) > 0)
-        for item in json.loads(response['list']):
-            fields = item['fields']
-            self.assertTrue(fields["is_image"] == True)
-            self.assertTrue(fields["target"] == "core-image-minimal")
-
-    def test_tasks(self):
-        client = Client()
-        resp = client.get('http://localhost:8000/api/1.0/tasks')
-        self.assertEqual(resp.status_code, 200)
-        response = json.loads(resp.content)
-        self.assertTrue(len(json.loads(response['list'])) > 0)
-        recipe_id = self.get_recipes_id("pseudo-native")
-        print recipe_id
-        for item in json.loads(response['list']):
-            fields = item['fields']
-            if fields["build"] == 1 and fields["task_name"] == "do_populate_lic_setscene" and fields["recipe"] == recipe_id and fields["task_executed"] == True:
-                self.assertTrue(fields["message"] == "recipe pseudo-native-1.5.1-r4: task do_populate_lic_setscene: Succeeded")
-                self.assertTrue(fields["cpu_usage"] == "6.3")
-                self.assertTrue(fields["disk_io"] == 124)
-                self.assertTrue(fields["script_type"] == 2)
-                self.assertTrue(fields["path_to_sstate_obj"] == "")
-                self.assertTrue(fields["elapsed_time"] == "0.103494")
-                self.assertTrue("tmp/work/i686-linux/pseudo-native/1.5.1-r4/temp/log.do_populate_lic_setscene.5867" in fields["logfile"])
-                self.assertTrue(fields["sstate_result"] == 0)
-                self.assertTrue(fields["outcome"] == 0)
-            if fields["build"] == 1 and fields["task_name"] == "do_populate_lic" and fields["recipe"] == recipe_id and fields["task_executed"] == True:
-                self.assertTrue(fields["cpu_usage"] == None)
-                self.assertTrue(fields["disk_io"] == None)
-                self.assertTrue(fields["script_type"] == 2)
-                self.assertTrue(fields["path_to_sstate_obj"] == "")
-                self.assertTrue(fields["elapsed_time"] == "0")
-                self.assertTrue(fields["logfile"], None)
-                self.assertTrue(fields["sstate_result"] == 3)
-                self.assertTrue(fields["outcome"] == 2)
-
-    def test_layers(self):
-        client = Client()
-        resp = client.get('http://localhost:8000/api/1.0/layers')
-        self.assertEqual(resp.status_code, 200)
-        response = json.loads(resp.content)
-        self.assertTrue(len(json.loads(response['list'])) > 0)
-        for item in json.loads(response['list']):
-            fields = item['fields']
-            if fields["name"] == "meta-yocto-bsp":
-                self.assertTrue(fields["local_path"].endswith("meta-yocto-bsp"))
-                self.assertTrue(fields["layer_index_url"] == "http://layers.openembedded.org/layerindex/layer/meta-yocto-bsp/")
-            elif fields["name"] == "meta":
-                self.assertTrue(fields["local_path"].endswith("/meta"))
-                self.assertTrue(fields["layer_index_url"] == "http://layers.openembedded.org/layerindex/layer/openembedded-core/")
-            elif fields["name"] == "meta-yocto":
-                self.assertTrue(fields["local_path"].endswith("/meta-yocto"))
-                self.assertTrue(fields["layer_index_url"] == "http://layers.openembedded.org/layerindex/layer/meta-yocto/")
-
-    def test_layerversions(self):
-        client = Client()
-        resp = client.get('http://localhost:8000/api/1.0/layerversions')
-        self.assertEqual(resp.status_code, 200)
-        response = json.loads(resp.content)
-        self.assertTrue(len(json.loads(response['list'])) > 0)
-        layer_id = self.get_layer_id("meta")
-        find = False
-        for item in json.loads(response['list']):
-            fields = item['fields']
-            if fields["layer"] == layer_id:
-                find = True
-                self.assertTrue(fields["build"] == 1)
-                self.assertTrue(fields["priority"] == 5)
-                self.assertTrue(fields["branch"] == "master")
-        self.assertTrue(find == True)
-
-    def test_recipes(self):
-        client = Client()
-        resp = client.get('http://localhost:8000/api/1.0/recipes')
-        self.assertEqual(resp.status_code, 200)
-        response = json.loads(resp.content)
-        self.assertTrue(len(json.loads(response['list'])) > 0)
-        find = False
-        for item in json.loads(response['list']):
-            fields = item['fields']
-            if fields["name"] == "busybox":
-                find = True
-                self.assertTrue(fields["version"] == "1.21.1-r0")
-                self.assertTrue(fields["license"] == "GPLv2 & bzip2")
-                self.assertTrue(fields["file_path"].endswith("/meta/recipes-core/busybox/busybox_1.21.1.bb"))
-                self.assertTrue(fields["summary"] == "Tiny versions of many common UNIX utilities in a single small executable.")
-                self.assertTrue(fields["description"] == "BusyBox combines tiny versions of many common UNIX utilities into a single small executable. It provides minimalist replacements for most of the utilities you usually find in GNU fileutils, shellutils, etc. The utilities in BusyBox generally have fewer options than their full-featured GNU cousins; however, the options that are included provide the expected functionality and behave very much like their GNU counterparts. BusyBox provides a fairly complete POSIX environment for any small or embedded system.")
-                self.assertTrue(fields["bugtracker"] == "https://bugs.busybox.net/")
-                self.assertTrue(fields["homepage"] == "http://www.busybox.net")
-                self.assertTrue(fields["section"] == "base")
-        self.assertTrue(find == True)
-
-    def test_task_dependencies(self):
-        client = Client()
-        resp = client.get('http://localhost:8000/api/1.0/task_dependencies')
-        self.assertEqual(resp.status_code, 200)
-        response = json.loads(resp.content)
-        self.assertTrue(len(json.loads(response['list'])) > 0)
-        ids = self.get_task_id()
-        do_install = ids["do_install"]
-        do_compile = ids["do_compile"]
-        entry = False
-        for item in json.loads(response['list']):
-            fields = item['fields']
-            if fields["task"] == do_install and fields["depends_on"] == do_compile:
-                entry = True
-        self.assertTrue(entry == True)
-
-    def test_target_installed_package(self):
-        client = Client()
-        resp = client.get('http://localhost:8000/api/1.0/target_installed_packages')
-        self.assertEqual(resp.status_code, 200)
-        response = json.loads(resp.content)
-        self.assertTrue(len(json.loads(response['list'])) > 0)
-        package = self.get_package_id("udev-utils")
-        find = False
-        for item in json.loads(response['list']):
-            fields = item['fields']
-            if fields["package"] == package:
-                self.assertTrue(fields["target"], 1)
-                find = True
-        self.assertTrue(find, True)
-
-    def test_packages(self):
-        client = Client()
-        resp = client.get('http://localhost:8000/api/1.0/packages')
-        self.assertEqual(resp.status_code, 200)
-        response = json.loads(resp.content)
-        self.assertTrue(response['count'] > 0)
-        for item in json.loads(response['list']):
-            fields = item['fields']
-            if fields["name"] == "base-files-dev":
-                self.assertTrue(fields["license"] == "GPLv2")
-                self.assertTrue(fields["description"] == "The base-files package creates the basic system directory structure and provides a small set of key configuration files for the system.  This package contains symbolic links, header files, and related items necessary for software development.")
-                self.assertTrue(fields["summary"] == "Miscellaneous files for the base system. - Development files")
-                self.assertTrue(fields["version"] == "3.0.14")
-                self.assertTrue(fields["build"] == 1)
-                self.assertTrue(fields["section"] == "devel")
-                self.assertTrue(fields["revision"] == "r73")
-                self.assertTrue(fields["size"] == 0)
-                self.assertTrue(fields["installed_size"] == 0)
-                self.assertTrue(self.get_recipe_name(fields["recipe"]) == "base-files")
-
-    def test_package_dependencies(self):
-        client = Client()
-        resp = client.get('http://localhost:8000/api/1.0/package_dependencies')
-        self.assertEqual(resp.status_code, 200)
-        response = json.loads(resp.content)
-        self.assertTrue(len(json.loads(response['list'])) > 0)
-        build_package = self.get_package_id("busybox")
-        build_package_id = self.get_package_id("busybox-syslog")
-        entry = False
-        for item in json.loads(response['list']):
-            fields = item['fields']
-            self.assertTrue(fields["target"] == 1)
-            if fields["package"] == build_package and fields["dep_type"] == 7 and fields["depends_on"] == build_package_id:
-                entry = True
-        self.assertTrue(entry == True)
-
-    def test_recipe_dependencies(self):
-        client = Client()
-        resp = client.get('http://localhost:8000/api/1.0/recipe_dependencies')
-        self.assertEqual(resp.status_code, 200)
-        response = json.loads(resp.content)
-        self.assertTrue(len(json.loads(response['list'])) > 0)
-        depends_on = self.get_recipes_id("autoconf-native")
-        recipe = self.get_recipes_id("ncurses")
-        entry = False
-        for item in json.loads(response['list']):
-            fields = item['fields']
-            if fields["recipe"] == recipe and fields["depends_on"] == depends_on and fields["dep_type"] == 0:
-                entry = True
-        self.assertTrue(entry == True)
-
-    def test_package_files(self):
-        client = Client()
-        resp = client.get('http://localhost:8000/api/1.0/package_files')
-        self.assertEqual(resp.status_code, 200)
-        response = json.loads(resp.content)
-        self.assertTrue(len(json.loads(response['list'])) > 0)
-        build_package = self.get_package_id("base-files")
-        entry = False
-        for item in json.loads(response['list']):
-            fields = item['fields']
-            if fields["path"] == "/etc/motd" and fields["package"] == build_package and fields["size"] == 0:
-                entry = True
-        self.assertTrue(entry == True)
-
-    def test_Variable(self):
-        client = Client()
-        resp = client.get('http://localhost:8000/api/1.0/variables')
-        self.assertEqual(resp.status_code, 200)
-        response = json.loads(resp.content)
-        self.assertTrue(len(json.loads(response['list'])) > 0)
-        for item in json.loads(response['list']):
-            fields = item['fields']
-            self.assertTrue(fields["build"] == 1)
-            if fields["variable_name"] == "USRBINPATH":
-                self.assertTrue(fields["variable_value"] == "/usr/bin")
-                self.assertTrue(fields["changed"] == False)
-                self.assertTrue(fields["description"] == "")
-            if fields["variable_name"] == "PREFERRED_PROVIDER_virtual/libx11":
-                self.assertTrue(fields["variable_value"] == "libx11")
-                self.assertTrue(fields["changed"] == False)
-                self.assertTrue(fields["description"] == "If multiple recipes provide an item, this variable determines which recipe should be given preference.")
-            if fields["variable_name"] == "base_libdir_nativesdk":
-                self.assertTrue(fields["variable_value"] == "/lib")
-
-    def test_VariableHistory(self):
-        client = Client()
-        resp = client.get('http://localhost:8000/api/1.0/variableshistory')
-        self.assertEqual(resp.status_code, 200)
-        response = json.loads(resp.content)
-        self.assertTrue(len(json.loads(response['list'])) > 0)
-        variable_id = self.get_variable_id("STAGING_INCDIR_NATIVE")
-        find = False
-        for item in json.loads(response['list']):
-            fields = item['fields']
-            if fields["variable"] == variable_id:
-                find = True
-                self.assertTrue(fields["file_name"] == "conf/bitbake.conf")
-                self.assertTrue(fields["operation"] == "set")
-                self.assertTrue(fields["line_number"] == 358)
-        self.assertTrue(find == True)
-
-    def get_task_id(self):
-        client = Client()
-        resp = client.get('http://localhost:8000/api/1.0/tasks')
-        self.assertEqual(resp.status_code, 200)
-        response = json.loads(resp.content)
-        self.assertTrue(len(json.loads(response['list'])) > 0)
-        for item in json.loads(response['list']):
-            fields = item['fields']
-            if fields["recipe"] == 7 and fields["task_name"] == "do_install":
-                do_install = item["pk"]
-            if fields["recipe"] == 7 and fields["task_name"] == "do_compile":
-                do_compile = item["pk"]
-        result = {}
-        result["do_install"] = do_install
-        result["do_compile"] = do_compile
-        return result
-
-    def get_recipes_id(self, value):
-        client = Client()
-        resp = client.get('http://localhost:8000/api/1.0/recipes')
-        self.assertEqual(resp.status_code, 200)
-        response = json.loads(resp.content)
-        self.assertTrue(len(json.loads(response['list'])) > 0)
-        for item in json.loads(response['list']):
-            fields = item['fields']
-            if fields["name"] == value:
-                return item["pk"]
-        return None
-
-    def get_recipe_name(self, value):
-        client = Client()
-        resp = client.get('http://localhost:8000/api/1.0/recipes')
-        self.assertEqual(resp.status_code, 200)
-        response = json.loads(resp.content)
-        self.assertTrue(len(json.loads(response['list'])) > 0)
-        for item in json.loads(response['list']):
-            fields = item['fields']
-            if item["pk"] == value:
-                return fields["name"]
-        return None
-
-    def get_layer_id(self, value):
-        client = Client()
-        resp = client.get('http://localhost:8000/api/1.0/layers')
-        self.assertEqual(resp.status_code, 200)
-        response = json.loads(resp.content)
-        self.assertTrue(len(json.loads(response['list'])) > 0)
-        for item in json.loads(response['list']):
-            fields = item['fields']
-            if fields["name"] == value:
-                return item["pk"]
-        return None
-
-    def get_package_id(self, field):
-        client = Client()
-        resp = client.get('http://localhost:8000/api/1.0/packages')
-        self.assertEqual(resp.status_code, 200)
-        response = json.loads(resp.content)
-        self.assertTrue(response['count'] > 0)
-        for item in json.loads(response['list']):
-            fields = item['fields']
-            if fields["name"] == field:
-                return item["pk"]
-        return None
-
-    def get_variable_id(self, field):
-        client = Client()
-        resp = client.get('http://localhost:8000/api/1.0/variables')
-        self.assertEqual(resp.status_code, 200)
-        response = json.loads(resp.content)
-        self.assertTrue(len(json.loads(response['list'])) > 0)
-        for item in json.loads(response['list']):
-            fields = item['fields']
-            if fields["variable_name"] == field:
-                return item["pk"]
-        return None

bitbake/lib/toaster/bldviewer/urls.py

@@ -1,35 +0,0 @@
-#
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-#
-# BitBake Toaster Implementation
-#
-# Copyright (C) 2013        Intel Corporation
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-from django.conf.urls import patterns, include, url
-from django.views.generic import RedirectView
-
-urlpatterns = patterns('bldviewer.views',
-        url(r'^builds/$', 'build', name='simple-all-builds'),
-        url(r'^build/(?P<build_id>\d+)/task/$', 'task', name='simple-task'),
-        url(r'^build/(?P<build_id>\d+)/packages/$', 'bpackage', name='simple-bpackage'),
-        url(r'^build/(?P<build_id>\d+)/package/(?P<package_id>\d+)/files/$', 'bfile', name='simple-bfile'),
-        url(r'^build/(?P<build_id>\d+)/target/(?P<target_id>\d+)/packages/$', 'tpackage', name='simple-tpackage'),
-        url(r'^build/(?P<build_id>\d+)/configuration/$', 'configuration', name='simple-configuration'),
-        url(r'^layers/$', 'layer', name='simple-all-layers'),
-        url(r'^layerversions/(?P<layerversion_id>\d+)/recipes/.*$', 'layer_versions_recipes', name='simple-layer_versions_recipes'),
-        url(r'^$', RedirectView.as_view( url= 'builds/')),
-)

bitbake/lib/toaster/bldviewer/views.py

@@ -1,287 +0,0 @@
-#
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-#
-# BitBake Toaster Implementation
-#
-# Copyright (C) 2013        Intel Corporation
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-import operator
-
-from django.db.models import Q
-from django.shortcuts import render
-from orm.models import Build, Target, Task, Layer, Layer_Version, Recipe, LogMessage, Variable, Target_Installed_Package
-from orm.models import Task_Dependency, Recipe_Dependency, Package, Package_File, Package_Dependency
-from orm.models import Target_Installed_Package, VariableHistory, Target_Image_File, Target_File
-from django.views.decorators.cache import cache_control
-from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
-
-
-def _build_page_range(paginator, index = 1):
-    try:
-        page = paginator.page(index)
-    except PageNotAnInteger:
-        page = paginator.page(1)
-    except  EmptyPage:
-        page = paginator.page(paginator.num_pages)
-
-    page.page_range = [page.number]
-    crt_range = 0
-    for i in range(1,5):
-        if (page.number + i) <= paginator.num_pages:
-            page.page_range = page.page_range + [ page.number + i]
-            crt_range +=1
-        if (page.number - i) > 0:
-            page.page_range =  [page.number -i] + page.page_range
-            crt_range +=1
-        if crt_range == 4:
-            break
-    return page
-
-@cache_control(no_store=True)
-def build(request):
-    template = 'simple_build.html'
-    logs = LogMessage.objects.all()
-
-    build_info = _build_page_range(Paginator(Build.objects.order_by("-id"), 10),request.GET.get('page', 1))
-
-    context = {'objects': build_info, 'logs': logs ,
-        'hideshowcols' : [
-                {'name': 'Output', 'order':10},
-                {'name': 'Log', 'order':11},
-            ]}
-
-    return render(request, template, context)
-
-
-def _find_task_revdep(task):
-    tp = []
-    for p in Task_Dependency.objects.filter(depends_on=task):
-        tp.append(p.task);
-    return tp
-
-def _find_task_provider(task):
-    task_revdeps = _find_task_revdep(task)
-    for tr in task_revdeps:
-        if tr.outcome != Task.OUTCOME_COVERED:
-            return tr
-    for tr in task_revdeps:
-        trc = _find_task_provider(tr)
-        if trc is not None:
-            return trc
-    return None
-
-def task(request, build_id):
-    template = 'simple_task.html'
-
-    tasks = _build_page_range(Paginator(Task.objects.filter(build=build_id, order__gt=0), 100),request.GET.get('page', 1))
-
-    for t in tasks:
-        if t.outcome == Task.OUTCOME_COVERED:
-            t.provider = _find_task_provider(t)
-
-    context = {'build': Build.objects.filter(pk=build_id)[0], 'objects': tasks}
-
-    return render(request, template, context)
-
-def configuration(request, build_id):
-    template = 'simple_configuration.html'
-    variables = _build_page_range(Paginator(Variable.objects.filter(build=build_id), 50), request.GET.get('page', 1))
-    context = {'build': Build.objects.filter(pk=build_id)[0], 'objects' : variables}
-    return render(request, template, context)
-
-def bpackage(request, build_id):
-    template = 'simple_bpackage.html'
-    packages = Package.objects.filter(build = build_id)
-    context = {'build': Build.objects.filter(pk=build_id)[0], 'objects' : packages}
-    return render(request, template, context)
-
-def bfile(request, build_id, package_id):
-    template = 'simple_bfile.html'
-    files = Package_File.objects.filter(package = package_id)
-    context = {'build': Build.objects.filter(pk=build_id)[0], 'objects' : files}
-    return render(request, template, context)
-
-def tpackage(request, build_id, target_id):
-    template = 'simple_package.html'
-    packages = map(lambda x: x.package, list(Target_Installed_Package.objects.filter(target=target_id)))
-    context = {'build': Build.objects.filter(pk=build_id)[0], 'objects' : packages}
-    return render(request, template, context)
-
-def layer(request):
-    template = 'simple_layer.html'
-    layer_info = Layer.objects.all()
-
-    for li in layer_info:
-        li.versions = Layer_Version.objects.filter(layer = li)
-        for liv in li.versions:
-            liv.count = Recipe.objects.filter(layer_version__id = liv.id).count()
-
-    context = {'objects': layer_info}
-
-    return render(request, template, context)
-
-
-def layer_versions_recipes(request, layerversion_id):
-    template = 'simple_recipe.html'
-    recipes = Recipe.objects.filter(layer_version__id = layerversion_id)
-
-    context = {'objects': recipes,
-            'layer_version' : Layer_Version.objects.filter( id = layerversion_id )[0]
-    }
-
-    return render(request, template, context)
-
-#### API
-
-import json
-from django.core import serializers
-from django.http import HttpResponse, HttpResponseBadRequest
-
-
-def model_explorer(request, model_name):
-
-    DESCENDING = 'desc'
-    response_data = {}
-    model_mapping = {
-        'build': Build,
-        'target': Target,
-        'target_file': Target_File,
-        'target_image_file': Target_Image_File,
-        'task': Task,
-        'task_dependency': Task_Dependency,
-        'package': Package,
-        'layer': Layer,
-        'layerversion': Layer_Version,
-        'recipe': Recipe,
-        'recipe_dependency': Recipe_Dependency,
-        'package': Package,
-        'package_dependency': Package_Dependency,
-        'target_installed_package': Target_Installed_Package,
-        'build_file': Package_File,
-        'variable': Variable,
-        'variablehistory': VariableHistory,
-        'logmessage': LogMessage,
-        }
-
-    if model_name not in model_mapping.keys():
-        return HttpResponseBadRequest()
-
-    model = model_mapping[model_name]
-
-    try:
-        limit = int(request.GET.get('limit', 0))
-    except ValueError:
-        limit = 0
-
-    try:
-        offset = int(request.GET.get('offset', 0))
-    except ValueError:
-        offset = 0
-
-    ordering_string, invalid = _validate_input(request.GET.get('orderby', ''),
-                                               model)
-    if invalid:
-        return HttpResponseBadRequest()
-
-    filter_string, invalid = _validate_input(request.GET.get('filter', ''),
-                                             model)
-    if invalid:
-        return HttpResponseBadRequest()
-
-    search_term = request.GET.get('search', '')
-
-    if filter_string:
-        filter_terms = _get_filtering_terms(filter_string)
-        try:
-            queryset = model.objects.filter(**filter_terms)
-        except ValueError:
-            queryset = []
-    else:
-        queryset = model.objects.all()
-
-    if search_term:
-        queryset = _get_search_results(search_term, queryset, model)
-
-    if ordering_string and queryset:
-        column, order = ordering_string.split(':')
-        if order.lower() == DESCENDING:
-            queryset = queryset.order_by('-' + column)
-        else:
-            queryset = queryset.order_by(column)
-
-    if offset and limit:
-        queryset = queryset[offset:(offset+limit)]
-    elif offset:
-        queryset = queryset[offset:]
-    elif limit:
-        queryset = queryset[:limit]
-
-    if queryset:
-        response_data['count'] = queryset.count()
-    else:
-        response_data['count'] = 0
-
-    response_data['list'] = serializers.serialize('json', queryset)
-
-    return HttpResponse(json.dumps(response_data),
-                        content_type='application/json')
-
-def _get_filtering_terms(filter_string):
-
-    search_terms = filter_string.split(":")
-    keys = search_terms[0].split(',')
-    values = search_terms[1].split(',')
-
-    return dict(zip(keys, values))
-
-def _validate_input(input, model):
-
-    invalid = 0
-
-    if input:
-        input_list = input.split(":")
-
-        # Check we have only one colon
-        if len(input_list) != 2:
-            invalid = 1
-            return None, invalid
-
-        # Check we have an equal number of terms both sides of the colon
-        if len(input_list[0].split(',')) != len(input_list[1].split(',')):
-            invalid = 1
-            return None, invalid
-
-        # Check we are looking for a valid field
-        valid_fields = model._meta.get_all_field_names()
-        for field in input_list[0].split(','):
-            if field not in valid_fields:
-                invalid = 1
-                return None, invalid
-
-    return input, invalid
-
-def _get_search_results(search_term, queryset, model):
-    search_objects = []
-    for st in search_term.split(" "):
-        q_map = map(lambda x: Q(**{x+'__icontains': st}),
-                model.search_allowed_fields)
-
-        search_objects.append(reduce(operator.or_, q_map))
-    search_object = reduce(operator.and_, search_objects)
-    queryset = queryset.filter(search_object)
-
-    return queryset

bitbake/lib/toaster/contrib/README

@@ -0,0 +1,6 @@
+contrib directory for toaster
+
+This directory holds code that works with Toaster, without being an integral part of the Toaster project.
+It is intended for testing code, testing fixtures, tools for Toaster, etc.
+
+NOTE: This directory is NOT a Python module.

bitbake/lib/toaster/contrib/django-aggregate-if-master/.gitignore

@@ -0,0 +1,10 @@
+*.pyc
+*.swp
+*.swo
+*.kpf
+*.egg-info/
+.idea
+.tox
+tmp/
+dist/
+.DS_Store

bitbake/lib/toaster/contrib/django-aggregate-if-master/.travis.yml

@@ -0,0 +1,50 @@
+language: python
+python:
+  - "2.7"
+  - "3.4"
+services:
+  - mysql
+  - postgresql
+env:
+  - DJANGO=1.4 DB=sqlite
+  - DJANGO=1.4 DB=mysql
+  - DJANGO=1.4 DB=postgres
+  - DJANGO=1.5 DB=sqlite
+  - DJANGO=1.5 DB=mysql
+  - DJANGO=1.5 DB=postgres
+  - DJANGO=1.6 DB=sqlite
+  - DJANGO=1.6 DB=mysql
+  - DJANGO=1.6 DB=postgres
+  - DJANGO=1.7 DB=sqlite
+  - DJANGO=1.7 DB=mysql
+  - DJANGO=1.7 DB=postgres
+
+matrix:
+  exclude:
+    - python: "3.4"
+      env: DJANGO=1.4 DB=sqlite
+    - python: "3.4"
+      env: DJANGO=1.4 DB=mysql
+    - python: "3.4"
+      env: DJANGO=1.4 DB=postgres
+    - python: "3.4"
+      env: DJANGO=1.5 DB=sqlite
+    - python: "3.4"
+      env: DJANGO=1.5 DB=mysql
+    - python: "3.4"
+      env: DJANGO=1.5 DB=postgres
+    - python: "3.4"
+      env: DJANGO=1.6 DB=mysql
+    - python: "3.4"
+      env: DJANGO=1.7 DB=mysql
+
+before_script:
+  - mysql -e 'create database aggregation;'
+  - psql -c 'create database aggregation;' -U postgres
+install:
+  - pip install six
+  - if [ "$DB" == "mysql" ]; then pip install mysql-python; fi
+  - if [ "$DB" == "postgres" ]; then pip install psycopg2; fi
+  - pip install -q Django==$DJANGO --use-mirrors
+script:
+  - ./runtests.py --settings=tests.test_$DB

bitbake/lib/toaster/contrib/django-aggregate-if-master/LICENSE

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2012 Henrique Bastos
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

bitbake/lib/toaster/contrib/django-aggregate-if-master/README.rst

@@ -0,0 +1,156 @@
+Django Aggregate If: Condition aggregates for Django
+====================================================
+
+.. image:: https://travis-ci.org/henriquebastos/django-aggregate-if.png?branch=master
+    :target: https://travis-ci.org/henriquebastos/django-aggregate-if
+    :alt: Test Status
+
+.. image:: https://landscape.io/github/henriquebastos/django-aggregate-if/master/landscape.png
+    :target: https://landscape.io/github/henriquebastos/django-aggregate-if/master
+    :alt: Code Helth
+
+.. image:: https://pypip.in/v/django-aggregate-if/badge.png
+    :target: https://crate.io/packages/django-aggregate-if/
+    :alt: Latest PyPI version
+
+.. image:: https://pypip.in/d/django-aggregate-if/badge.png
+    :target: https://crate.io/packages/django-aggregate-if/
+    :alt: Number of PyPI downloads
+
+*Aggregate-if* adds conditional aggregates to Django.
+
+Conditional aggregates can help you reduce the ammount of queries to obtain
+aggregated information, like statistics for example.
+
+Imagine you have a model ``Offer`` like this one:
+
+.. code-block:: python
+
+    class Offer(models.Model):
+        sponsor = models.ForeignKey(User)
+        price = models.DecimalField(max_digits=9, decimal_places=2)
+        status = models.CharField(max_length=30)
+        expire_at = models.DateField(null=True, blank=True)
+        created_at = models.DateTimeField(auto_now_add=True)
+        updated_at = models.DateTimeField(auto_now=True)
+
+        OPEN = "OPEN"
+        REVOKED = "REVOKED"
+        PAID = "PAID"
+
+Let's say you want to know:
+
+#. How many offers exists in total;
+#. How many of them are OPEN, REVOKED or PAID;
+#. How much money was offered in total;
+#. How much money is in OPEN, REVOKED and PAID offers;
+
+To get these informations, you could query:
+
+.. code-block:: python
+
+    from django.db.models import Count, Sum
+
+    Offer.objects.count()
+    Offer.objects.filter(status=Offer.OPEN).aggregate(Count('pk'))
+    Offer.objects.filter(status=Offer.REVOKED).aggregate(Count('pk'))
+    Offer.objects.filter(status=Offer.PAID).aggregate(Count('pk'))
+    Offer.objects.aggregate(Sum('price'))
+    Offer.objects.filter(status=Offer.OPEN).aggregate(Sum('price'))
+    Offer.objects.filter(status=Offer.REVOKED).aggregate(Sum('price'))
+    Offer.objects.filter(status=Offer.PAID).aggregate(Sum('price'))
+
+In this case, **8 queries** were needed to retrieve the desired information.
+
+With conditional aggregates you can get it all with only **1 query**:
+
+.. code-block:: python
+
+    from django.db.models import Q
+    from aggregate_if import Count, Sum
+
+    Offer.objects.aggregate(
+        pk__count=Count('pk'),
+        pk__open__count=Count('pk', only=Q(status=Offer.OPEN)),
+        pk__revoked__count=Count('pk', only=Q(status=Offer.REVOKED)),
+        pk__paid__count=Count('pk', only=Q(status=Offer.PAID)),
+        pk__sum=Sum('price'),
+        pk__open__sum=Sum('price', only=Q(status=Offer.OPEN)),
+        pk__revoked__sum=Sum('price'), only=Q(status=Offer.REVOKED)),
+        pk__paid__sum=Sum('price'), only=Q(status=Offer.PAID))
+    )
+
+Installation
+------------
+
+*Aggregate-if* works with Django 1.4, 1.5, 1.6 and 1.7.
+
+To install it, simply:
+
+.. code-block:: bash
+
+    $ pip install django-aggregate-if
+
+Inspiration
+-----------
+
+There is a 5 years old `ticket 11305`_ that will (*hopefully*) implement this feature into
+Django 1.8.
+
+Using Django 1.6, I still wanted to avoid creating custom queries for very simple
+conditional aggregations. So I've cherry picked those ideas and others from the
+internet and built this library.
+
+This library uses the same API and tests proposed on `ticket 11305`_, so when the
+new feature is available you can easily replace ``django-aggregate-if``.
+
+Limitations
+-----------
+
+Conditions involving joins with aliases are not supported yet. If you want to
+help adding this feature, you're welcome to check the `first issue`_.
+
+Contributors
+------------
+
+* `Henrique Bastos <http://github.com/henriquebastos>`_
+* `Iuri de Silvio <https://github.com/iurisilvio>`_
+* `Hampus Stjernhav <https://github.com/champ>`_
+* `Bradley Martsberger <https://github.com/martsberger>`_
+* `Markus Bertheau <https://github.com/mbertheau>`_
+* `end0 <https://github.com/end0>`_
+* `Scott Sexton <https://github.com/scottsexton>`_
+* `Mauler <https://github.com/mauler>`_
+* `trbs <https://github.com/trbs>`_
+
+Changelog
+---------
+
+0.5
+    - Support for Django 1.7
+
+0.4
+    - Use tox to run tests.
+    - Add support for Django 1.6.
+    - Add support for Python3.
+    - The ``only`` parameter now freely supports joins independent of the main query.
+    - Adds support for alias relabeling permitting excludes and updates with aggregates filtered on remote foreign key relations.
+
+0.3.1
+    - Fix quotation escaping.
+    - Fix boolean casts on Postgres.
+
+0.2
+    - Fix postgres issue with LIKE conditions.
+
+0.1
+    - Initial release.
+
+
+License
+=======
+
+The MIT License.
+
+.. _ticket 11305: https://code.djangoproject.com/ticket/11305
+.. _first issue: https://github.com/henriquebastos/django-aggregate-if/issues/1

bitbake/lib/toaster/contrib/django-aggregate-if-master/aggregate_if.py

@@ -0,0 +1,164 @@
+# coding: utf-8
+'''
+Implements conditional aggregates.
+
+This code was based on the work of others found on the internet:
+
+1. http://web.archive.org/web/20101115170804/http://www.voteruniverse.com/Members/jlantz/blog/conditional-aggregates-in-django
+2. https://code.djangoproject.com/ticket/11305
+3. https://groups.google.com/forum/?fromgroups=#!topic/django-users/cjzloTUwmS0
+4. https://groups.google.com/forum/?fromgroups=#!topic/django-users/vVprMpsAnPo
+'''
+from __future__ import unicode_literals
+from django.utils import six
+import django
+from django.db.models.aggregates import Aggregate as DjangoAggregate
+from django.db.models.sql.aggregates import Aggregate as DjangoSqlAggregate
+
+
+VERSION = django.VERSION[:2]
+
+
+class SqlAggregate(DjangoSqlAggregate):
+    conditional_template = '%(function)s(CASE WHEN %(condition)s THEN %(field)s ELSE null END)'
+
+    def __init__(self, col, source=None, is_summary=False, condition=None, **extra):
+        super(SqlAggregate, self).__init__(col, source, is_summary, **extra)
+        self.condition = condition
+
+    def relabel_aliases(self, change_map):
+        if VERSION < (1, 7):
+            super(SqlAggregate, self).relabel_aliases(change_map)
+        if self.has_condition:
+            condition_change_map = dict((k, v) for k, v in \
+                change_map.items() if k in self.condition.query.alias_map
+            )
+            self.condition.query.change_aliases(condition_change_map)
+
+    def relabeled_clone(self, change_map):
+        self.relabel_aliases(change_map)
+        return super(SqlAggregate, self).relabeled_clone(change_map)
+
+    def as_sql(self, qn, connection):
+        if self.has_condition:
+            self.sql_template = self.conditional_template
+            self.extra['condition'] = self._condition_as_sql(qn, connection)
+
+        return super(SqlAggregate, self).as_sql(qn, connection)
+
+    @property
+    def has_condition(self):
+        # Warning: bool(QuerySet) will hit the database
+        return self.condition is not None
+
+    def _condition_as_sql(self, qn, connection):
+        '''
+        Return sql for condition.
+        '''
+        def escape(value):
+            if isinstance(value, bool):
+                value = str(int(value))
+            if isinstance(value, six.string_types):
+                # Escape params used with LIKE
+                if '%' in value:
+                    value = value.replace('%', '%%')
+                # Escape single quotes
+                if "'" in value:
+                    value = value.replace("'", "''")
+                # Add single quote to text values
+                value = "'" + value + "'"
+            return value
+
+        sql, param = self.condition.query.where.as_sql(qn, connection)
+        param = map(escape, param)
+
+        return sql % tuple(param)
+
+
+class SqlSum(SqlAggregate):
+    sql_function = 'SUM'
+
+
+class SqlCount(SqlAggregate):
+    is_ordinal = True
+    sql_function = 'COUNT'
+    sql_template = '%(function)s(%(distinct)s%(field)s)'
+    conditional_template = '%(function)s(%(distinct)sCASE WHEN %(condition)s THEN %(field)s ELSE null END)'
+
+    def __init__(self, col, distinct=False, **extra):
+        super(SqlCount, self).__init__(col, distinct=distinct and 'DISTINCT ' or '', **extra)
+
+
+class SqlAvg(SqlAggregate):
+    is_computed = True
+    sql_function = 'AVG'
+
+
+class SqlMax(SqlAggregate):
+    sql_function = 'MAX'
+
+
+class SqlMin(SqlAggregate):
+    sql_function = 'MIN'
+
+
+class Aggregate(DjangoAggregate):
+    def __init__(self, lookup, only=None, **extra):
+        super(Aggregate, self).__init__(lookup, **extra)
+        self.only = only
+        self.condition = None
+
+    def _get_fields_from_Q(self, q):
+        fields = []
+        for child in q.children:
+            if hasattr(child, 'children'):
+                fields.extend(self._get_fields_from_Q(child))
+            else:
+                fields.append(child)
+        return fields
+
+    def add_to_query(self, query, alias, col, source, is_summary):
+        if self.only:
+            self.condition = query.model._default_manager.filter(self.only)
+            for child in self._get_fields_from_Q(self.only):
+                field_list = child[0].split('__')
+                # Pop off the last field if it's a query term ('gte', 'contains', 'isnull', etc.)
+                if field_list[-1] in query.query_terms:
+                    field_list.pop()
+                # setup_joins have different returns in Django 1.5 and 1.6, but the order of what we need remains.
+                result = query.setup_joins(field_list, query.model._meta, query.get_initial_alias(), None)
+                join_list = result[3]
+
+                fname = 'promote_alias_chain' if VERSION < (1, 5) else 'promote_joins'
+                args = (join_list, True) if VERSION < (1, 7) else (join_list,)
+
+                promote = getattr(query, fname)
+                promote(*args)
+
+        aggregate = self.sql_klass(col, source=source, is_summary=is_summary, condition=self.condition, **self.extra)
+        query.aggregates[alias] = aggregate
+
+
+class Sum(Aggregate):
+    name = 'Sum'
+    sql_klass = SqlSum
+
+
+class Count(Aggregate):
+    name = 'Count'
+    sql_klass = SqlCount
+
+
+class Avg(Aggregate):
+    name = 'Avg'
+    sql_klass = SqlAvg
+
+
+class Max(Aggregate):
+    name = 'Max'
+    sql_klass = SqlMax
+
+
+class Min(Aggregate):
+    name = 'Min'
+    sql_klass = SqlMin

bitbake/lib/toaster/contrib/django-aggregate-if-master/runtests.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+
+import os
+import sys
+from optparse import OptionParser
+
+
+def parse_args():
+    parser = OptionParser()
+    parser.add_option('-s', '--settings', help='Define settings.')
+    parser.add_option('-t', '--unittest', help='Define which test to run. Default all.')
+    options, args = parser.parse_args()
+
+    if not options.settings:
+        parser.print_help()
+        sys.exit(1)
+
+    if not options.unittest:
+        options.unittest = ['aggregation']
+
+    return options
+
+
+def get_runner(settings_module):
+    '''
+    Asks Django for the TestRunner defined in settings or the default one.
+    '''
+    os.environ['DJANGO_SETTINGS_MODULE'] = settings_module
+
+    import django
+    from django.test.utils import get_runner
+    from django.conf import settings
+
+    if hasattr(django, 'setup'):
+        django.setup()
+
+    return get_runner(settings)
+
+
+def runtests():
+    options = parse_args()
+    TestRunner = get_runner(options.settings)
+    runner = TestRunner(verbosity=1, interactive=True, failfast=False)
+    sys.exit(runner.run_tests([]))
+
+
+if __name__ == '__main__':
+    runtests()

bitbake/lib/toaster/contrib/django-aggregate-if-master/setup.py

@@ -0,0 +1,33 @@
+# coding: utf-8
+from setuptools import setup
+import os
+
+
+setup(name='django-aggregate-if',
+      version='0.5',
+      description='Conditional aggregates for Django, just like the famous SumIf in Excel.',
+      long_description=open(os.path.join(os.path.dirname(__file__), "README.rst")).read(),
+      author="Henrique Bastos", author_email="henrique@bastos.net",
+      license="MIT",
+      py_modules=['aggregate_if'],
+      install_requires=[
+          'six>=1.6.1',
+      ],
+      zip_safe=False,
+      platforms='any',
+      include_package_data=True,
+      classifiers=[
+          'Development Status :: 5 - Production/Stable',
+          'Framework :: Django',
+          'Intended Audience :: Developers',
+          'License :: OSI Approved :: MIT License',
+          'Natural Language :: English',
+          'Operating System :: OS Independent',
+          'Programming Language :: Python',
+          'Programming Language :: Python :: 2.7',
+          'Programming Language :: Python :: 3',
+          'Topic :: Database',
+          'Topic :: Software Development :: Libraries',
+      ],
+      url='http://github.com/henriquebastos/django-aggregate-if/',
+)

bitbake/lib/toaster/contrib/django-aggregate-if-master/tox.ini

@@ -0,0 +1,198 @@
+[tox]
+envlist =
+    py27-django1.4-sqlite,
+    py27-django1.4-postgres,
+    py27-django1.4-mysql,
+
+    py27-django1.5-sqlite,
+    py27-django1.5-postgres,
+    py27-django1.5-mysql,
+
+    py27-django1.6-sqlite,
+    py27-django1.6-postgres,
+    py27-django1.6-mysql,
+
+    py27-django1.7-sqlite,
+    py27-django1.7-postgres,
+    py27-django1.7-mysql,
+
+    py34-django1.6-sqlite,
+    py34-django1.6-postgres,
+    #py34-django1.6-mysql
+
+    py34-django1.7-sqlite,
+    py34-django1.7-postgres,
+    #py34-django1.7-mysql
+
+[testenv]
+whitelist_externals=
+    mysql
+    psql
+
+# Python 2.7
+# Django 1.4
+[testenv:py27-django1.4-sqlite]
+basepython = python2.7
+deps =
+    django==1.4
+commands = python runtests.py --settings tests.test_sqlite
+
+[testenv:py27-django1.4-postgres]
+basepython = python2.7
+deps =
+    django==1.4
+    psycopg2
+commands =
+    psql -c 'create database aggregation;' postgres
+    python runtests.py --settings tests.test_postgres
+    psql -c 'drop database aggregation;' postgres
+
+[testenv:py27-django1.4-mysql]
+basepython = python2.7
+deps =
+    django==1.4
+    mysql-python
+commands =
+    mysql -e 'create database aggregation;'
+    python runtests.py --settings tests.test_mysql
+    mysql -e 'drop database aggregation;'
+
+# Django 1.5
+[testenv:py27-django1.5-sqlite]
+basepython = python2.7
+deps =
+    django==1.5
+commands = python runtests.py --settings tests.test_sqlite
+
+[testenv:py27-django1.5-postgres]
+basepython = python2.7
+deps =
+    django==1.5
+    psycopg2
+commands =
+    psql -c 'create database aggregation;' postgres
+    python runtests.py --settings tests.test_postgres
+    psql -c 'drop database aggregation;' postgres
+
+[testenv:py27-django1.5-mysql]
+basepython = python2.7
+deps =
+    django==1.5
+    mysql-python
+commands =
+    mysql -e 'create database aggregation;'
+    python runtests.py --settings tests.test_mysql
+    mysql -e 'drop database aggregation;'
+
+# Django 1.6
+[testenv:py27-django1.6-sqlite]
+basepython = python2.7
+deps =
+    django==1.6
+commands = python runtests.py --settings tests.test_sqlite
+
+[testenv:py27-django1.6-postgres]
+basepython = python2.7
+deps =
+    django==1.6
+    psycopg2
+commands =
+    psql -c 'create database aggregation;' postgres
+    python runtests.py --settings tests.test_postgres
+    psql -c 'drop database aggregation;' postgres
+
+[testenv:py27-django1.6-mysql]
+basepython = python2.7
+deps =
+    django==1.6
+    mysql-python
+commands =
+    mysql -e 'create database aggregation;'
+    python runtests.py --settings tests.test_mysql
+    mysql -e 'drop database aggregation;'
+
+
+# Python 2.7 and Django 1.7
+[testenv:py27-django1.7-sqlite]
+basepython = python2.7
+deps =
+    django==1.7
+commands = python runtests.py --settings tests.test_sqlite
+
+[testenv:py27-django1.7-postgres]
+basepython = python2.7
+deps =
+    django==1.7
+    psycopg2
+commands =
+    psql -c 'create database aggregation;' postgres
+    python runtests.py --settings tests.test_postgres
+    psql -c 'drop database aggregation;' postgres
+
+[testenv:py27-django1.7-mysql]
+basepython = python2.7
+deps =
+    django==1.7
+    mysql-python
+commands =
+    mysql -e 'create database aggregation;'
+    python runtests.py --settings tests.test_mysql
+    mysql -e 'drop database aggregation;'
+
+
+# Python 3.4
+# Django 1.6
+[testenv:py34-django1.6-sqlite]
+basepython = python3.4
+deps =
+    django==1.6
+commands = python runtests.py --settings tests.test_sqlite
+
+[testenv:py34-django1.6-postgres]
+basepython = python3.4
+deps =
+    django==1.6
+    psycopg2
+commands =
+    psql -c 'create database aggregation;' postgres
+    python runtests.py --settings tests.test_postgres
+    psql -c 'drop database aggregation;' postgres
+
+[testenv:py34-django1.6-mysql]
+basepython = python3.4
+deps =
+    django==1.6
+    mysql-python3
+commands =
+    mysql -e 'create database aggregation;'
+    python runtests.py --settings tests.test_mysql
+    mysql -e 'drop database aggregation;'
+
+
+# Python 3.4
+# Django 1.7
+[testenv:py34-django1.7-sqlite]
+basepython = python3.4
+deps =
+    django==1.7
+commands = python runtests.py --settings tests.test_sqlite
+
+[testenv:py34-django1.7-postgres]
+basepython = python3.4
+deps =
+    django==1.7
+    psycopg2
+commands =
+    psql -c 'create database aggregation;' postgres
+    python runtests.py --settings tests.test_postgres
+    psql -c 'drop database aggregation;' postgres
+
+[testenv:py34-django1.7-mysql]
+basepython = python3.4
+deps =
+    django==1.7
+    mysql-python3
+commands =
+    mysql -e 'create database aggregation;'
+    python runtests.py --settings tests.test_mysql
+    mysql -e 'drop database aggregation;'

bitbake/lib/toaster/contrib/tts/README

@@ -0,0 +1,41 @@
+
+Toaster Testing Framework
+Yocto Project
+
+
+Rationale
+------------
+As Toaster contributions grow with the number of people that contribute code, verifying each patch prior to submitting upstream becomes a hard-to-scale problem for humans. We devised this system in order to run patch-level validation, trying to eliminate common problems from submitted patches, in an automated fashion.
+
+The Toaster Testing Framework is a set of Python scripts that provides an extensible way to write smoke and regression tests that will be run on each patch set sent for review on the toaster mailing list.
+
+
+Usage
+------------
+There are three main executable scripts in this directory.
+    *    runner.py  is designed to be run from the command line. It requires, as mandatory parameter, a branch name on poky-contrib, branch which contains the patches to be tested. The program will auto-discover the available tests residing in this directory by looking for unittest classes, and will run the tests on the branch dumping the output to the standard output. Optionally, it can take parameters inhibiting the branch checkout, or specifying a single test to be run, for debugging purposes.
+    *    launcher.py  is a designed to be run from a crontab or similar scheduling mechanism. It looks up a backlog file containing branches-to-test (named tasks in the source code), select the first one in FIFO manner, and launch runner.py on it. It will await for completion, and email the standard output and standard error dumps from the runner.py execution
+    *    recv.py  is an email receiver, designed to be called as a pipe from a .forward file. It is used to monitor a mailing list, for example, and add tasks to the backlog based on review requests coming on the mailing list.
+
+
+Installation
+------------
+As prerequisite, we expect a functioning email system on a machine with Python2.
+
+The broad steps to installation
+* set up the .forward on the receiving email account to pipe to the recv.py file
+* edit config.py and settings.json to alter for local installation settings
+* on email receive, verify backlog.txt to see that the tasks are received and marked for processing
+* execute launcher.py in command line to verify that a test occurs with no problems, and that the outgoing email is delivered
+* add launcher.py
+
+
+
+Contribute
+------------
+What we need are tests. Add your own tests to either tests.py file, or to a new file.
+Use "config.logger" to write logs that will make it to email.
+
+Commonly used code should be going to shellutils, and configuration to config.py.
+
+Contribute code by emailing patches to the list: toaster@yoctoproject.org (membership required)

bitbake/lib/toaster/contrib/tts/TODO

@@ -0,0 +1,9 @@
+We need to implement tests:
+
+automated link checker; currently
+$ linkchecker -t 1000 -F csv http://localhost:8000/
+
+integrate the w3c-validation service; currently
+$ python urlcheck.py
+
+

bitbake/lib/toaster/contrib/tts/config.py

@@ -0,0 +1,98 @@
+#!/usr/bin/python
+
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# Copyright (C) 2015 Alexandru Damian for Intel Corp.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+# This is the configuration/single module for tts
+# everything that would be a global variable goes here
+
+import os, sys, logging
+import socket
+
+LOGDIR = "log"
+SETTINGS_FILE = os.path.join(os.path.dirname(__file__), "settings.json")
+TEST_DIR_NAME = "tts_testdir"
+
+DEBUG = True
+
+OWN_PID = os.getpid()
+
+W3C_VALIDATOR = "http://icarus.local/w3c-validator/check?doctype=HTML5&uri="
+
+TOASTER_PORT = 56789
+
+TESTDIR = None
+
+#we parse the w3c URL to know where to connect
+
+import urlparse
+
+def get_public_ip():
+    temp_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    parsed_url = urlparse.urlparse("http://icarus.local/w3c-validator/check?doctype=HTML5&uri=")
+    temp_socket.connect((parsed_url.netloc, 80 if parsed_url.port is None else parsed_url.port))
+    public_ip = temp_socket.getsockname()[0]
+    temp_socket.close()
+    return public_ip
+
+TOASTER_BASEURL = "http://%s:%d/" % (get_public_ip(), TOASTER_PORT)
+
+
+OWN_EMAIL_ADDRESS = "Toaster Testing Framework <alexandru.damian@intel.com>"
+REPORT_EMAIL_ADDRESS = "alexandru.damian@intel.com"
+
+# make sure we have the basic logging infrastructure
+
+#pylint: disable=invalid-name
+# we disable the invalid name because the module-level "logger" is used througout bitbake
+logger = logging.getLogger("toastertest")
+__console__ = logging.StreamHandler(sys.stdout)
+__console__.setFormatter(logging.Formatter("%(asctime)s %(levelname)s: %(message)s"))
+logger.addHandler(__console__)
+logger.setLevel(logging.DEBUG)
+
+
+# singleton file names
+LOCKFILE = "/tmp/ttf.lock"
+BACKLOGFILE = os.path.join(os.path.dirname(__file__), "backlog.txt")
+
+# task states
+def enum(*sequential, **named):
+    enums = dict(zip(sequential, range(len(sequential))), **named)
+    reverse = dict((value, key) for key, value in enums.iteritems())
+    enums['reverse_mapping'] = reverse
+    return type('Enum', (), enums)
+
+
+class TASKS(object):
+    #pylint: disable=too-few-public-methods
+    PENDING = "PENDING"
+    INPROGRESS = "INPROGRESS"
+    DONE = "DONE"
+
+    @staticmethod
+    def next_task(task):
+        if task == TASKS.PENDING:
+            return TASKS.INPROGRESS
+        if task == TASKS.INPROGRESS:
+            return TASKS.DONE
+        raise Exception("Invalid next task state for %s" % task)
+
+# TTS specific
+CONTRIB_REPO = "git@git.yoctoproject.org:poky-contrib"
+

bitbake/lib/toaster/contrib/tts/launcher.py

@@ -0,0 +1,101 @@
+#!/usr/bin/python
+
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# Copyright (C) 2015 Alexandru Damian for Intel Corp.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+# Program to run the next task listed from the backlog.txt; designed to be
+# run from crontab.
+
+from __future__ import print_function
+import sys, os, config, shellutils
+from shellutils import ShellCmdException
+
+# Import smtplib for the actual sending function
+import smtplib
+
+# Import the email modules we'll need
+from email.mime.text import MIMEText
+
+def _take_lockfile():
+    return shellutils.lockfile(shellutils.mk_lock_filename())
+
+
+def read_next_task_by_state(task_state, task_name=None):
+    if not os.path.exists(os.path.join(os.path.dirname(__file__), config.BACKLOGFILE)):
+        return None
+    os.rename(config.BACKLOGFILE, config.BACKLOGFILE + ".tmp")
+    task = None
+    with open(config.BACKLOGFILE + ".tmp", "r") as f_in:
+        with open(config.BACKLOGFILE, "w") as f_out:
+            for line in f_in.readlines():
+                if task is None:
+                    fields = line.strip().split("|", 2)
+                    if fields[1] == task_state:
+                        if task_name is None or task_name == fields[0]:
+                            task = fields[0]
+                            print("Updating %s %s to %s" % (task, task_state, config.TASKS.next_task(task_state)))
+                            line = "%s|%s\n" % (task, config.TASKS.next_task(task_state))
+                f_out.write(line)
+    os.remove(config.BACKLOGFILE + ".tmp")
+    return task
+
+def send_report(task_name, plaintext, errtext=None):
+    if errtext is None:
+        msg = MIMEText(plaintext)
+    else:
+        if plaintext is None:
+            plaintext = ""
+        msg = MIMEText("--STDOUT dump--\n\n%s\n\n--STDERR dump--\n\n%s" % (plaintext, errtext))
+
+    msg['Subject'] = "[review-request] %s - smoke test results" % task_name
+    msg['From'] = config.OWN_EMAIL_ADDRESS
+    msg['To'] = config.REPORT_EMAIL_ADDRESS
+
+    smtp_connection = smtplib.SMTP("localhost")
+    smtp_connection.sendmail(config.OWN_EMAIL_ADDRESS, [config.REPORT_EMAIL_ADDRESS], msg.as_string())
+    smtp_connection.quit()
+
+def main():
+    # we don't do anything if we have another instance of us running
+    lock_file = _take_lockfile()
+
+    if lock_file is None:
+        if config.DEBUG:
+            print("Concurrent script in progress, exiting")
+        sys.exit(1)
+
+    next_task = read_next_task_by_state(config.TASKS.PENDING)
+    if next_task is not None:
+        print("Next task is", next_task)
+        errtext = None
+        out = None
+        try:
+            out = shellutils.run_shell_cmd("%s %s" % (os.path.join(os.path.dirname(__file__), "runner.py"), next_task))
+        except ShellCmdException as exc:
+            print("Failed while running the test runner: %s", exc)
+            errtext = exc.__str__()
+        send_report(next_task, out, errtext)
+        read_next_task_by_state(config.TASKS.INPROGRESS, next_task)
+    else:
+        print("No task")
+
+    shellutils.unlockfile(lock_file)
+
+
+if __name__ == "__main__":
+    main()

bitbake/lib/toaster/contrib/tts/recv.py

@@ -0,0 +1,56 @@
+#!/usr/bin/python
+
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# Copyright (C) 2015 Alexandru Damian for Intel Corp.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+# Program to receive review requests by email and log tasks to backlog.txt
+# Designed to be run by the email system from a .forward file:
+#
+# cat .forward
+# |[full/path]/recv.py
+
+from __future__ import print_function
+import sys, config, shellutils
+
+from email.parser import Parser
+
+def recv_mail(datastring):
+    headers = Parser().parsestr(datastring)
+    return headers['subject']
+
+def main():
+    lock_file = shellutils.lockfile(shellutils.mk_lock_filename(), retry=True)
+
+    if lock_file is None:
+        if config.DEBUG:
+            print("Concurrent script in progress, exiting")
+        sys.exit(1)
+
+    subject = recv_mail(sys.stdin.read())
+
+    subject_parts = subject.split()
+    if "[review-request]" in subject_parts:
+        task_name = subject_parts[subject_parts.index("[review-request]") + 1]
+        with open(config.BACKLOGFILE, "a") as fout:
+            line = "%s|%s\n" % (task_name, config.TASKS.PENDING)
+            fout.write(line)
+
+    shellutils.unlockfile(lock_file)
+
+if __name__ == "__main__":
+    main()