bitbake: command: Fix getCmdLineAction bugs

Executing "bitbake" doesn't get a sane message since the None return value
wasn't being handled correctly. Also fix msg -> cmd_action['msg'] as
otherwise an invalid variable is accessed which then crashes the server
due to the previous bug.

(Bitbake rev: c6211291ae07410832031a5274690437cc2b09a6)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

.gitignore

@@ -1,11 +1,12 @@
 *.pyc
 *.pyo
+/*.patch
 build*/conf/local.conf
 build*/conf/bblayers.conf
 build*/downloads
 build*/tmp/
 build*/sstate-cache
-build*/pyshtables.py
+pyshtables.py
 pstage/
 scripts/oe-git-proxy-socks
 sources/
@@ -16,5 +17,25 @@ meta-*
 *.orig
 *.rej
 *~
-
-
+documentation/adt-manual/adt-manual.html
+documentation/adt-manual/adt-manual.pdf
+documentation/adt-manual/adt-manual.tgz
+documentation/dev-manual/dev-manual.html
+documentation/dev-manual/dev-manual.pdf
+documentation/dev-manual/dev-manual.tgz
+documentation/poky-ref-manual/poky-ref-manual.html
+documentation/poky-ref-manual/poky-ref-manual.pdf
+documentation/poky-ref-manual/poky-ref-manual.tgz
+documentation/poky-ref-manual/bsp-guide.html
+documentation/poky-ref-manual/bsp-guide.pdf
+documentation/bsp-guide/bsp-guide.html
+documentation/bsp-guide/bsp-guide.pdf
+documentation/bsp-guide/bsp-guide.tgz
+documentation/yocto-project-qs/yocto-project-qs.html
+documentation/yocto-project-qs/yocto-project-qs.tgz
+documentation/kernel-manual/kernel-manual.html
+documentation/kernel-manual/kernel-manual.tgz
+documentation/kernel-manual/kernel-manual.pdf
+documentation/mega-manual/mega-manual.html
+documentation/mega-manual/mega-manual.pdf
+documentation/mega-manual/mega-manual.tgz

README.hardware

@@ -96,8 +96,9 @@ USB Device:
 
      # dd if=core-image-minimal-atom-pc.hddimg of=/dev/sdb
 
-  If the device fails to boot with "Boot error" displayed, it is likely the BIOS
-  cannot understand the physical layout of the disk (or rather it expects a
+  If the device fails to boot with "Boot error" displayed, or apparently
+  stops just after the SYSLINUX version banner, it is likely the BIOS cannot
+  understand the physical layout of the disk (or rather it expects a
   particular layout and cannot handle anything else). There are two possible
   solutions to this problem:
 
@@ -106,26 +107,47 @@ USB Device:
      geometry from the device.
 
   2. Without such an option, the BIOS generally boots the device in USB-ZIP
-     mode.
+     mode. To write an image to a USB device that will be bootable in
+     USB-ZIP mode, carry out the following actions:
 
-     a. Configure the USB device for USB-ZIP mode:
+     a. Determine the geometry of your USB device using fdisk:
+
+     # fdisk /dev/sdb
+     Command (m for help): p
+
+     Disk /dev/sdb: 4011 MB, 4011491328 bytes
+     124 heads, 62 sectors/track, 1019 cylinders, total 7834944 sectors
+     ...
+
+     Command (m for help): q
+
+     b. Configure the USB device for USB-ZIP mode:
      
-     # mkdiskimage -4 /dev/sdb 0 63 62
+     # mkdiskimage -4 /dev/sdb 1019 124 62
 
-     Where 63 and 62 are the head and sector count as reported by fdisk.
-     Remove and reinsert the device to allow the kernel to detect the new
-     partition layout.
+     Where 1019, 124 and 62 are the cylinder, head and sectors/track counts
+     as reported by fdisk (substitute the values reported for your device).
+     When the operation has finished and the access LED (if any) on the
+     device stops flashing, remove and reinsert the device to allow the
+     kernel to detect the new partition layout.
 
-     b. Copy the contents of the poky image to the USB-ZIP mode device:
+     c. Copy the contents of the poky image to the USB-ZIP mode device:
 
+     # mkdir /tmp/image
+     # mkdir /tmp/usbkey
      # mount -o loop core-image-minimal-atom-pc.hddimg  /tmp/image
      # mount /dev/sdb4 /tmp/usbkey
      # cp -rf /tmp/image/* /tmp/usbkey
 
-     c. Install the syslinux boot loader:
+     d. Install the syslinux boot loader:
 
      # syslinux /dev/sdb4
 
+     e. Unmount everything:
+
+     # umount /tmp/image
+     # umount /tmp/usbkey
+
   Install the boot device in the target board and configure the BIOS to boot
   from it.
 
@@ -241,8 +263,8 @@ Setup instructions
 You will need the following:
 * NFS root setup on your workstation
 * TFTP server installed on your workstation
-* Null modem cable connected from your workstation to the first serial port
-  on the board
+* Straight-thru 9-conductor serial cable (DB9, M/F) connected from your 
+  PC to UART1
 * Ethernet connected to the first ethernet port on the board
 
 --- Preparation ---

bitbake/bin/bitbake

@@ -40,7 +40,7 @@ from bb import cooker
 from bb import ui
 from bb import server
 
-__version__ = "1.15.0"
+__version__ = "1.15.2"
 logger = logging.getLogger("BitBake")
 
 
@@ -69,7 +69,7 @@ def get_ui(config):
         return getattr(module, interface).main
     except AttributeError:
         sys.exit("FATAL: Invalid user interface '%s' specified.\n"
-                 "Valid interfaces: depexp, goggle, ncurses, knotty [default]." % interface)
+                 "Valid interfaces: depexp, goggle, ncurses, hob, knotty [default], knotty2." % interface)
 
 
 # Display bitbake/OE warnings via the BitBake.Warnings logger, ignoring others"""
@@ -168,6 +168,8 @@ Default BBFILES are the .bb files in the current directory.""")
     parser.add_option("", "--server-only", help = "Run bitbake without UI,  the frontend can connect with bitbake server itself",
                action = "store_true", dest = "server_only", default = False)
 
+    parser.add_option("-B", "--bind", help = "The name/address for the bitbake server to bind to",
+               action = "store", dest = "bind", default = False)
     options, args = parser.parse_args(sys.argv)
 
     configuration = BBConfiguration(options)
@@ -189,8 +191,14 @@ Default BBFILES are the .bb files in the current directory.""")
         sys.exit("FATAL: Invalid server type '%s' specified.\n"
                  "Valid interfaces: xmlrpc, process [default], none." % servertype)
 
-    if configuration.server_only and configuration.servertype != "xmlrpc":
-        sys.exit("FATAL: If '--server-only' is defined, we must set the servertype as 'xmlrpc'.\n")
+    if configuration.server_only:
+        if configuration.servertype != "xmlrpc":
+            sys.exit("FATAL: If '--server-only' is defined, we must set the servertype as 'xmlrpc'.\n")
+        if not configuration.bind:
+            sys.exit("FATAL: The '--server-only' option requires a name/address to bind to with the -B option.\n")
+
+    if configuration.bind and configuration.servertype != "xmlrpc":
+        sys.exit("FATAL: If '-B' or '--bind' is defined, we must set the servertype as 'xmlrpc'.\n")
 
     # Save a logfile for cooker into the current working directory. When the
     # server is daemonized this logfile will be truncated.
@@ -212,8 +220,11 @@ Default BBFILES are the .bb files in the current directory.""")
     bb.utils.clean_environment()
 
     server = server.BitBakeServer()
+    if configuration.bind:
+        server.initServer((configuration.bind, 0))
+    else:
+        server.initServer()
 
-    server.initServer()
     idle = server.getServerIdleCB()
 
     cooker = bb.cooker.BBCooker(configuration, idle, initialenv)

bitbake/bin/bitbake-dumpsig

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+import os
+import sys
+import warnings
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
+
+import bb.siggen
+
+bb.siggen.dump_sigfile(sys.argv[1])

bitbake/bin/bitbake-layers

@@ -4,11 +4,16 @@
 # displaying useful information, or acting against them.
 # See the help output for details on available commands.
 
+# Copyright (C) 2011 Mentor Graphics Corporation
+# Copyright (C) 2012 Intel Corporation
+
 import cmd
 import logging
+import warnings
 import os
 import sys
 import fnmatch
+from collections import defaultdict
 
 bindir = os.path.dirname(__file__)
 topdir = os.path.dirname(bindir)
@@ -24,6 +29,7 @@ import bb.fetch2
 
 logger = logging.getLogger('BitBake')
 
+warnings.filterwarnings("ignore", category=DeprecationWarning)
 
 def main(args):
     # Set up logging
@@ -38,6 +44,8 @@ def main(args):
 
     cmds = Commands(initialenv)
     if args:
+        # Allow user to specify e.g. show-layers instead of show_layers
+        args = [args[0].replace('-', '_')] + args[1:]
         cmds.onecmd(' '.join(args))
     else:
         cmds.do_help('')
@@ -55,6 +63,7 @@ class Commands(cmd.Cmd):
         self.config_data = self.cooker.configuration.data
         bb.providers.logger.setLevel(logging.ERROR)
         self.cooker_data = None
+        self.bblayers = (self.config_data.getVar('BBLAYERS', True) or "").split()
 
     def register_idle_function(self, function, data):
         pass
@@ -90,14 +99,14 @@ class Commands(cmd.Cmd):
         """display general help or help on a specified command"""
         if topic:
             sys.stdout.write('%s: ' % topic)
-            cmd.Cmd.do_help(self,topic)
+            cmd.Cmd.do_help(self, topic.replace('-', '_'))
         else:
             sys.stdout.write("usage: bitbake-layers <command> [arguments]\n\n")
             sys.stdout.write("Available commands:\n")
             procnames = self.get_names()
             for procname in procnames:
                 if procname[:3] == 'do_':
-                    sys.stdout.write("  %s\n" % procname[3:])
+                    sys.stdout.write("  %s\n" % procname[3:].replace('_', '-'))
                     doc = getattr(self, procname).__doc__
                     if doc:
                         sys.stdout.write("    %s\n" % doc.splitlines()[0])
@@ -108,34 +117,219 @@ class Commands(cmd.Cmd):
         logger.plain('')
         logger.plain("%s  %s  %s" % ("layer".ljust(20), "path".ljust(40), "priority"))
         logger.plain('=' * 74)
-        layerdirs = str(self.config_data.getVar('BBLAYERS', True)).split()
-        for layerdir in layerdirs:
-            layername = '?'
+        for layerdir in self.bblayers:
+            layername = self.get_layer_name(layerdir)
             layerpri = 0
             for layer, _, regex, pri in self.cooker.status.bbfile_config_priorities:
                 if regex.match(os.path.join(layerdir, 'test')):
-                    layername = layer
                     layerpri = pri
                     break
 
             logger.plain("%s  %s  %d" % (layername.ljust(20), layerdir.ljust(40), layerpri))
 
+
+    def version_str(self, pe, pv, pr = None):
+        verstr = "%s" % pv
+        if pr:
+            verstr = "%s-%s" % (verstr, pr)
+        if pe:
+            verstr = "%s:%s" % (pe, verstr)
+        return verstr
+
+
     def do_show_overlayed(self, args):
-        """list overlayed recipes (where there is a recipe in another layer that has a higher layer priority)
+        """list overlayed recipes (where the same recipe exists in another layer that has a higher layer priority)
 
-usage: show_overlayed
+usage: show-overlayed [-f] [-s]
 
-Highest priority recipes are listed with the recipes they overlay as subitems.
+Lists the names of overlayed recipes and the available versions in each
+layer, with the preferred version first. Note that skipped recipes that
+are overlayed will also be listed, with a " (skipped)" suffix.
+
+Options:
+  -f   instead of the default formatting, list filenames of higher priority
+       recipes with the ones they overlay indented underneath
+  -s   only list overlayed recipes where the version is the same
 """
         self.check_prepare_cooker()
-        if self.cooker.overlayed:
-            logger.plain('Overlayed recipes:')
-            for f in self.cooker.overlayed.iterkeys():
-                logger.plain('%s' % f)
-                for of in self.cooker.overlayed[f]:
-                    logger.plain('  %s' % of)
-        else:
-            logger.plain('No overlayed recipes found')
+
+        show_filenames = False
+        show_same_ver_only = False
+        for arg in args.split():
+            if arg == '-f':
+                show_filenames = True
+            elif arg == '-s':
+                show_same_ver_only = True
+            else:
+                sys.stderr.write("show-overlayed: invalid option %s\n" % arg)
+                self.do_help('')
+                return
+
+        items_listed = self.list_recipes('Overlayed recipes', None, True, show_same_ver_only, show_filenames, True)
+
+        # Check for overlayed .bbclass files
+        classes = defaultdict(list)
+        for layerdir in self.bblayers:
+            classdir = os.path.join(layerdir, 'classes')
+            if os.path.exists(classdir):
+                for classfile in os.listdir(classdir):
+                    if os.path.splitext(classfile)[1] == '.bbclass':
+                        classes[classfile].append(classdir)
+
+        # Locating classes and other files is a bit more complicated than recipes -
+        # layer priority is not a factor; instead BitBake uses the first matching
+        # file in BBPATH, which is manipulated directly by each layer's
+        # conf/layer.conf in turn, thus the order of layers in bblayers.conf is a
+        # factor - however, each layer.conf is free to either prepend or append to
+        # BBPATH (or indeed do crazy stuff with it). Thus the order in BBPATH might
+        # not be exactly the order present in bblayers.conf either.
+        bbpath = str(self.config_data.getVar('BBPATH', True))
+        overlayed_class_found = False
+        for (classfile, classdirs) in classes.items():
+            if len(classdirs) > 1:
+                if not overlayed_class_found:
+                    logger.plain('=== Overlayed classes ===')
+                    overlayed_class_found = True
+
+                mainfile = bb.utils.which(bbpath, os.path.join('classes', classfile))
+                if show_filenames:
+                    logger.plain('%s' % mainfile)
+                else:
+                    # We effectively have to guess the layer here
+                    logger.plain('%s:' % classfile)
+                    mainlayername = '?'
+                    for layerdir in self.bblayers:
+                        classdir = os.path.join(layerdir, 'classes')
+                        if mainfile.startswith(classdir):
+                            mainlayername = self.get_layer_name(layerdir)
+                    logger.plain('  %s' % mainlayername)
+                for classdir in classdirs:
+                    fullpath = os.path.join(classdir, classfile)
+                    if fullpath != mainfile:
+                        if show_filenames:
+                            print('  %s' % fullpath)
+                        else:
+                            print('  %s' % self.get_layer_name(os.path.dirname(classdir)))
+
+        if overlayed_class_found:
+            items_listed = True;
+
+        if not items_listed:
+            logger.plain('No overlayed files found.')
+
+
+    def do_show_recipes(self, args):
+        """list available recipes, showing the layer they are provided by
+
+usage: show-recipes [-f] [-m] [pnspec]
+
+Lists the names of overlayed recipes and the available versions in each
+layer, with the preferred version first. Optionally you may specify
+pnspec to match a specified recipe name (supports wildcards). Note that
+skipped recipes will also be listed, with a " (skipped)" suffix.
+
+Options:
+  -f   instead of the default formatting, list filenames of higher priority
+       recipes with other available recipes indented underneath
+  -m   only list where multiple recipes (in the same layer or different
+       layers) exist for the same recipe name
+"""
+        self.check_prepare_cooker()
+
+        show_filenames = False
+        show_multi_provider_only = False
+        pnspec = None
+        title = 'Available recipes:'
+        for arg in args.split():
+            if arg == '-f':
+                show_filenames = True
+            elif arg == '-m':
+                show_multi_provider_only = True
+            elif not arg.startswith('-'):
+                pnspec = arg
+                title = 'Available recipes matching %s:' % pnspec
+            else:
+                sys.stderr.write("show-recipes: invalid option %s\n" % arg)
+                self.do_help('')
+                return
+        self.list_recipes(title, pnspec, False, False, show_filenames, show_multi_provider_only)
+
+
+    def list_recipes(self, title, pnspec, show_overlayed_only, show_same_ver_only, show_filenames, show_multi_provider_only):
+        pkg_pn = self.cooker.status.pkg_pn
+        (latest_versions, preferred_versions) = bb.providers.findProviders(self.cooker.configuration.data, self.cooker.status, pkg_pn)
+        allproviders = bb.providers.allProviders(self.cooker.status)
+
+        # Ensure we list skipped recipes
+        # We are largely guessing about PN, PV and the preferred version here,
+        # but we have no choice since skipped recipes are not fully parsed
+        skiplist = self.cooker.skiplist.keys()
+        skiplist.sort( key=lambda fileitem: self.cooker.calc_bbfile_priority(fileitem) )
+        skiplist.reverse()
+        for fn in skiplist:
+            recipe_parts = os.path.splitext(os.path.basename(fn))[0].split('_')
+            p = recipe_parts[0]
+            if len(recipe_parts) > 1:
+                ver = (None, recipe_parts[1], None)
+            else:
+                ver = (None, 'unknown', None)
+            allproviders[p].append((ver, fn))
+            if not p in pkg_pn:
+                pkg_pn[p] = 'dummy'
+                preferred_versions[p] = (ver, fn)
+
+        def print_item(f, pn, ver, layer, ispref):
+            if f in skiplist:
+                skipped = ' (skipped)'
+            else:
+                skipped = ''
+            if show_filenames:
+                if ispref:
+                    logger.plain("%s%s", f, skipped)
+                else:
+                    logger.plain("  %s%s", f, skipped)
+            else:
+                if ispref:
+                    logger.plain("%s:", pn)
+                logger.plain("  %s %s%s", layer.ljust(20), ver, skipped)
+
+        preffiles = []
+        items_listed = False
+        for p in sorted(pkg_pn):
+            if pnspec:
+                if not fnmatch.fnmatch(p, pnspec):
+                    continue
+
+            if len(allproviders[p]) > 1 or not show_multi_provider_only:
+                pref = preferred_versions[p]
+                preffile = bb.cache.Cache.virtualfn2realfn(pref[1])[0]
+                if preffile not in preffiles:
+                    preflayer = self.get_file_layer(preffile)
+                    multilayer = False
+                    same_ver = True
+                    provs = []
+                    for prov in allproviders[p]:
+                        provfile = bb.cache.Cache.virtualfn2realfn(prov[1])[0]
+                        provlayer = self.get_file_layer(provfile)
+                        provs.append((provfile, provlayer, prov[0]))
+                        if provlayer != preflayer:
+                            multilayer = True
+                        if prov[0] != pref[0]:
+                            same_ver = False
+
+                    if (multilayer or not show_overlayed_only) and (same_ver or not show_same_ver_only):
+                        if not items_listed:
+                            logger.plain('=== %s ===' % title)
+                            items_listed = True
+                        print_item(preffile, p, self.version_str(pref[0][0], pref[0][1]), preflayer, True)
+                        for (provfile, provlayer, provver) in provs:
+                            if provfile != preffile:
+                                print_item(provfile, p, self.version_str(provver[0], provver[1]), provlayer, False)
+                        # Ensure we don't show two entries for BBCLASSEXTENDed recipes
+                        preffiles.append(preffile)
+
+        return items_listed
+
 
     def do_flatten(self, args):
         """flattens layer configuration into a separate output directory.
@@ -179,22 +373,20 @@ build results (as the layer priority order has effectively changed).
             return
 
         self.check_prepare_cooker()
-        layers = (self.config_data.getVar('BBLAYERS', True) or "").split()
+        layers = self.bblayers
         if len(arglist) > 2:
             layernames = arglist[:-1]
             found_layernames = []
             found_layerdirs = []
             for layerdir in layers:
-                for layername, _, regex, _ in self.cooker.status.bbfile_config_priorities:
-                    if layername in layernames:
-                        if regex.match(os.path.join(layerdir, 'test')):
-                            found_layerdirs.append(layerdir)
-                            found_layernames.append(layername)
-                            break
+                layername = self.get_layer_name(layerdir)
+                if layername in layernames:
+                    found_layerdirs.append(layerdir)
+                    found_layernames.append(layername)
 
             for layername in layernames:
                 if not layername in found_layernames:
-                    logger.error('Unable to find layer %s in current configuration, please run "%s show_layers" to list configured layers' % (layername, os.path.basename(sys.argv[0])))
+                    logger.error('Unable to find layer %s in current configuration, please run "%s show-layers" to list configured layers' % (layername, os.path.basename(sys.argv[0])))
                     return
             layers = found_layerdirs
         else:
@@ -266,10 +458,9 @@ build results (as the layer priority order has effectively changed).
         first_regex = None
         layerdir = layers[0]
         for layername, pattern, regex, _ in self.cooker.status.bbfile_config_priorities:
-            if (not layernames) or layername in layernames:
-                if regex.match(os.path.join(layerdir, 'test')):
-                    first_regex = regex
-                    break
+            if regex.match(os.path.join(layerdir, 'test')):
+                first_regex = regex
+                break
 
         if first_regex:
             # Find the BBFILES entries that match (which will have come from this conf/layer.conf file)
@@ -295,17 +486,22 @@ build results (as the layer priority order has effectively changed).
                             if not entry_found:
                                 logger.warning("File %s does not match the flattened layer's BBFILES setting, you may need to edit conf/layer.conf or move the file elsewhere" % f1full)
 
-    def get_append_layer(self, appendname):
+    def get_file_layer(self, filename):
         for layer, _, regex, _ in self.cooker.status.bbfile_config_priorities:
-            if regex.match(appendname):
-                return layer
+            if regex.match(filename):
+                for layerdir in self.bblayers:
+                    if regex.match(os.path.join(layerdir, 'test')):
+                        return self.get_layer_name(layerdir)
         return "?"
 
+    def get_layer_name(self, layerdir):
+        return os.path.basename(layerdir.rstrip(os.sep))
+
     def apply_append(self, appendname, recipename):
         appendfile = open(appendname, 'r')
         recipefile = open(recipename, 'a')
         recipefile.write('\n')
-        recipefile.write('##### bbappended from %s #####\n' % self.get_append_layer(appendname))
+        recipefile.write('##### bbappended from %s #####\n' % self.get_file_layer(appendname))
         recipefile.writelines(appendfile.readlines())
         recipefile.close()
         appendfile.close()
@@ -313,7 +509,7 @@ build results (as the layer priority order has effectively changed).
     def do_show_appends(self, args):
         """list bbappend files and recipe files they apply to
 
-usage: show_appends
+usage: show-appends
 
 Recipes are listed with the bbappends that apply to them as subitems.
 """

bitbake/doc/manual/usermanual.xml

@@ -204,7 +204,7 @@ include</literal> directive.</para>
             <section>
                 <title>Inheritance</title>
                 <para><emphasis>NOTE:</emphasis> This is only supported in .bb and .bbclass files.</para>
-                <para>The <literal>inherit</literal> directive is a means of specifying what classes of functionality your .bb requires.  It is a rudimentary form of inheritance.  For example, you can easily abstract out the tasks involved in building a package that uses autoconf and automake, and put that into a bbclass for your packages to make use of.  A given bbclass is located by searching for classes/filename.oeclass in <envar>BBPATH</envar>, where filename is what you inherited.</para>
+                <para>The <literal>inherit</literal> directive is a means of specifying what classes of functionality your .bb requires.  It is a rudimentary form of inheritance.  For example, you can easily abstract out the tasks involved in building a package that uses autoconf and automake, and put that into a bbclass for your packages to make use of.  A given bbclass is located by searching for classes/filename.bbclass in <envar>BBPATH</envar>, where filename is what you inherited.</para>
             </section>
             <section>
                 <title>Tasks</title>
@@ -251,7 +251,7 @@ of the event and the content of the <varname>FILE</varname> variable.</para>
             <section>
                 <title>Variants</title>
                 <para>Two BitBake features exist to facilitate the creation of multiple buildable incarnations from a single recipe file.</para>
-                <para>The first is <varname>BBCLASSEXTEND</varname>.  This variable is a space separated list of classes used to "extend" the recipe for each variant.  As an example, setting <screen>BBCLASSEXTEND = "native"</screen> results in a second incarnation of the current recipe being available.  This second incarantion will have the "native" class inherited.</para>
+                <para>The first is <varname>BBCLASSEXTEND</varname>.  This variable is a space separated list of classes used to "extend" the recipe for each variant.  As an example, setting <screen>BBCLASSEXTEND = "native"</screen> results in a second incarnation of the current recipe being available.  This second incarnation will have the "native" class inherited.</para>
                 <para>The second feature is <varname>BBVERSIONS</varname>.  This variable allows a single recipe to build multiple versions of a project from a single recipe file, and allows you to specify conditional metadata (using the <varname>OVERRIDES</varname> mechanism) for a single version, or an optionally named range of versions:</para>
                 <para><screen>BBVERSIONS = "1.0 2.0 git"
 SRC_URI_git = "git://someurl/somepath.git"</screen></para>

bitbake/lib/bb/__init__.py

@@ -21,7 +21,7 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-__version__ = "1.15.0"
+__version__ = "1.15.2"
 
 import sys
 if sys.version_info < (2, 6, 0):
@@ -79,9 +79,8 @@ if "BBDEBUG" in os.environ:
     if level:
         bb.msg.set_debug_level(level)
 
-if os.environ.get("BBFETCH2"):
-    from bb import fetch2 as fetch
-    sys.modules['bb.fetch'] = sys.modules['bb.fetch2']
+from bb import fetch2 as fetch
+sys.modules['bb.fetch'] = sys.modules['bb.fetch2']
 
 # Messaging convenience functions
 def plain(*args):

bitbake/lib/bb/build.py

@@ -53,7 +53,7 @@ class FuncFailed(Exception):
         self.logfile = logfile
         self.name = name
         if name:
-            self.msg = "Function '%s' failed" % name
+            self.msg = 'Function failed: %s' % name
         else:
             self.msg = "Function failed"
 
@@ -70,9 +70,9 @@ class TaskBase(event.Event):
 
     def __init__(self, t, d ):
         self._task = t
-        self._package = d.getVar("PF", 1)
+        self._package = d.getVar("PF", True)
         event.Event.__init__(self)
-        self._message = "package %s: task %s: %s" % (d.getVar("PF", 1), t, bb.event.getName(self)[4:])
+        self._message = "package %s: task %s: %s" % (d.getVar("PF", True), t, self.getDisplayName())
 
     def getTask(self):
         return self._task
@@ -80,6 +80,9 @@ class TaskBase(event.Event):
     def setTask(self, task):
         self._task = task
 
+    def getDisplayName(self):
+        return bb.event.getName(self)[4:]
+
     task = property(getTask, setTask, None, "task property")
 
 class TaskStarted(TaskBase):
@@ -91,9 +94,20 @@ class TaskSucceeded(TaskBase):
 class TaskFailed(TaskBase):
     """Task execution failed"""
 
+    def __init__(self, task, logfile, metadata, errprinted = False):
+        self.logfile = logfile
+        self.errprinted = errprinted
+        super(TaskFailed, self).__init__(task, metadata)
+
+class TaskFailedSilent(TaskBase):
+    """Task execution failed (silently)"""
     def __init__(self, task, logfile, metadata):
         self.logfile = logfile
-        super(TaskFailed, self).__init__(task, metadata)
+        super(TaskFailedSilent, self).__init__(task, metadata)
+
+    def getDisplayName(self):
+        # Don't need to tell the user it was silent
+        return "Failed"
 
 class TaskInvalid(TaskBase):
 
@@ -152,8 +166,6 @@ def exec_func(func, d, dirs = None):
         bb.utils.mkdirhier(adir)
 
     ispython = flags.get('python')
-    if flags.get('fakeroot') and not flags.get('task'):
-        bb.fatal("Function %s specifies fakeroot but isn't a task?!" % func)
 
     lockflag = flags.get('lockfiles')
     if lockflag:
@@ -222,9 +234,10 @@ def exec_func_shell(function, d, runfile, cwd=None):
 
     with open(runfile, 'w') as script:
         script.write('#!/bin/sh -e\n')
-        if bb.msg.loggerDefaultVerbose:
-            script.write("set -x\n")
         data.emit_func(function, script, d)
+
+        if bb.msg.loggerVerboseLogs:
+            script.write("set -x\n")
         if cwd:
             script.write("cd %s\n" % cwd)
         script.write("%s\n" % function)
@@ -232,6 +245,10 @@ def exec_func_shell(function, d, runfile, cwd=None):
     os.chmod(runfile, 0775)
 
     cmd = runfile
+    if d.getVarFlag(function, 'fakeroot'):
+        fakerootcmd = d.getVar('FAKEROOT', True)
+        if fakerootcmd:
+            cmd = [fakerootcmd, runfile]
 
     if bb.msg.loggerDefaultVerbose:
         logfile = LogTee(logger, sys.stdout)
@@ -274,18 +291,26 @@ def _exec_task(fn, task, d, quieterr):
 
     bb.utils.mkdirhier(tempdir)
     loglink = os.path.join(tempdir, 'log.{0}'.format(task))
-    logfn = os.path.join(tempdir, 'log.{0}.{1}'.format(task, os.getpid()))
+    logbase = 'log.{0}.{1}'.format(task, os.getpid())
+    logfn = os.path.join(tempdir, logbase)
     if loglink:
         bb.utils.remove(loglink)
 
         try:
-           os.symlink(logfn, loglink)
+           os.symlink(logbase, loglink)
         except OSError:
            pass
 
     prefuncs = localdata.getVarFlag(task, 'prefuncs', expand=True)
     postfuncs = localdata.getVarFlag(task, 'postfuncs', expand=True)
 
+    class ErrorCheckHandler(logging.Handler):
+        def __init__(self):
+            self.triggered = False
+            logging.Handler.__init__(self, logging.ERROR)
+        def emit(self, record):
+            self.triggered = True
+
     # Handle logfiles
     si = file('/dev/null', 'r')
     try:
@@ -311,6 +336,9 @@ def _exec_task(fn, task, d, quieterr):
     handler.setLevel(logging.DEBUG - 2)
     bblogger.addHandler(handler)
 
+    errchk = ErrorCheckHandler()
+    bblogger.addHandler(errchk)
+
     localdata.setVar('BB_LOGFILE', logfn)
 
     event.fire(TaskStarted(task, localdata), localdata)
@@ -321,9 +349,12 @@ def _exec_task(fn, task, d, quieterr):
         for func in (postfuncs or '').split():
             exec_func(func, localdata)
     except FuncFailed as exc:
-        if not quieterr:
+        if quieterr:
+            event.fire(TaskFailedSilent(task, logfn, localdata), localdata)
+        else:
+            errprinted = errchk.triggered
             logger.error(str(exc))
-            event.fire(TaskFailed(task, logfn, localdata), localdata)
+            event.fire(TaskFailed(task, logfn, localdata, errprinted), localdata)
         return 1
     finally:
         sys.stdout.flush()
@@ -366,7 +397,7 @@ def exec_task(fn, task, d):
         if not quieterr:
             logger.error("Build of %s failed" % (task))
             logger.error(format_exc())
-            failedevent = TaskFailed(task, None, d)
+            failedevent = TaskFailed(task, None, d, True)
             event.fire(failedevent, d)
         return 1
 
@@ -427,6 +458,24 @@ def del_stamp(task, d, file_name = None):
     stamp = stamp_internal(task, d, file_name)
     bb.utils.remove(stamp)
 
+def write_taint(task, d, file_name = None):
+    """
+    Creates a "taint" file which will force the specified task and its
+    dependents to be re-run the next time by influencing the value of its
+    taskhash.
+    (d can be a data dict or dataCache)
+    """
+    import uuid
+    if file_name:
+        taintfn = d.stamp[file_name] + '.' + task + '.taint'
+    else:
+        taintfn = d.getVar('STAMP', True) + '.' + task + '.taint'
+    bb.utils.mkdirhier(os.path.dirname(taintfn))
+    # The specific content of the taint file is not really important,
+    # we just need it to be random, so a random UUID is used
+    with open(taintfn, 'w') as taintf:
+        taintf.write(str(uuid.uuid4()))
+
 def stampfile(taskname, d, file_name = None):
     """
     Return the stamp for a given task

bitbake/lib/bb/cache.py

@@ -42,10 +42,10 @@ except ImportError:
     logger.info("Importing cPickle failed. "
                 "Falling back to a very slow implementation.")
 
-__cache_version__ = "142"
+__cache_version__ = "143"
 
-def getCacheFile(path, filename):
-    return os.path.join(path, filename)
+def getCacheFile(path, filename, data_hash):
+    return os.path.join(path, filename + "." + data_hash)
 
 # RecipeInfoCommon defines common data retrieving methods
 # from meta data for caches. CoreRecipeInfo as well as other
@@ -137,9 +137,6 @@ class CoreRecipeInfo(RecipeInfoCommon):
         self.rdepends_pkg     = self.pkgvar('RDEPENDS', self.packages, metadata)
         self.rrecommends_pkg  = self.pkgvar('RRECOMMENDS', self.packages, metadata)
         self.inherits         = self.getvar('__inherit_cache', metadata)
-        self.summary          = self.getvar('SUMMARY', metadata)
-        self.license          = self.getvar('LICENSE', metadata)
-        self.section          = self.getvar('SECTION', metadata)
         self.fakerootenv      = self.getvar('FAKEROOTENV', metadata)
         self.fakerootdirs     = self.getvar('FAKEROOTDIRS', metadata)
         self.fakerootnoenv    = self.getvar('FAKEROOTNOENV', metadata)
@@ -174,9 +171,6 @@ class CoreRecipeInfo(RecipeInfoCommon):
 
         cachedata.basetaskhash = {}
         cachedata.inherits = {}
-        cachedata.summary = {}
-        cachedata.license = {}
-        cachedata.section = {}
         cachedata.fakerootenv = {}
         cachedata.fakerootnoenv = {}
         cachedata.fakerootdirs = {}
@@ -240,9 +234,6 @@ class CoreRecipeInfo(RecipeInfoCommon):
             cachedata.basetaskhash[identifier] = taskhash
 
         cachedata.inherits[fn] = self.inherits
-        cachedata.summary[fn] = self.summary
-        cachedata.license[fn] = self.license
-        cachedata.section[fn] = self.section
         cachedata.fakerootenv[fn] = self.fakerootenv
         cachedata.fakerootnoenv[fn] = self.fakerootnoenv
         cachedata.fakerootdirs[fn] = self.fakerootdirs
@@ -254,7 +245,7 @@ class Cache(object):
     BitBake Cache implementation
     """
 
-    def __init__(self, data, caches_array):
+    def __init__(self, data, data_hash, caches_array):
         # Pass caches_array information into Cache Constructor
         # It will be used in later for deciding whether we 
         # need extra cache file dump/load support 
@@ -266,6 +257,7 @@ class Cache(object):
         self.data = None
         self.data_fn = None
         self.cacheclean = True
+        self.data_hash = data_hash
 
         if self.cachedir in [None, '']:
             self.has_cache = False
@@ -274,26 +266,17 @@ class Cache(object):
             return
 
         self.has_cache = True
-        self.cachefile = getCacheFile(self.cachedir, "bb_cache.dat")
+        self.cachefile = getCacheFile(self.cachedir, "bb_cache.dat", self.data_hash)
 
         logger.debug(1, "Using cache in '%s'", self.cachedir)
         bb.utils.mkdirhier(self.cachedir)
 
-        # If any of configuration.data's dependencies are newer than the
-        # cache there isn't even any point in loading it...
-        newest_mtime = 0
-        deps = data.getVar("__base_depends")
-
-        old_mtimes = [old_mtime for _, old_mtime in deps]
-        old_mtimes.append(newest_mtime)
-        newest_mtime = max(old_mtimes)
-
         cache_ok = True
         if self.caches_array:
             for cache_class in self.caches_array:
                 if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
-                    cachefile = getCacheFile(self.cachedir, cache_class.cachefile)
-                    cache_ok = cache_ok and (bb.parse.cached_mtime_noerror(cachefile) >= newest_mtime)
+                    cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
+                    cache_ok = cache_ok and os.path.exists(cachefile)
                     cache_class.init_cacheData(self)
         if cache_ok:
             self.load_cachefile()
@@ -327,7 +310,7 @@ class Cache(object):
         # Calculate the correct cachesize of all those cache files
         for cache_class in self.caches_array:
             if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
-                cachefile = getCacheFile(self.cachedir, cache_class.cachefile)
+                cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
                 with open(cachefile, "rb") as cachefile:
                     cachesize += os.fstat(cachefile.fileno()).st_size
 
@@ -335,7 +318,7 @@ class Cache(object):
         
         for cache_class in self.caches_array:
             if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
-                cachefile = getCacheFile(self.cachedir, cache_class.cachefile)
+                cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
                 with open(cachefile, "rb") as cachefile:
                     pickled = pickle.Unpickler(cachefile)                    
                     while cachefile:
@@ -353,7 +336,7 @@ class Cache(object):
                         current_percent = 100 * current_progress / cachesize
                         if current_percent > previous_percent:
                             previous_percent = current_percent
-                            bb.event.fire(bb.event.CacheLoadProgress(current_progress),
+                            bb.event.fire(bb.event.CacheLoadProgress(current_progress, cachesize),
                                           self.data)
 
                     previous_progress += current_progress
@@ -588,7 +571,7 @@ class Cache(object):
         for cache_class in self.caches_array:
             if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
                 cache_class_name = cache_class.__name__
-                cachefile = getCacheFile(self.cachedir, cache_class.cachefile)
+                cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
                 file_dict[cache_class_name] = open(cachefile, "wb")
                 pickler_dict[cache_class_name] =  pickle.Pickler(file_dict[cache_class_name], pickle.HIGHEST_PROTOCOL)
                    
@@ -693,7 +676,7 @@ def init(cooker):
     Files causing parsing errors are evicted from the cache.
 
     """
-    return Cache(cooker.configuration.data)
+    return Cache(cooker.configuration.data, cooker.configuration.data_hash)
 
 
 class CacheData(object):

bitbake/lib/bb/cache_extra.py

@@ -40,6 +40,7 @@ class HobRecipeInfo(RecipeInfoCommon):
         self.summary = self.getvar('SUMMARY', metadata)
         self.license = self.getvar('LICENSE', metadata)
         self.section = self.getvar('SECTION', metadata)
+        self.description = self.getvar('DESCRIPTION', metadata)
 
     @classmethod
     def init_cacheData(cls, cachedata):
@@ -47,8 +48,10 @@ class HobRecipeInfo(RecipeInfoCommon):
         cachedata.summary = {}
         cachedata.license = {}
         cachedata.section = {}
+        cachedata.description = {}
 
     def add_cacheData(self, cachedata, fn):
         cachedata.summary[fn] = self.summary
         cachedata.license[fn] = self.license
         cachedata.section[fn] = self.section
+        cachedata.description[fn] = self.description

bitbake/lib/bb/codeparser.py

@@ -34,6 +34,9 @@ def check_indent(codestr):
 
 pythonparsecache = {}
 shellparsecache = {}
+pythonparsecacheextras = {}
+shellparsecacheextras = {}
+
 
 def parser_cachefile(d):
     cachedir = (d.getVar("PERSISTENT_DIR", True) or
@@ -86,22 +89,8 @@ def parser_cache_save(d):
             i = i + 1
             continue
 
-        try:
-            p = pickle.Unpickler(file(cachefile, "rb"))
-            data, version = p.load()
-        except (IOError, EOFError, ValueError):
-            data, version = None, None
-
-        if version != PARSERCACHE_VERSION:
-            shellcache = shellparsecache
-            pythoncache = pythonparsecache
-        else:
-            for h in pythonparsecache:
-                if h not in data[0]:
-                    pythoncache[h] = pythonparsecache[h]
-            for h in shellparsecache:
-                if h not in data[1]:
-                    shellcache[h] = shellparsecache[h]
+        shellcache = shellparsecacheextras
+        pythoncache = pythonparsecacheextras
 
         p = pickle.Pickler(file(cachefile + "-" + str(i), "wb"), -1)
         p.dump([[pythoncache, shellcache], PARSERCACHE_VERSION])
@@ -109,6 +98,12 @@ def parser_cache_save(d):
     bb.utils.unlockfile(lf)
     bb.utils.unlockfile(glf)
 
+def internSet(items):
+    new = set()
+    for i in items:
+        new.add(intern(i))
+    return new
+
 def parser_cache_savemerge(d):
     cachefile = parser_cachefile(d)
     if not cachefile:
@@ -144,6 +139,21 @@ def parser_cache_savemerge(d):
                 data[1][h] = extradata[1][h]
         os.unlink(f)
 
+    # When the dicts are originally created, python calls intern() on the set keys
+    # which significantly improves memory usage. Sadly the pickle/unpickle process 
+    # doesn't call intern() on the keys and results in the same strings being duplicated
+    # in memory. This also means pickle will save the same string multiple times in 
+    # the cache file. By interning the data here, the cache file shrinks dramatically
+    # meaning faster load times and the reloaded cache files also consume much less 
+    # memory. This is worth any performance hit from this loops and the use of the 
+    # intern() data storage.
+    # Python 3.x may behave better in this area
+    for h in data[0]:
+        data[0][h]["refs"] = internSet(data[0][h]["refs"])
+        data[0][h]["execs"] = internSet(data[0][h]["execs"])
+    for h in data[1]:
+        data[1][h]["execs"] = internSet(data[1][h]["execs"])
+
     p = pickle.Pickler(file(cachefile, "wb"), -1)
     p.dump([data, PARSERCACHE_VERSION])
 
@@ -230,6 +240,12 @@ class PythonParser():
             self.execs = pythonparsecache[h]["execs"]
             return
 
+        if h in pythonparsecacheextras:
+            self.references = pythonparsecacheextras[h]["refs"]
+            self.execs = pythonparsecacheextras[h]["execs"]
+            return
+
+
         code = compile(check_indent(str(node)), "<string>", "exec",
                        ast.PyCF_ONLY_AST)
 
@@ -240,9 +256,9 @@ class PythonParser():
         self.references.update(self.var_references)
         self.references.update(self.var_execs)
 
-        pythonparsecache[h] = {}
-        pythonparsecache[h]["refs"] = self.references
-        pythonparsecache[h]["execs"] = self.execs
+        pythonparsecacheextras[h] = {}
+        pythonparsecacheextras[h]["refs"] = self.references
+        pythonparsecacheextras[h]["execs"] = self.execs
 
 class ShellParser():
     def __init__(self, name, log):
@@ -264,6 +280,10 @@ class ShellParser():
             self.execs = shellparsecache[h]["execs"]
             return self.execs
 
+        if h in shellparsecacheextras:
+            self.execs = shellparsecacheextras[h]["execs"]
+            return self.execs
+
         try:
             tokens, _ = pyshyacc.parse(value, eof=True, debug=False)
         except pyshlex.NeedMore:
@@ -273,8 +293,8 @@ class ShellParser():
             self.process_tokens(token)
         self.execs = set(cmd for cmd in self.allexecs if cmd not in self.funcdefs)
 
-        shellparsecache[h] = {}
-        shellparsecache[h]["execs"] = self.execs
+        shellparsecacheextras[h] = {}
+        shellparsecacheextras[h]["execs"] = self.execs
 
         return self.execs
 

bitbake/lib/bb/command.py

@@ -44,6 +44,9 @@ class CommandFailed(CommandExit):
         self.error = message
         CommandExit.__init__(self, 1)
 
+class CommandError(Exception):
+    pass
+
 class Command:
     """
     A queue of asynchronous commands for bitbake
@@ -57,21 +60,26 @@ class Command:
         self.currentAsyncCommand = None
 
     def runCommand(self, commandline):
-        try:
-            command = commandline.pop(0)
-            if command in CommandsSync.__dict__:
-                # Can run synchronous commands straight away
-                return getattr(CommandsSync, command)(self.cmds_sync, self, commandline)
-            if self.currentAsyncCommand is not None:
-                return "Busy (%s in progress)" % self.currentAsyncCommand[0]
-            if command not in CommandsAsync.__dict__:
-                return "No such command"
-            self.currentAsyncCommand = (command, commandline)
-            self.cooker.server_registration_cb(self.cooker.runCommands, self.cooker)
-            return True
-        except:
-            import traceback
-            return traceback.format_exc()
+        command = commandline.pop(0)
+        if hasattr(CommandsSync, command):
+            # Can run synchronous commands straight away
+            command_method = getattr(self.cmds_sync, command)
+            try:
+                result = command_method(self, commandline)
+            except CommandError as exc:
+                return None, exc.args[0]
+            except Exception:
+                import traceback
+                return None, traceback.format_exc()
+            else:
+                return result, None
+        if self.currentAsyncCommand is not None:
+            return None, "Busy (%s in progress)" % self.currentAsyncCommand[0]
+        if command not in CommandsAsync.__dict__:
+            return None, "No such command"
+        self.currentAsyncCommand = (command, commandline)
+        self.cooker.server_registration_cb(self.cooker.runCommands, self.cooker)
+        return True, None
 
     def runAsyncCommand(self):
         try:
@@ -139,7 +147,13 @@ class CommandsSync:
         """
         Get any command parsed from the commandline
         """
-        return command.cooker.commandlineAction
+        cmd_action = command.cooker.commandlineAction
+        if cmd_action is None:
+            return None
+        elif 'msg' in cmd_action and cmd_action['msg']:
+            raise CommandError(cmd_action['msg'])
+        else:
+            return cmd_action['action']
 
     def getVariable(self, command, params):
         """
@@ -173,6 +187,18 @@ class CommandsSync:
         """
         command.cooker.reset()
 
+    def getCpuCount(self, command, params):
+        """
+        Get the CPU count on the bitbake server
+        """
+        return bb.utils.cpu_count()
+
+    def setConfFilter(self, command, params):
+        """
+        Set the configuration file parsing filter
+        """
+        filterfunc = params[0]
+        bb.parse.parse_py.ConfHandler.confFilters.append(filterfunc)
 
 class CommandsAsync:
     """
@@ -232,10 +258,7 @@ class CommandsAsync:
         klass) rather than generating a tree for all packages.
         """
         klass = params[0]
-        if len(params) > 1:
-            pkg_list = params[1]
-        else:
-            pkg_list = []
+        pkg_list = params[1]
 
         command.cooker.generateTargetsTree(klass, pkg_list)
         command.finishAsyncCommand()
@@ -350,3 +373,13 @@ class CommandsAsync:
         command.cooker.parseConfigurationFiles(prefiles, postfiles)
         command.finishAsyncCommand()
     parseConfigurationFiles.needcache = False
+
+    def triggerEvent(self, command, params):
+        """
+        Trigger a certain event
+        """
+        event = params[0]
+        bb.event.fire(eval(event), command.cooker.configuration.data)
+        command.currentAsyncCommand = None
+    triggerEvent.needcache = False
+

bitbake/lib/bb/cooker.py

@@ -36,6 +36,7 @@ from functools import wraps
 from collections import defaultdict
 import bb, bb.exceptions, bb.command
 from bb import utils, data, parse, event, cache, providers, taskdata, runqueue
+import Queue
 import prserv.serv
 
 logger      = logging.getLogger("BitBake")
@@ -44,9 +45,9 @@ buildlog    = logging.getLogger("BitBake.Build")
 parselog    = logging.getLogger("BitBake.Parsing")
 providerlog = logging.getLogger("BitBake.Provider")
 
-class MultipleMatches(Exception):
+class NoSpecificMatch(bb.BBHandledException):
     """
-    Exception raised when multiple file matches are found
+    Exception raised when no or multiple file matches are found
     """
 
 class NothingToBuild(Exception):
@@ -54,6 +55,11 @@ class NothingToBuild(Exception):
     Exception raised when there is nothing to build
     """
 
+class CollectionError(bb.BBHandledException):
+    """
+    Exception raised when layer configuration is incorrect
+    """
+
 class state:
     initial, parsing, running, shutdown, stop = range(5)
 
@@ -172,7 +178,7 @@ class BBCooker:
         self.configuration.data = bb.data.init()
 
         if not self.server_registration_cb:
-            bb.data.setVar("BB_WORKERCONTEXT", "1", self.configuration.data)
+            self.configuration.data.setVar("BB_WORKERCONTEXT", "1")
 
         filtered_keys = bb.utils.approved_variables()
         bb.data.inheritFromOS(self.configuration.data, self.savedenv, filtered_keys)
@@ -200,6 +206,10 @@ class BBCooker:
 
     def parseConfiguration(self):
 
+        # Set log file verbosity
+        verboselogs = bb.utils.to_boolean(self.configuration.data.getVar("BB_VERBOSE_LOGS", "0"))
+        if verboselogs:
+            bb.msg.loggerVerboseLogs = True
 
         # Change nice level if we're asked to
         nice = self.configuration.data.getVar("BB_NICE_LEVEL", True)
@@ -258,20 +268,8 @@ class BBCooker:
         # Need files parsed
         self.updateCache()
 
-        # Need to ensure data store is expanded
-        localdata = data.createCopy(self.configuration.data)
-        bb.data.update_data(localdata)
-        bb.data.expandKeys(localdata)
-
         pkg_pn = self.status.pkg_pn
-        preferred_versions = {}
-        latest_versions = {}
-
-        # Sort by priority
-        for pn in pkg_pn:
-            (last_ver, last_file, pref_ver, pref_file) = bb.providers.findBestProvider(pn, localdata, self.status)
-            preferred_versions[pn] = (pref_ver, pref_file)
-            latest_versions[pn] = (last_ver, last_file)
+        (latest_versions, preferred_versions) = bb.providers.findProviders(self.configuration.data, self.status, pkg_pn)
 
         logger.plain("%-35s %25s %25s", "Package Name", "Latest Version", "Preferred Version")
         logger.plain("%-35s %25s %25s\n", "============", "==============", "=================")
@@ -300,7 +298,7 @@ class BBCooker:
             # this showEnvironment() code path doesn't use the cache
             self.parseConfiguration()
             self.status = bb.cache.CacheData(self.caches_array)
-            self.handleCollections( self.configuration.data.getVar("BBFILE_COLLECTIONS", 1) )
+            self.handleCollections( self.configuration.data.getVar("BBFILE_COLLECTIONS", True) )
 
             fn, cls = bb.cache.Cache.virtualfn2realfn(buildfile)
             fn = self.matchFile(fn)
@@ -345,6 +343,7 @@ class BBCooker:
         """
         Prepare a runqueue and taskdata object for iteration over pkgs_to_build
         """
+        bb.event.fire(bb.event.TreeDataPreparationStarted(), self.configuration.data)
         # Need files parsed
         self.updateCache()
         # If we are told to do the None task then query the default task
@@ -361,11 +360,14 @@ class BBCooker:
         taskdata = bb.taskdata.TaskData(False, skiplist=self.skiplist)
 
         runlist = []
+        current = 0
         for k in pkgs_to_build:
             taskdata.add_provider(localdata, self.status, k)
             runlist.append([k, "do_%s" % task])
+            current += 1
+            bb.event.fire(bb.event.TreeDataPreparationProgress(current, len(pkgs_to_build)), self.configuration.data)
         taskdata.add_unresolved(localdata, self.status)
-
+        bb.event.fire(bb.event.TreeDataPreparationCompleted(len(pkgs_to_build)), self.configuration.data)
         return runlist, taskdata
 
     def generateTaskDepTreeData(self, pkgs_to_build, task):
@@ -455,8 +457,8 @@ class BBCooker:
         depend_tree["depends"] = {}
         depend_tree["pn"] = {}
         depend_tree["rdepends-pn"] = {}
-        depend_tree["packages"] = {}
         depend_tree["rdepends-pkg"] = {}
+        depend_tree["rrecs-pkg"] = {}
 
         for task in xrange(len(tasks_fnid)):
             fnid = tasks_fnid[task]
@@ -466,6 +468,10 @@ class BBCooker:
             summary = self.status.summary[fn]
             lic = self.status.license[fn]
             section = self.status.section[fn]
+            description = self.status.description[fn]
+            rdepends = self.status.rundeps[fn]
+            rrecs = self.status.runrecs[fn]
+            inherits = self.status.inherits.get(fn, None)
             if pn not in depend_tree["pn"]:
                 depend_tree["pn"][pn] = {}
                 depend_tree["pn"][pn]["filename"] = fn
@@ -473,32 +479,40 @@ class BBCooker:
                 depend_tree["pn"][pn]["summary"] = summary
                 depend_tree["pn"][pn]["license"] = lic
                 depend_tree["pn"][pn]["section"] = section
+                depend_tree["pn"][pn]["description"] = description
+                depend_tree["pn"][pn]["inherits"] = inherits
 
             if fnid not in seen_fnids:
                 seen_fnids.append(fnid)
-                packages = []
 
                 depend_tree["depends"][pn] = []
                 for dep in taskdata.depids[fnid]:
-                    depend_tree["depends"][pn].append(taskdata.build_names_index[dep])
+                    item = taskdata.build_names_index[dep]
+                    pn_provider = ""
+                    targetid = taskdata.getbuild_id(item)
+                    if targetid in taskdata.build_targets and taskdata.build_targets[targetid]:
+                        id = taskdata.build_targets[targetid][0]
+                        fn_provider = taskdata.fn_index[id]
+                        pn_provider = self.status.pkg_fn[fn_provider]
+                    else:
+                        pn_provider = item
+                    depend_tree["depends"][pn].append(pn_provider)
 
                 depend_tree["rdepends-pn"][pn] = []
                 for rdep in taskdata.rdepids[fnid]:
-                    depend_tree["rdepends-pn"][pn].append(taskdata.run_names_index[rdep])
+                    item = taskdata.run_names_index[rdep]
+                    pn_rprovider = ""
+                    targetid = taskdata.getrun_id(item)
+                    if targetid in taskdata.run_targets and taskdata.run_targets[targetid]:
+                        id = taskdata.run_targets[targetid][0]
+                        fn_rprovider = taskdata.fn_index[id]
+                        pn_rprovider = self.status.pkg_fn[fn_rprovider]
+                    else:
+                        pn_rprovider = item
+                    depend_tree["rdepends-pn"][pn].append(pn_rprovider)
 
-                rdepends = self.status.rundeps[fn]
-                for package in rdepends:
-                    depend_tree["rdepends-pkg"][package] = []
-                    for rdepend in rdepends[package]:
-                        depend_tree["rdepends-pkg"][package].append(rdepend)
-                    packages.append(package)
-
-                for package in packages:
-                    if package not in depend_tree["packages"]:
-                        depend_tree["packages"][package] = {}
-                        depend_tree["packages"][package]["pn"] = pn
-                        depend_tree["packages"][package]["filename"] = fn
-                        depend_tree["packages"][package]["version"] = version
+                depend_tree["rdepends-pkg"].update(rdepends)
+                depend_tree["rrecs-pkg"].update(rrecs)
 
         return depend_tree
 
@@ -855,12 +869,15 @@ class BBCooker:
         if data.getVar("BB_WORKERCONTEXT", False) is None:
             bb.fetch.fetcher_init(data)
         bb.codeparser.parser_cache_init(data)
-        bb.parse.init_parser(data)
         bb.event.fire(bb.event.ConfigParsed(), data)
+        bb.parse.init_parser(data)
+        data.setVar('BBINCLUDED',bb.parse.get_file_depends(data))
         self.configuration.data = data
+        self.configuration.data_hash = data.get_hash()
 
     def handleCollections( self, collections ):
         """Handle collections"""
+        errors = False
         self.status.bbfile_config_priorities = []
         if collections:
             collection_priorities = {}
@@ -869,12 +886,13 @@ class BBCooker:
             min_prio = 0
             for c in collection_list:
                 # Get collection priority if defined explicitly
-                priority = self.configuration.data.getVar("BBFILE_PRIORITY_%s" % c, 1)
+                priority = self.configuration.data.getVar("BBFILE_PRIORITY_%s" % c, True)
                 if priority:
                     try:
                         prio = int(priority)
                     except ValueError:
                         parselog.error("invalid value for BBFILE_PRIORITY_%s: \"%s\"", c, priority)
+                        errors = True
                     if min_prio == 0 or prio < min_prio:
                         min_prio = prio
                     collection_priorities[c] = prio
@@ -882,7 +900,7 @@ class BBCooker:
                     collection_priorities[c] = None
 
                 # Check dependencies and store information for priority calculation
-                deps = self.configuration.data.getVar("LAYERDEPENDS_%s" % c, 1)
+                deps = self.configuration.data.getVar("LAYERDEPENDS_%s" % c, True)
                 if deps:
                     depnamelist = []
                     deplist = deps.split()
@@ -893,6 +911,7 @@ class BBCooker:
                                 depver = int(depsplit[1])
                             except ValueError:
                                 parselog.error("invalid version value in LAYERDEPENDS_%s: \"%s\"", c, dep)
+                                errors = True
                                 continue
                         else:
                             depver = None
@@ -901,19 +920,23 @@ class BBCooker:
 
                         if dep in collection_list:
                             if depver:
-                                layerver = self.configuration.data.getVar("LAYERVERSION_%s" % dep, 1)
+                                layerver = self.configuration.data.getVar("LAYERVERSION_%s" % dep, True)
                                 if layerver:
                                     try:
                                         lver = int(layerver)
                                     except ValueError:
                                         parselog.error("invalid value for LAYERVERSION_%s: \"%s\"", c, layerver)
+                                        errors = True
                                         continue
                                     if lver <> depver:
                                         parselog.error("Layer dependency %s of layer %s is at version %d, expected %d", dep, c, lver, depver)
+                                        errors = True
                                 else:
                                     parselog.error("Layer dependency %s of layer %s has no version, expected %d", dep, c, depver)
+                                    errors = True
                         else:
                             parselog.error("Layer dependency %s of layer %s not found", dep, c)
+                            errors = True
                     collection_depends[c] = depnamelist
                 else:
                     collection_depends[c] = []
@@ -934,16 +957,21 @@ class BBCooker:
             # Calculate all layer priorities using calc_layer_priority and store in bbfile_config_priorities
             for c in collection_list:
                 calc_layer_priority(c)
-                regex = self.configuration.data.getVar("BBFILE_PATTERN_%s" % c, 1)
+                regex = self.configuration.data.getVar("BBFILE_PATTERN_%s" % c, True)
                 if regex == None:
                     parselog.error("BBFILE_PATTERN_%s not defined" % c)
+                    errors = True
                     continue
                 try:
                     cre = re.compile(regex)
                 except re.error:
                     parselog.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression", c, regex)
+                    errors = True
                     continue
                 self.status.bbfile_config_priorities.append((c, regex, cre, collection_priorities[c]))
+        if errors:
+            # We've already printed the actual error(s)
+            raise CollectionError("Errors during parsing layer configuration")
 
     def buildSetVars(self):
         """
@@ -979,10 +1007,15 @@ class BBCooker:
         """
         matches = self.matchFiles(buildfile)
         if len(matches) != 1:
-            parselog.error("Unable to match %s (%s matches found):" % (buildfile, len(matches)))
-            for f in matches:
-                parselog.error("    %s" % f)
-            raise MultipleMatches
+            if matches:
+                msg = "Unable to match '%s' to a specific recipe file - %s matches found:" % (buildfile, len(matches))
+                if matches:
+                    for f in matches:
+                        msg += "\n    %s" % f
+                parselog.error(msg)
+            else:
+                parselog.error("Unable to find any recipe file matching '%s'" % buildfile)
+            raise NoSpecificMatch
         return matches[0]
 
     def buildFile(self, buildfile, task):
@@ -998,7 +1031,7 @@ class BBCooker:
         # buildFile() doesn't use the cache
         self.parseConfiguration()
         self.status = bb.cache.CacheData(self.caches_array)
-        self.handleCollections( self.configuration.data.getVar("BBFILE_COLLECTIONS", 1) )
+        self.handleCollections( self.configuration.data.getVar("BBFILE_COLLECTIONS", True) )
 
         # If we are told to do the None task then query the default task
         if (task == None):
@@ -1033,10 +1066,10 @@ class BBCooker:
         self.status.rundeps[fn] = []
         self.status.runrecs[fn] = []
 
-        # Remove stamp for target if force mode active
+        # Invalidate task for target if force mode active
         if self.configuration.force:
-            logger.verbose("Remove stamp %s, %s", task, fn)
-            bb.build.del_stamp('do_%s' % task, self.status, fn)
+            logger.verbose("Invalidate task %s, %s", task, fn)
+            bb.parse.siggen.invalidate_task('do_%s' % task, self.status, fn)
 
         # Setup taskdata structure
         taskdata = bb.taskdata.TaskData(self.configuration.abort)
@@ -1060,8 +1093,6 @@ class BBCooker:
             try:
                 retval = rq.execute_runqueue()
             except runqueue.TaskFailure as exc:
-                for fnid in exc.args:
-                    buildlog.error("'%s' failed" % taskdata.fn_index[fnid])
                 failures += len(exc.args)
                 retval = False
             except SystemExit as exc:
@@ -1069,7 +1100,7 @@ class BBCooker:
                 return False
 
             if not retval:
-                bb.event.fire(bb.event.BuildCompleted(buildname, item, failures), self.configuration.event_data)
+                bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runq_fnid), buildname, item, failures), self.configuration.event_data)
                 self.command.finishAsyncCommand()
                 return False
             if retval is True:
@@ -1090,6 +1121,7 @@ class BBCooker:
         if (task == None):
             task = self.configuration.cmd
 
+        universe = ('universe' in targets)
         targets = self.checkPackages(targets)
 
         def buildTargetsIdle(server, rq, abort):
@@ -1101,8 +1133,6 @@ class BBCooker:
             try:
                 retval = rq.execute_runqueue()
             except runqueue.TaskFailure as exc:
-                for fnid in exc.args:
-                    buildlog.error("'%s' failed" % taskdata.fn_index[fnid])
                 failures += len(exc.args)
                 retval = False
             except SystemExit as exc:
@@ -1110,7 +1140,7 @@ class BBCooker:
                 return False
 
             if not retval:
-                bb.event.fire(bb.event.BuildCompleted(buildname, targets, failures), self.configuration.data)
+                bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runq_fnid), buildname, targets, failures), self.configuration.data)
                 self.command.finishAsyncCommand()
                 return False
             if retval is True:
@@ -1135,6 +1165,8 @@ class BBCooker:
         taskdata.add_unresolved(localdata, self.status)
 
         rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
+        if universe:
+            rq.rqdata.warn_multi_bb = True
 
         self.server_registration_cb(buildTargetsIdle, rq)
 
@@ -1143,7 +1175,7 @@ class BBCooker:
             return
 
         if self.state in (state.shutdown, state.stop):
-            self.parser.shutdown(clean=False)
+            self.parser.shutdown(clean=False, force = True)
             sys.exit(1)
 
         if self.state != state.parsing:
@@ -1153,13 +1185,13 @@ class BBCooker:
                 del self.status
             self.status = bb.cache.CacheData(self.caches_array)
 
-            ignore = self.configuration.data.getVar("ASSUME_PROVIDED", 1) or ""
+            ignore = self.configuration.data.getVar("ASSUME_PROVIDED", True) or ""
             self.status.ignored_dependencies = set(ignore.split())
 
             for dep in self.configuration.extra_assume_provided:
                 self.status.ignored_dependencies.add(dep)
 
-            self.handleCollections( self.configuration.data.getVar("BBFILE_COLLECTIONS", 1) )
+            self.handleCollections( self.configuration.data.getVar("BBFILE_COLLECTIONS", True) )
 
             (filelist, masked) = self.collect_bbfiles()
             self.configuration.data.renameVar("__depends", "__base_depends")
@@ -1188,6 +1220,7 @@ class BBCooker:
                 pkgs_to_build.append(t)
 
         if 'universe' in pkgs_to_build:
+            parselog.warn("The \"universe\" target is only intended for testing and may produce errors.")
             parselog.debug(1, "collating packages for \"universe\"")
             pkgs_to_build.remove('universe')
             for t in self.status.universe_target:
@@ -1253,7 +1286,7 @@ class BBCooker:
                     if g not in newfiles:
                         newfiles.append(g)
 
-        bbmask = self.configuration.data.getVar('BBMASK', 1)
+        bbmask = self.configuration.data.getVar('BBMASK', True)
 
         if bbmask:
             try:
@@ -1402,7 +1435,7 @@ def _parse(fn, data, include=True):
 
 @catch_parse_error
 def _inherit(bbclass, data):
-    bb.parse.BBHandler.inherit([bbclass], data)
+    bb.parse.BBHandler.inherit(bbclass, "configuration INHERITs", 0, data)
     return data
 
 class ParsingFailure(Exception):
@@ -1411,26 +1444,94 @@ class ParsingFailure(Exception):
         self.recipe = recipe
         Exception.__init__(self, realexception, recipe)
 
-def parse_file(task):
-    filename, appends, caches_array = task
-    try:
-        return True, bb.cache.Cache.parse(filename, appends, parse_file.cfg, caches_array)
-    except Exception as exc:
-        tb = sys.exc_info()[2]
-        exc.recipe = filename
-        exc.traceback = list(bb.exceptions.extract_traceback(tb, context=3))
-        raise exc
-    # Need to turn BaseExceptions into Exceptions here so we gracefully shutdown
-    # and for example a worker thread doesn't just exit on its own in response to
-    # a SystemExit event for example.
-    except BaseException as exc:
-        raise ParsingFailure(exc, filename)
+class Feeder(multiprocessing.Process):
+    def __init__(self, jobs, to_parsers, quit):
+        self.quit = quit
+        self.jobs = jobs
+        self.to_parsers = to_parsers
+        multiprocessing.Process.__init__(self)
+
+    def run(self):
+        while True:
+            try:
+                quit = self.quit.get_nowait()
+            except Queue.Empty:
+                pass
+            else:
+                if quit == 'cancel':
+                    self.to_parsers.cancel_join_thread()
+                break
+
+            try:
+                job = self.jobs.pop()
+            except IndexError:
+                break
+
+            try:
+                self.to_parsers.put(job, timeout=0.5)
+            except Queue.Full:
+                self.jobs.insert(0, job)
+                continue
+
+class Parser(multiprocessing.Process):
+    def __init__(self, jobs, results, quit, init):
+        self.jobs = jobs
+        self.results = results
+        self.quit = quit
+        self.init = init
+        multiprocessing.Process.__init__(self)
+
+    def run(self):
+        if self.init:
+            self.init()
+
+        pending = []
+        while True:
+            try:
+                self.quit.get_nowait()
+            except Queue.Empty:
+                pass
+            else:
+                self.results.cancel_join_thread()
+                break
+
+            if pending:
+                result = pending.pop()
+            else:
+                try:
+                    job = self.jobs.get(timeout=0.25)
+                except Queue.Empty:
+                    continue
+
+                if job is None:
+                    break
+                result = self.parse(*job)
+
+            try:
+                self.results.put(result, timeout=0.25)
+            except Queue.Full:
+                pending.append(result)
+
+    def parse(self, filename, appends, caches_array):
+        try:
+            return True, bb.cache.Cache.parse(filename, appends, self.cfg, caches_array)
+        except Exception as exc:
+            tb = sys.exc_info()[2]
+            exc.recipe = filename
+            exc.traceback = list(bb.exceptions.extract_traceback(tb, context=3))
+            return True, exc
+        # Need to turn BaseExceptions into Exceptions here so we gracefully shutdown
+        # and for example a worker thread doesn't just exit on its own in response to
+        # a SystemExit event for example.
+        except BaseException as exc:
+            return True, ParsingFailure(exc, filename)
 
 class CookerParser(object):
     def __init__(self, cooker, filelist, masked):
         self.filelist = filelist
         self.cooker = cooker
         self.cfgdata = cooker.configuration.data
+        self.cfghash = cooker.configuration.data_hash
 
         # Accounting statistics
         self.parsed = 0
@@ -1446,7 +1547,7 @@ class CookerParser(object):
         self.num_processes = int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS", True) or
                                  multiprocessing.cpu_count())
 
-        self.bb_cache = bb.cache.Cache(self.cfgdata, cooker.caches_array)
+        self.bb_cache = bb.cache.Cache(self.cfgdata, self.cfghash, cooker.caches_array)
         self.fromcache = []
         self.willparse = []
         for filename in self.filelist:
@@ -1459,26 +1560,36 @@ class CookerParser(object):
         self.progress_chunk = max(self.toparse / 100, 1)
 
         self.start()
+        self.haveshutdown = False
 
     def start(self):
-        def init(cfg):
-            parse_file.cfg = cfg
-            multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, args=(self.cooker.configuration.data, ), exitpriority=1)
-
         self.results = self.load_cached()
-
+        self.processes = []
         if self.toparse:
             bb.event.fire(bb.event.ParseStarted(self.toparse), self.cfgdata)
+            def init():
+                Parser.cfg = self.cfgdata
+                multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, args=(self.cfgdata,), exitpriority=1)
 
-            self.pool = multiprocessing.Pool(self.num_processes, init, [self.cfgdata])
-            parsed = self.pool.imap(parse_file, self.willparse)
-            self.pool.close()
+            self.feeder_quit = multiprocessing.Queue(maxsize=1)
+            self.parser_quit = multiprocessing.Queue(maxsize=self.num_processes)
+            self.jobs = multiprocessing.Queue(maxsize=self.num_processes)
+            self.result_queue = multiprocessing.Queue()
+            self.feeder = Feeder(self.willparse, self.jobs, self.feeder_quit)
+            self.feeder.start()
+            for i in range(0, self.num_processes):
+                parser = Parser(self.jobs, self.result_queue, self.parser_quit, init)
+                parser.start()
+                self.processes.append(parser)
 
-            self.results = itertools.chain(self.results, parsed)
+            self.results = itertools.chain(self.results, self.parse_generator())
 
-    def shutdown(self, clean=True):
+    def shutdown(self, clean=True, force=False):
         if not self.toparse:
             return
+        if self.haveshutdown:
+            return
+        self.haveshutdown = True
 
         if clean:
             event = bb.event.ParseCompleted(self.cached, self.parsed,
@@ -1486,9 +1597,25 @@ class CookerParser(object):
                                             self.virtuals, self.error,
                                             self.total)
             bb.event.fire(event, self.cfgdata)
+            self.feeder_quit.put(None)
+            for process in self.processes:
+                self.jobs.put(None)
         else:
-            self.pool.terminate()
-        self.pool.join()
+            self.feeder_quit.put('cancel')
+
+            self.parser_quit.cancel_join_thread()
+            for process in self.processes:
+                self.parser_quit.put(None)
+
+            self.jobs.cancel_join_thread()
+
+        for process in self.processes:
+            if force:
+                process.join(.1)
+                process.terminate()
+            else:
+                process.join()
+        self.feeder.join()
 
         sync = threading.Thread(target=self.bb_cache.sync)
         sync.start()
@@ -1500,34 +1627,52 @@ class CookerParser(object):
             cached, infos = self.bb_cache.load(filename, appends, self.cfgdata)
             yield not cached, infos
 
+    def parse_generator(self):
+        while True:
+            if self.parsed >= self.toparse:
+                break
+
+            try:
+                result = self.result_queue.get(timeout=0.25)
+            except Queue.Empty:
+                pass
+            else:
+                value = result[1]
+                if isinstance(value, BaseException):
+                    raise value
+                else:
+                    yield result
+
     def parse_next(self):
+        result = []
+        parsed = None
         try:
             parsed, result = self.results.next()
         except StopIteration:
             self.shutdown()
             return False
         except ParsingFailure as exc:
-            self.shutdown(clean=False)
-            bb.fatal('Unable to parse %s: %s' %
+            logger.error('Unable to parse %s: %s' %
                      (exc.recipe, bb.exceptions.to_string(exc.realexception)))
+            self.shutdown(clean=False)
         except (bb.parse.ParseError, bb.data_smart.ExpansionError) as exc:
-            bb.fatal(str(exc))
+            logger.error(str(exc))
+            self.shutdown(clean=False)
         except SyntaxError as exc:
             logger.error('Unable to parse %s', exc.recipe)
-            sys.exit(1)
+            self.shutdown(clean=False)
         except Exception as exc:
             etype, value, tb = sys.exc_info()
             logger.error('Unable to parse %s', value.recipe,
                          exc_info=(etype, value, exc.traceback))
             self.shutdown(clean=False)
-            sys.exit(1)
 
         self.current += 1
         self.virtuals += len(result)
         if parsed:
             self.parsed += 1
             if self.parsed % self.progress_chunk == 0:
-                bb.event.fire(bb.event.ParseProgress(self.parsed),
+                bb.event.fire(bb.event.ParseProgress(self.parsed, self.toparse),
                               self.cfgdata)
         else:
             self.cached += 1

bitbake/lib/bb/data.py

@@ -333,7 +333,7 @@ def generate_dependencies(d):
                     deps[dep], values[dep] = build_dependencies(dep, keys, shelldeps, vardepvals, d)
                 newdeps |=  deps[dep]
             newdeps -= seen
-        #print "For %s: %s" % (task, str(taskdeps[task]))
+        #print "For %s: %s" % (task, str(deps[task]))
     return tasklist, deps, values
 
 def inherits_class(klass, d):

bitbake/lib/bb/data_smart.py

@@ -31,6 +31,7 @@ BitBake build tools.
 import copy, re
 from collections import MutableMapping
 import logging
+import hashlib
 import bb, bb.codeparser
 from bb   import utils
 from bb.COW  import COWDictBase
@@ -57,7 +58,7 @@ class VariableParse:
             if self.varname and key:
                 if self.varname == key:
                     raise Exception("variable %s references itself!" % self.varname)
-            var = self.d.getVar(key, 1)
+            var = self.d.getVar(key, True)
             if var is not None:
                 self.references.add(key)
                 return var
@@ -459,3 +460,16 @@ class DataSmart(MutableMapping):
 
     def __delitem__(self, var):
         self.delVar(var)
+
+    def get_hash(self):
+        data = {}
+        config_whitelist = set((self.getVar("BB_HASHCONFIG_WHITELIST", True) or "").split())
+        keys = set(key for key in iter(self) if not key.startswith("__"))
+        for key in keys:
+            if key in config_whitelist:
+                continue
+            value = self.getVar(key, False) or ""
+            data.update({key:value})
+
+        data_str = str([(k, data[k]) for k in sorted(data.keys())])
+        return hashlib.md5(data_str).hexdigest()

bitbake/lib/bb/event.py

@@ -204,6 +204,27 @@ def getName(e):
     else:
         return e.__name__
 
+class OperationStarted(Event):
+    """An operation has begun"""
+    def __init__(self, msg = "Operation Started"):
+        Event.__init__(self)
+        self.msg = msg
+
+class OperationCompleted(Event):
+    """An operation has completed"""
+    def __init__(self, total, msg = "Operation Completed"):
+        Event.__init__(self)
+        self.total = total
+        self.msg = msg
+
+class OperationProgress(Event):
+    """An operation is in progress"""
+    def __init__(self, current, total, msg = "Operation in Progress"):
+        Event.__init__(self)
+        self.current = current
+        self.total = total
+        self.msg = msg + ": %s/%s" % (current, total);
+
 class ConfigParsed(Event):
     """Configuration Parsing Complete"""
 
@@ -276,14 +297,20 @@ class BuildBase(Event):
 
 
 
-class BuildStarted(BuildBase):
+class BuildStarted(BuildBase, OperationStarted):
     """bbmake build run started"""
+    def __init__(self, n, p, failures = 0):
+        OperationStarted.__init__(self, "Building Started")
+        BuildBase.__init__(self, n, p, failures)
 
-
-class BuildCompleted(BuildBase):
+class BuildCompleted(BuildBase, OperationCompleted):
     """bbmake build run completed"""
-
-
+    def __init__(self, total, n, p, failures = 0):
+        if not failures:
+            OperationCompleted.__init__(self, total, "Building Succeeded")
+        else:
+            OperationCompleted.__init__(self, total, "Building Failed")
+        BuildBase.__init__(self, n, p, failures)
 
 
 class NoProvider(Event):
@@ -329,17 +356,16 @@ class MultipleProviders(Event):
         """
         return self._candidates
 
-class ParseStarted(Event):
+class ParseStarted(OperationStarted):
     """Recipe parsing for the runqueue has begun"""
     def __init__(self, total):
-        Event.__init__(self)
+        OperationStarted.__init__(self, "Recipe parsing Started")
         self.total = total
 
-class ParseCompleted(Event):
+class ParseCompleted(OperationCompleted):
     """Recipe parsing for the runqueue has completed"""
-
     def __init__(self, cached, parsed, skipped, masked, virtuals, errors, total):
-        Event.__init__(self)
+        OperationCompleted.__init__(self, total, "Recipe parsing Completed")
         self.cached = cached
         self.parsed = parsed
         self.skipped = skipped
@@ -347,33 +373,44 @@ class ParseCompleted(Event):
         self.masked = masked
         self.errors = errors
         self.sofar = cached + parsed
-        self.total = total
 
-class ParseProgress(Event):
+class ParseProgress(OperationProgress):
     """Recipe parsing progress"""
+    def __init__(self, current, total):
+        OperationProgress.__init__(self, current, total, "Recipe parsing")
 
-    def __init__(self, current):
-        self.current = current
 
-class CacheLoadStarted(Event):
+class CacheLoadStarted(OperationStarted):
     """Loading of the dependency cache has begun"""
     def __init__(self, total):
-        Event.__init__(self)
+        OperationStarted.__init__(self, "Loading cache Started")
         self.total = total
 
-class CacheLoadProgress(Event):
+class CacheLoadProgress(OperationProgress):
     """Cache loading progress"""
-    def __init__(self, current):
-        Event.__init__(self)
-        self.current = current
+    def __init__(self, current, total):
+        OperationProgress.__init__(self, current, total, "Loading cache")
 
-class CacheLoadCompleted(Event):
+class CacheLoadCompleted(OperationCompleted):
     """Cache loading is complete"""
     def __init__(self, total, num_entries):
-        Event.__init__(self)
-        self.total = total
+        OperationCompleted.__init__(self, total, "Loading cache Completed")
         self.num_entries = num_entries
 
+class TreeDataPreparationStarted(OperationStarted):
+    """Tree data preparation started"""
+    def __init__(self):
+        OperationStarted.__init__(self, "Preparing tree data Started")
+
+class TreeDataPreparationProgress(OperationProgress):
+    """Tree data preparation is in progress"""
+    def __init__(self, current, total):
+        OperationProgress.__init__(self, current, total, "Preparing tree data")
+
+class TreeDataPreparationCompleted(OperationCompleted):
+    """Tree data preparation completed"""
+    def __init__(self, total):
+        OperationCompleted.__init__(self, total, "Preparing tree data Completed")
 
 class DepTreeGenerated(Event):
     """
@@ -467,3 +504,34 @@ class LogHandler(logging.Handler):
     def filter(self, record):
         record.taskpid = worker_pid
         return True
+
+class RequestPackageInfo(Event):
+    """
+    Event to request package information
+    """
+
+class PackageInfo(Event):
+    """
+    Package information for GUI
+    """
+    def __init__(self, pkginfolist):
+        Event.__init__(self)
+        self._pkginfolist = pkginfolist
+
+class SanityCheck(Event):
+    """
+    Event to issue sanity check
+    """
+
+class SanityCheckPassed(Event):
+    """
+    Event to indicate sanity check is passed
+    """
+
+class SanityCheckFailed(Event):
+    """
+    Event to indicate sanity check has failed
+    """
+    def __init__(self, msg):
+        Event.__init__(self)
+        self._msg = msg

bitbake/lib/bb/fetch/__init__.py

@@ -1,832 +0,0 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-"""
-BitBake 'Fetch' implementations
-
-Classes for obtaining upstream sources for the
-BitBake build tools.
-"""
-
-# Copyright (C) 2003, 2004  Chris Larson
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Based on functions from the base bb module, Copyright 2003 Holger Schurig
-
-from __future__ import absolute_import
-from __future__ import print_function
-import os, re
-import logging
-import bb
-from   bb import data
-from   bb import persist_data
-from   bb import utils
-
-__version__ = "1"
-
-logger = logging.getLogger("BitBake.Fetch")
-
-class MalformedUrl(Exception):
-    """Exception raised when encountering an invalid url"""
-
-class FetchError(Exception):
-    """Exception raised when a download fails"""
-
-class NoMethodError(Exception):
-    """Exception raised when there is no method to obtain a supplied url or set of urls"""
-
-class MissingParameterError(Exception):
-    """Exception raised when a fetch method is missing a critical parameter in the url"""
-
-class ParameterError(Exception):
-    """Exception raised when a url cannot be proccessed due to invalid parameters."""
-
-class MD5SumError(Exception):
-    """Exception raised when a MD5SUM of a file does not match the expected one"""
-
-class InvalidSRCREV(Exception):
-    """Exception raised when an invalid SRCREV is encountered"""
-
-def decodeurl(url):
-    """Decodes an URL into the tokens (scheme, network location, path,
-    user, password, parameters).
-    """
-
-    m = re.compile('(?P<type>[^:]*)://((?P<user>.+)@)?(?P<location>[^;]+)(;(?P<parm>.*))?').match(url)
-    if not m:
-        raise MalformedUrl(url)
-
-    type = m.group('type')
-    location = m.group('location')
-    if not location:
-        raise MalformedUrl(url)
-    user = m.group('user')
-    parm = m.group('parm')
-
-    locidx = location.find('/')
-    if locidx != -1 and type.lower() != 'file':
-        host = location[:locidx]
-        path = location[locidx:]
-    else:
-        host = ""
-        path = location
-    if user:
-        m = re.compile('(?P<user>[^:]+)(:?(?P<pswd>.*))').match(user)
-        if m:
-            user = m.group('user')
-            pswd = m.group('pswd')
-    else:
-        user = ''
-        pswd = ''
-
-    p = {}
-    if parm:
-        for s in parm.split(';'):
-            s1, s2 = s.split('=')
-            p[s1] = s2
-
-    return (type, host, path, user, pswd, p)
-
-def encodeurl(decoded):
-    """Encodes a URL from tokens (scheme, network location, path,
-    user, password, parameters).
-    """
-
-    (type, host, path, user, pswd, p) = decoded
-
-    if not type or not path:
-        raise MissingParameterError("Type or path url components missing when encoding %s" % decoded)
-    url = '%s://' % type
-    if user:
-        url += "%s" % user
-        if pswd:
-            url += ":%s" % pswd
-        url += "@"
-    if host:
-        url += "%s" % host
-    url += "%s" % path
-    if p:
-        for parm in p:
-            url += ";%s=%s" % (parm, p[parm])
-
-    return url
-
-def uri_replace(uri, uri_find, uri_replace, d):
-    if not uri or not uri_find or not uri_replace:
-        logger.debug(1, "uri_replace: passed an undefined value, not replacing")
-    uri_decoded = list(decodeurl(uri))
-    uri_find_decoded = list(decodeurl(uri_find))
-    uri_replace_decoded = list(decodeurl(uri_replace))
-    result_decoded = ['', '', '', '', '', {}]
-    for i in uri_find_decoded:
-        loc = uri_find_decoded.index(i)
-        result_decoded[loc] = uri_decoded[loc]
-        if isinstance(i, basestring):
-            if (re.match(i, uri_decoded[loc])):
-                result_decoded[loc] = re.sub(i, uri_replace_decoded[loc], uri_decoded[loc])
-                if uri_find_decoded.index(i) == 2:
-                    if d:
-                        localfn = bb.fetch.localpath(uri, d)
-                        if localfn:
-                            result_decoded[loc] = os.path.join(os.path.dirname(result_decoded[loc]), os.path.basename(bb.fetch.localpath(uri, d)))
-            else:
-                return uri
-    return encodeurl(result_decoded)
-
-methods = []
-urldata_cache = {}
-saved_headrevs = {}
-
-def fetcher_init(d):
-    """
-    Called to initialize the fetchers once the configuration data is known.
-    Calls before this must not hit the cache.
-    """
-    # When to drop SCM head revisions controlled by user policy
-    srcrev_policy = d.getVar('BB_SRCREV_POLICY', 1) or "clear"
-    if srcrev_policy == "cache":
-        logger.debug(1, "Keeping SRCREV cache due to cache policy of: %s", srcrev_policy)
-    elif srcrev_policy == "clear":
-        logger.debug(1, "Clearing SRCREV cache due to cache policy of: %s", srcrev_policy)
-        revs = persist_data.persist('BB_URI_HEADREVS', d)
-        try:
-            bb.fetch.saved_headrevs = revs.items()
-        except:
-            pass
-        revs.clear()
-    else:
-        raise FetchError("Invalid SRCREV cache policy of: %s" % srcrev_policy)
-
-    for m in methods:
-        if hasattr(m, "init"):
-            m.init(d)
-
-def fetcher_compare_revisions(d):
-    """
-    Compare the revisions in the persistant cache with current values and
-    return true/false on whether they've changed.
-    """
-
-    data = persist_data.persist('BB_URI_HEADREVS', d).items()
-    data2 = bb.fetch.saved_headrevs
-
-    changed = False
-    for key in data:
-        if key not in data2 or data2[key] != data[key]:
-            logger.debug(1, "%s changed", key)
-            changed = True
-            return True
-        else:
-            logger.debug(2, "%s did not change", key)
-    return False
-
-# Function call order is usually:
-#   1. init
-#   2. go
-#   3. localpaths
-# localpath can be called at any time
-
-def init(urls, d, setup = True):
-    urldata = {}
-
-    fn = d.getVar('FILE', 1)
-    if fn in urldata_cache:
-        urldata = urldata_cache[fn]
-
-    for url in urls:
-        if url not in urldata:
-            urldata[url] = FetchData(url, d)
-
-    if setup:
-        for url in urldata:
-            if not urldata[url].setup:
-                urldata[url].setup_localpath(d)
-
-    urldata_cache[fn] = urldata
-    return urldata
-
-def mirror_from_string(data):
-    return [ i.split() for i in (data or "").replace('\\n','\n').split('\n') if i ]
-
-def verify_checksum(u, ud, d):
-    """
-    verify the MD5 and SHA256 checksum for downloaded src
-
-    return value:
-        - True: checksum matched
-        - False: checksum unmatched
-
-    if checksum is missing in recipes file, "BB_STRICT_CHECKSUM" decide the return value.
-    if BB_STRICT_CHECKSUM = "1" then return false as unmatched, otherwise return true as
-    matched
-    """
-
-    if not ud.type in ["http", "https", "ftp", "ftps"]:
-        return
-
-    md5data = bb.utils.md5_file(ud.localpath)
-    sha256data = bb.utils.sha256_file(ud.localpath)
-
-    if (ud.md5_expected == None or ud.sha256_expected == None):
-        logger.warn('Missing SRC_URI checksum for %s, consider adding to the recipe:\n'
-                    'SRC_URI[%s] = "%s"\nSRC_URI[%s] = "%s"',
-                    ud.localpath, ud.md5_name, md5data,
-                    ud.sha256_name, sha256data)
-        if d.getVar("BB_STRICT_CHECKSUM", True) == "1":
-            raise FetchError("No checksum specified for %s." % u)
-        return
-
-    if (ud.md5_expected != md5data or ud.sha256_expected != sha256data):
-        logger.error('The checksums for "%s" did not match.\n'
-                     '  MD5: expected "%s", got "%s"\n'
-                     '  SHA256: expected "%s", got "%s"\n',
-                     ud.localpath, ud.md5_expected, md5data,
-                     ud.sha256_expected, sha256data)
-        raise FetchError("%s checksum mismatch." % u)
-
-def go(d, urls = None):
-    """
-    Fetch all urls
-    init must have previously been called
-    """
-    if not urls:
-        urls = d.getVar("SRC_URI", 1).split()
-    urldata = init(urls, d, True)
-
-    for u in urls:
-        ud = urldata[u]
-        m = ud.method
-        localpath = ""
-
-        if not ud.localfile:
-            continue
-
-        lf = bb.utils.lockfile(ud.lockfile)
-
-        if m.try_premirror(u, ud, d):
-            # First try fetching uri, u, from PREMIRRORS
-            mirrors = mirror_from_string(d.getVar('PREMIRRORS', True))
-            localpath = try_mirrors(d, u, mirrors, False, m.forcefetch(u, ud, d))
-        elif os.path.exists(ud.localfile):
-            localpath = ud.localfile
-
-        # Need to re-test forcefetch() which will return true if our copy is too old
-        if m.forcefetch(u, ud, d) or not localpath:
-            # Next try fetching from the original uri, u
-            try:
-                m.go(u, ud, d)
-                localpath = ud.localpath
-            except FetchError:
-                # Remove any incomplete file
-                bb.utils.remove(ud.localpath)
-                # Finally, try fetching uri, u, from MIRRORS
-                mirrors = mirror_from_string(d.getVar('MIRRORS', True))
-                localpath = try_mirrors (d, u, mirrors)
-                if not localpath or not os.path.exists(localpath):
-                    raise FetchError("Unable to fetch URL %s from any source." % u)
-
-        ud.localpath = localpath
-
-        if os.path.exists(ud.md5):
-            # Touch the md5 file to show active use of the download
-            try:
-                os.utime(ud.md5, None)
-            except:
-                # Errors aren't fatal here
-                pass
-        else:
-            # Only check the checksums if we've not seen this item before
-            verify_checksum(u, ud, d)
-            Fetch.write_md5sum(u, ud, d)
-
-        bb.utils.unlockfile(lf)
-
-def checkstatus(d, urls = None):
-    """
-    Check all urls exist upstream
-    init must have previously been called
-    """
-    urldata = init([], d, True)
-
-    if not urls:
-        urls = urldata
-
-    for u in urls:
-        ud = urldata[u]
-        m = ud.method
-        logger.debug(1, "Testing URL %s", u)
-        # First try checking uri, u, from PREMIRRORS
-        mirrors = mirror_from_string(d.getVar('PREMIRRORS', True))
-        ret = try_mirrors(d, u, mirrors, True)
-        if not ret:
-            # Next try checking from the original uri, u
-            try:
-                ret = m.checkstatus(u, ud, d)
-            except:
-                # Finally, try checking uri, u, from MIRRORS
-                mirrors = mirror_from_string(d.getVar('MIRRORS', True))
-                ret = try_mirrors (d, u, mirrors, True)
-
-        if not ret:
-            raise FetchError("URL %s doesn't work" % u)
-
-def localpaths(d):
-    """
-    Return a list of the local filenames, assuming successful fetch
-    """
-    local = []
-    urldata = init([], d, True)
-
-    for u in urldata:
-        ud = urldata[u]
-        local.append(ud.localpath)
-
-    return local
-
-srcrev_internal_call = False
-
-def get_autorev(d):
-    return get_srcrev(d)
-
-def get_srcrev(d):
-    """
-    Return the version string for the current package
-    (usually to be used as PV)
-    Most packages usually only have one SCM so we just pass on the call.
-    In the multi SCM case, we build a value based on SRCREV_FORMAT which must
-    have been set.
-    """
-
-    #
-    # Ugly code alert. localpath in the fetchers will try to evaluate SRCREV which
-    # could translate into a call to here. If it does, we need to catch this
-    # and provide some way so it knows get_srcrev is active instead of being
-    # some number etc. hence the srcrev_internal_call tracking and the magic
-    # "SRCREVINACTION" return value.
-    #
-    # Neater solutions welcome!
-    #
-    if bb.fetch.srcrev_internal_call:
-        return "SRCREVINACTION"
-
-    scms = []
-
-    # Only call setup_localpath on URIs which supports_srcrev()
-    urldata = init(d.getVar('SRC_URI', 1).split(), d, False)
-    for u in urldata:
-        ud = urldata[u]
-        if ud.method.supports_srcrev():
-            if not ud.setup:
-                ud.setup_localpath(d)
-            scms.append(u)
-
-    if len(scms) == 0:
-        logger.error("SRCREV was used yet no valid SCM was found in SRC_URI")
-        raise ParameterError
-
-    if d.getVar('BB_SRCREV_POLICY', True) != "cache":
-        d.setVar('__BB_DONT_CACHE', '1')
-
-    if len(scms) == 1:
-        return urldata[scms[0]].method.sortable_revision(scms[0], urldata[scms[0]], d)
-
-    #
-    # Mutiple SCMs are in SRC_URI so we resort to SRCREV_FORMAT
-    #
-    format = d.getVar('SRCREV_FORMAT', 1)
-    if not format:
-        logger.error("The SRCREV_FORMAT variable must be set when multiple SCMs are used.")
-        raise ParameterError
-
-    for scm in scms:
-        if 'name' in urldata[scm].parm:
-            name = urldata[scm].parm["name"]
-            rev = urldata[scm].method.sortable_revision(scm, urldata[scm], d)
-            format = format.replace(name, rev)
-
-    return format
-
-def localpath(url, d, cache = True):
-    """
-    Called from the parser with cache=False since the cache isn't ready
-    at this point. Also called from classed in OE e.g. patch.bbclass
-    """
-    ud = init([url], d)
-    if ud[url].method:
-        return ud[url].localpath
-    return url
-
-def runfetchcmd(cmd, d, quiet = False):
-    """
-    Run cmd returning the command output
-    Raise an error if interrupted or cmd fails
-    Optionally echo command output to stdout
-    """
-
-    # Need to export PATH as binary could be in metadata paths
-    # rather than host provided
-    # Also include some other variables.
-    # FIXME: Should really include all export varaiables?
-    exportvars = ['PATH', 'GIT_PROXY_COMMAND', 'GIT_PROXY_HOST',
-                  'GIT_PROXY_PORT', 'GIT_CONFIG', 'http_proxy', 'ftp_proxy',
-                  'https_proxy', 'no_proxy', 'ALL_PROXY', 'all_proxy',
-                  'KRB5CCNAME', 'SSH_AUTH_SOCK', 'SSH_AGENT_PID', 'HOME']
-
-    for var in exportvars:
-        val = data.getVar(var, d, True)
-        if val:
-            cmd = 'export ' + var + '=\"%s\"; %s' % (val, cmd)
-
-    logger.debug(1, "Running %s", cmd)
-
-    # redirect stderr to stdout
-    stdout_handle = os.popen(cmd + " 2>&1", "r")
-    output = ""
-
-    while True:
-        line = stdout_handle.readline()
-        if not line:
-            break
-        if not quiet:
-            print(line, end=' ')
-        output += line
-
-    status = stdout_handle.close() or 0
-    signal = status >> 8
-    exitstatus = status & 0xff
-
-    if signal:
-        raise FetchError("Fetch command %s failed with signal %s, output:\n%s" % (cmd, signal, output))
-    elif status != 0:
-        raise FetchError("Fetch command %s failed with exit code %s, output:\n%s" % (cmd, status, output))
-
-    return output
-
-def try_mirrors(d, uri, mirrors, check = False, force = False):
-    """
-    Try to use a mirrored version of the sources.
-    This method will be automatically called before the fetchers go.
-
-    d Is a bb.data instance
-    uri is the original uri we're trying to download
-    mirrors is the list of mirrors we're going to try
-    """
-    fpath = os.path.join(data.getVar("DL_DIR", d, 1), os.path.basename(uri))
-    if not check and os.access(fpath, os.R_OK) and not force:
-        logger.debug(1, "%s already exists, skipping checkout.", fpath)
-        return fpath
-
-    ld = d.createCopy()
-    for (find, replace) in mirrors:
-        newuri = uri_replace(uri, find, replace, ld)
-        if newuri != uri:
-            try:
-                ud = FetchData(newuri, ld)
-            except bb.fetch.NoMethodError:
-                logger.debug(1, "No method for %s", uri)
-                continue
-
-            ud.setup_localpath(ld)
-
-            try:
-                if check:
-                    found = ud.method.checkstatus(newuri, ud, ld)
-                    if found:
-                        return found
-                else:
-                    ud.method.go(newuri, ud, ld)
-                    return ud.localpath
-            except (bb.fetch.MissingParameterError,
-                    bb.fetch.FetchError,
-                    bb.fetch.MD5SumError):
-                import sys
-                (type, value, traceback) = sys.exc_info()
-                logger.debug(2, "Mirror fetch failure: %s", value)
-                bb.utils.remove(ud.localpath)
-                continue
-    return None
-
-
-class FetchData(object):
-    """
-    A class which represents the fetcher state for a given URI.
-    """
-    def __init__(self, url, d):
-        self.localfile = ""
-        (self.type, self.host, self.path, self.user, self.pswd, self.parm) = decodeurl(data.expand(url, d))
-        self.date = Fetch.getSRCDate(self, d)
-        self.url = url
-        if not self.user and "user" in self.parm:
-            self.user = self.parm["user"]
-        if not self.pswd and "pswd" in self.parm:
-            self.pswd = self.parm["pswd"]
-        self.setup = False
-
-        if "name" in self.parm:
-            self.md5_name = "%s.md5sum" % self.parm["name"]
-            self.sha256_name = "%s.sha256sum" % self.parm["name"]
-        else:
-            self.md5_name = "md5sum"
-            self.sha256_name = "sha256sum"
-        self.md5_expected = d.getVarFlag("SRC_URI", self.md5_name)
-        self.sha256_expected = d.getVarFlag("SRC_URI", self.sha256_name)
-
-        for m in methods:
-            if m.supports(url, self, d):
-                self.method = m
-                return
-        raise NoMethodError("Missing implementation for url %s" % url)
-
-    def setup_localpath(self, d):
-        self.setup = True
-        if "localpath" in self.parm:
-            # if user sets localpath for file, use it instead.
-            self.localpath = self.parm["localpath"]
-            self.basename = os.path.basename(self.localpath)
-        else:
-            premirrors = d.getVar('PREMIRRORS', True)
-            local = ""
-            if premirrors and self.url:
-                aurl = self.url.split(";")[0]
-                mirrors = mirror_from_string(premirrors)
-                for (find, replace) in mirrors:
-                    if replace.startswith("file://"):
-                        path = aurl.split("://")[1]
-                        path = path.split(";")[0]
-                        local = replace.split("://")[1] + os.path.basename(path)
-                        if local == aurl or not os.path.exists(local) or os.path.isdir(local):
-                            local = ""
-                self.localpath = local
-            if not local:
-                try:
-                    bb.fetch.srcrev_internal_call = True
-                    self.localpath = self.method.localpath(self.url, self, d)
-                finally:
-                    bb.fetch.srcrev_internal_call = False
-                # We have to clear data's internal caches since the cached value of SRCREV is now wrong.
-                # Horrible...
-                bb.data.delVar("ISHOULDNEVEREXIST", d)
-
-        if self.localpath is not None:
-            # Note: These files should always be in DL_DIR whereas localpath may not be.
-            basepath = bb.data.expand("${DL_DIR}/%s" % os.path.basename(self.localpath), d)
-            self.md5 = basepath + '.md5'
-            self.lockfile = basepath + '.lock'
-
-
-class Fetch(object):
-    """Base class for 'fetch'ing data"""
-
-    def __init__(self, urls = []):
-        self.urls = []
-
-    def supports(self, url, urldata, d):
-        """
-        Check to see if this fetch class supports a given url.
-        """
-        return 0
-
-    def localpath(self, url, urldata, d):
-        """
-        Return the local filename of a given url assuming a successful fetch.
-        Can also setup variables in urldata for use in go (saving code duplication
-        and duplicate code execution)
-        """
-        return url
-    def _strip_leading_slashes(self, relpath):
-        """
-        Remove leading slash as os.path.join can't cope
-        """
-        while os.path.isabs(relpath):
-            relpath = relpath[1:]
-        return relpath
-
-    def setUrls(self, urls):
-        self.__urls = urls
-
-    def getUrls(self):
-        return self.__urls
-
-    urls = property(getUrls, setUrls, None, "Urls property")
-
-    def forcefetch(self, url, urldata, d):
-        """
-        Force a fetch, even if localpath exists?
-        """
-        return False
-
-    def supports_srcrev(self):
-        """
-        The fetcher supports auto source revisions (SRCREV)
-        """
-        return False
-
-    def go(self, url, urldata, d):
-        """
-        Fetch urls
-        Assumes localpath was called first
-        """
-        raise NoMethodError("Missing implementation for url")
-
-    def try_premirror(self, url, urldata, d):
-        """
-        Should premirrors be used?
-        """
-        if urldata.method.forcefetch(url, urldata, d):
-            return True
-        elif os.path.exists(urldata.md5) and os.path.exists(urldata.localfile):
-            return False
-        else:
-            return True
-
-    def checkstatus(self, url, urldata, d):
-        """
-        Check the status of a URL
-        Assumes localpath was called first
-        """
-        logger.info("URL %s could not be checked for status since no method exists.", url)
-        return True
-
-    def getSRCDate(urldata, d):
-        """
-        Return the SRC Date for the component
-
-        d the bb.data module
-        """
-        if "srcdate" in urldata.parm:
-            return urldata.parm['srcdate']
-
-        pn = data.getVar("PN", d, 1)
-
-        if pn:
-            return data.getVar("SRCDATE_%s" % pn, d, 1) or data.getVar("CVSDATE_%s" % pn, d, 1) or data.getVar("SRCDATE", d, 1) or data.getVar("CVSDATE", d, 1) or data.getVar("DATE", d, 1)
-
-        return data.getVar("SRCDATE", d, 1) or data.getVar("CVSDATE", d, 1) or data.getVar("DATE", d, 1)
-    getSRCDate = staticmethod(getSRCDate)
-
-    def srcrev_internal_helper(ud, d):
-        """
-        Return:
-            a) a source revision if specified
-            b) True if auto srcrev is in action
-            c) False otherwise
-        """
-
-        if 'rev' in ud.parm:
-            return ud.parm['rev']
-
-        if 'tag' in ud.parm:
-            return ud.parm['tag']
-
-        rev = None
-        if 'name' in ud.parm:
-            pn = data.getVar("PN", d, 1)
-            rev = data.getVar("SRCREV_%s_pn-%s" % (ud.parm['name'], pn), d, 1)
-            if not rev:
-                rev = data.getVar("SRCREV_pn-%s_%s" % (pn, ud.parm['name']), d, 1)
-            if not rev:
-                rev = data.getVar("SRCREV_%s" % (ud.parm['name']), d, 1)
-        if not rev:
-            rev = data.getVar("SRCREV", d, 1)
-        if rev == "INVALID":
-            raise InvalidSRCREV("Please set SRCREV to a valid value")
-        if not rev:
-            return False
-        if rev == "SRCREVINACTION":
-            return True
-        return rev
-
-    srcrev_internal_helper = staticmethod(srcrev_internal_helper)
-
-    def localcount_internal_helper(ud, d):
-        """
-        Return:
-            a) a locked localcount if specified
-            b) None otherwise
-        """
-
-        localcount = None
-        if 'name' in ud.parm:
-            pn = data.getVar("PN", d, 1)
-            localcount = data.getVar("LOCALCOUNT_" + ud.parm['name'], d, 1)
-        if not localcount:
-            localcount = data.getVar("LOCALCOUNT", d, 1)
-        return localcount
-
-    localcount_internal_helper = staticmethod(localcount_internal_helper)
-
-    def verify_md5sum(ud, got_sum):
-        """
-        Verify the md5sum we wanted with the one we got
-        """
-        wanted_sum = ud.parm.get('md5sum')
-        if not wanted_sum:
-            return True
-
-        return wanted_sum == got_sum
-    verify_md5sum = staticmethod(verify_md5sum)
-
-    def write_md5sum(url, ud, d):
-        md5data = bb.utils.md5_file(ud.localpath)
-        # verify the md5sum
-        if not Fetch.verify_md5sum(ud, md5data):
-            raise MD5SumError(url)
-
-        md5out = file(ud.md5, 'w')
-        md5out.write(md5data)
-        md5out.close()
-    write_md5sum = staticmethod(write_md5sum)
-
-    def latest_revision(self, url, ud, d):
-        """
-        Look in the cache for the latest revision, if not present ask the SCM.
-        """
-        if not hasattr(self, "_latest_revision"):
-            raise ParameterError
-
-        revs = persist_data.persist('BB_URI_HEADREVS', d)
-        key = self.generate_revision_key(url, ud, d)
-        try:
-            return revs[key]
-        except KeyError:
-            revs[key] = rev = self._latest_revision(url, ud, d)
-            return rev
-
-    def sortable_revision(self, url, ud, d):
-        """
-
-        """
-        if hasattr(self, "_sortable_revision"):
-            return self._sortable_revision(url, ud, d)
-
-        localcounts = persist_data.persist('BB_URI_LOCALCOUNT', d)
-        key = self.generate_revision_key(url, ud, d)
-
-        latest_rev = self._build_revision(url, ud, d)
-        last_rev = localcounts.get(key + '_rev')
-        uselocalcount = d.getVar("BB_LOCALCOUNT_OVERRIDE", True) or False
-        count = None
-        if uselocalcount:
-            count = Fetch.localcount_internal_helper(ud, d)
-        if count is None:
-            count = localcounts.get(key + '_count')
-
-        if last_rev == latest_rev:
-            return str(count + "+" + latest_rev)
-
-        buildindex_provided = hasattr(self, "_sortable_buildindex")
-        if buildindex_provided:
-            count = self._sortable_buildindex(url, ud, d, latest_rev)
-
-        if count is None:
-            count = "0"
-        elif uselocalcount or buildindex_provided:
-            count = str(count)
-        else:
-            count = str(int(count) + 1)
-
-        localcounts[key + '_rev'] = latest_rev
-        localcounts[key + '_count'] = count
-
-        return str(count + "+" + latest_rev)
-
-    def generate_revision_key(self, url, ud, d):
-        key = self._revision_key(url, ud, d)
-        return "%s-%s" % (key, d.getVar("PN", True) or "")
-
-from . import cvs
-from . import git
-from . import local
-from . import svn
-from . import wget
-from . import svk
-from . import ssh
-from . import perforce
-from . import bzr
-from . import hg
-from . import osc
-from . import repo
-
-methods.append(local.Local())
-methods.append(wget.Wget())
-methods.append(svn.Svn())
-methods.append(git.Git())
-methods.append(cvs.Cvs())
-methods.append(svk.Svk())
-methods.append(ssh.SSH())
-methods.append(perforce.Perforce())
-methods.append(bzr.Bzr())
-methods.append(hg.Hg())
-methods.append(osc.Osc())
-methods.append(repo.Repo())

bitbake/lib/bb/fetch/bzr.py

@@ -1,148 +0,0 @@
-"""
-BitBake 'Fetch' implementation for bzr.
-
-"""
-
-# Copyright (C) 2007 Ross Burton
-# Copyright (C) 2007 Richard Purdie
-#
-#   Classes for obtaining upstream sources for the
-#   BitBake build tools.
-#   Copyright (C) 2003, 2004  Chris Larson
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-import os
-import sys
-import logging
-import bb
-from bb import data
-from bb.fetch import Fetch, FetchError, runfetchcmd, logger
-
-class Bzr(Fetch):
-    def supports(self, url, ud, d):
-        return ud.type in ['bzr']
-
-    def localpath (self, url, ud, d):
-
-        # Create paths to bzr checkouts
-        relpath = self._strip_leading_slashes(ud.path)
-        ud.pkgdir = os.path.join(data.expand('${BZRDIR}', d), ud.host, relpath)
-
-        revision = Fetch.srcrev_internal_helper(ud, d)
-        if revision is True:
-            ud.revision = self.latest_revision(url, ud, d)
-        elif revision:
-            ud.revision = revision
-
-        if not ud.revision:
-            ud.revision = self.latest_revision(url, ud, d)
-
-        ud.localfile = data.expand('bzr_%s_%s_%s.tar.gz' % (ud.host, ud.path.replace('/', '.'), ud.revision), d)
-
-        return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
-
-    def _buildbzrcommand(self, ud, d, command):
-        """
-        Build up an bzr commandline based on ud
-        command is "fetch", "update", "revno"
-        """
-
-        basecmd = data.expand('${FETCHCMD_bzr}', d)
-
-        proto =  ud.parm.get('proto', 'http')
-
-        bzrroot = ud.host + ud.path
-
-        options = []
-
-        if command == "revno":
-            bzrcmd = "%s revno %s %s://%s" % (basecmd, " ".join(options), proto, bzrroot)
-        else:
-            if ud.revision:
-                options.append("-r %s" % ud.revision)
-
-            if command == "fetch":
-                bzrcmd = "%s co %s %s://%s" % (basecmd, " ".join(options), proto, bzrroot)
-            elif command == "update":
-                bzrcmd = "%s pull %s --overwrite" % (basecmd, " ".join(options))
-            else:
-                raise FetchError("Invalid bzr command %s" % command)
-
-        return bzrcmd
-
-    def go(self, loc, ud, d):
-        """Fetch url"""
-
-        if os.access(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir), '.bzr'), os.R_OK):
-            bzrcmd = self._buildbzrcommand(ud, d, "update")
-            logger.debug(1, "BZR Update %s", loc)
-            os.chdir(os.path.join (ud.pkgdir, os.path.basename(ud.path)))
-            runfetchcmd(bzrcmd, d)
-        else:
-            bb.utils.remove(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir)), True)
-            bzrcmd = self._buildbzrcommand(ud, d, "fetch")
-            logger.debug(1, "BZR Checkout %s", loc)
-            bb.utils.mkdirhier(ud.pkgdir)
-            os.chdir(ud.pkgdir)
-            logger.debug(1, "Running %s", bzrcmd)
-            runfetchcmd(bzrcmd, d)
-
-        os.chdir(ud.pkgdir)
-
-        scmdata = ud.parm.get("scmdata", "")
-        if scmdata == "keep":
-            tar_flags = ""
-        else:
-            tar_flags = "--exclude '.bzr' --exclude '.bzrtags'"
-
-        # tar them up to a defined filename
-        try:
-            runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.basename(ud.pkgdir)), d)
-        except:
-            t, v, tb = sys.exc_info()
-            try:
-                os.unlink(ud.localpath)
-            except OSError:
-                pass
-            raise t, v, tb
-
-    def supports_srcrev(self):
-        return True
-
-    def _revision_key(self, url, ud, d):
-        """
-        Return a unique key for the url
-        """
-        return "bzr:" + ud.pkgdir
-
-    def _latest_revision(self, url, ud, d):
-        """
-        Return the latest upstream revision number
-        """
-        logger.debug(2, "BZR fetcher hitting network for %s", url)
-
-        output = runfetchcmd(self._buildbzrcommand(ud, d, "revno"), d, True)
-
-        return output.strip()
-
-    def _sortable_revision(self, url, ud, d):
-        """
-        Return a sortable revision number which in our case is the revision number
-        """
-
-        return self._build_revision(url, ud, d)
-
-    def _build_revision(self, url, ud, d):
-        return ud.revision

bitbake/lib/bb/fetch/cvs.py

@@ -1,172 +0,0 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-"""
-BitBake 'Fetch' implementations
-
-Classes for obtaining upstream sources for the
-BitBake build tools.
-
-"""
-
-# Copyright (C) 2003, 2004  Chris Larson
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-#Based on functions from the base bb module, Copyright 2003 Holger Schurig
-#
-
-import os
-import logging
-import bb
-from   bb import data
-from bb.fetch import Fetch, FetchError, MissingParameterError, logger
-
-class Cvs(Fetch):
-    """
-    Class to fetch a module or modules from cvs repositories
-    """
-    def supports(self, url, ud, d):
-        """
-        Check to see if a given url can be fetched with cvs.
-        """
-        return ud.type in ['cvs']
-
-    def localpath(self, url, ud, d):
-        if not "module" in ud.parm:
-            raise MissingParameterError("cvs method needs a 'module' parameter")
-        ud.module = ud.parm["module"]
-
-        ud.tag = ud.parm.get('tag', "")
-
-        # Override the default date in certain cases
-        if 'date' in ud.parm:
-            ud.date = ud.parm['date']
-        elif ud.tag:
-            ud.date = ""
-
-        norecurse = ''
-        if 'norecurse' in ud.parm:
-            norecurse = '_norecurse'
-
-        fullpath = ''
-        if 'fullpath' in ud.parm:
-            fullpath = '_fullpath'
-
-        ud.localfile = data.expand('%s_%s_%s_%s%s%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.tag, ud.date, norecurse, fullpath), d)
-
-        return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
-
-    def forcefetch(self, url, ud, d):
-        if (ud.date == "now"):
-            return True
-        return False
-
-    def go(self, loc, ud, d):
-
-        method = ud.parm.get('method', 'pserver')
-        localdir = ud.parm.get('localdir', ud.module)
-        cvs_port = ud.parm.get('port', '')
-
-        cvs_rsh = None
-        if method == "ext":
-            if "rsh" in ud.parm:
-                cvs_rsh = ud.parm["rsh"]
-
-        if method == "dir":
-            cvsroot = ud.path
-        else:
-            cvsroot = ":" + method
-            cvsproxyhost = data.getVar('CVS_PROXY_HOST', d, True)
-            if cvsproxyhost:
-                cvsroot += ";proxy=" + cvsproxyhost
-            cvsproxyport = data.getVar('CVS_PROXY_PORT', d, True)
-            if cvsproxyport:
-                cvsroot += ";proxyport=" + cvsproxyport
-            cvsroot += ":" + ud.user
-            if ud.pswd:
-                cvsroot += ":" + ud.pswd
-            cvsroot += "@" + ud.host + ":" + cvs_port + ud.path
-
-        options = []
-        if 'norecurse' in ud.parm:
-            options.append("-l")
-        if ud.date:
-            # treat YYYYMMDDHHMM specially for CVS
-            if len(ud.date) == 12:
-                options.append("-D \"%s %s:%s UTC\"" % (ud.date[0:8], ud.date[8:10], ud.date[10:12]))
-            else:
-                options.append("-D \"%s UTC\"" % ud.date)
-        if ud.tag:
-            options.append("-r %s" % ud.tag)
-
-        localdata = data.createCopy(d)
-        data.setVar('OVERRIDES', "cvs:%s" % data.getVar('OVERRIDES', localdata), localdata)
-        data.update_data(localdata)
-
-        data.setVar('CVSROOT', cvsroot, localdata)
-        data.setVar('CVSCOOPTS', " ".join(options), localdata)
-        data.setVar('CVSMODULE', ud.module, localdata)
-        cvscmd = data.getVar('FETCHCOMMAND', localdata, 1)
-        cvsupdatecmd = data.getVar('UPDATECOMMAND', localdata, 1)
-
-        if cvs_rsh:
-            cvscmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvscmd)
-            cvsupdatecmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvsupdatecmd)
-
-        # create module directory
-        logger.debug(2, "Fetch: checking for module directory")
-        pkg = data.expand('${PN}', d)
-        pkgdir = os.path.join(data.expand('${CVSDIR}', localdata), pkg)
-        moddir = os.path.join(pkgdir, localdir)
-        if os.access(os.path.join(moddir, 'CVS'), os.R_OK):
-            logger.info("Update " + loc)
-            # update sources there
-            os.chdir(moddir)
-            myret = os.system(cvsupdatecmd)
-        else:
-            logger.info("Fetch " + loc)
-            # check out sources there
-            bb.utils.mkdirhier(pkgdir)
-            os.chdir(pkgdir)
-            logger.debug(1, "Running %s", cvscmd)
-            myret = os.system(cvscmd)
-
-        if myret != 0 or not os.access(moddir, os.R_OK):
-            try:
-                os.rmdir(moddir)
-            except OSError:
-                pass
-            raise FetchError(ud.module)
-
-        scmdata = ud.parm.get("scmdata", "")
-        if scmdata == "keep":
-            tar_flags = ""
-        else:
-            tar_flags = "--exclude 'CVS'"
-
-        # tar them up to a defined filename
-        if 'fullpath' in ud.parm:
-            os.chdir(pkgdir)
-            myret = os.system("tar %s -czf %s %s" % (tar_flags, ud.localpath, localdir))
-        else:
-            os.chdir(moddir)
-            os.chdir('..')
-            myret = os.system("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.basename(moddir)))
-
-        if myret != 0:
-            try:
-                os.unlink(ud.localpath)
-            except OSError:
-                pass
-            raise FetchError(ud.module)

bitbake/lib/bb/fetch/git.py

@@ -1,339 +0,0 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-"""
-BitBake 'Fetch' git implementation
-
-"""
-
-#Copyright (C) 2005 Richard Purdie
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-import os
-import bb
-import bb.persist_data
-from   bb    import data
-from   bb.fetch import Fetch
-from   bb.fetch import runfetchcmd
-from   bb.fetch import logger
-
-class Git(Fetch):
-    """Class to fetch a module or modules from git repositories"""
-    def init(self, d):
-        #
-        # Only enable _sortable revision if the key is set
-        #
-        if d.getVar("BB_GIT_CLONE_FOR_SRCREV", True):
-            self._sortable_buildindex = self._sortable_buildindex_disabled
-    def supports(self, url, ud, d):
-        """
-        Check to see if a given url can be fetched with git.
-        """
-        return ud.type in ['git']
-
-    def localpath(self, url, ud, d):
-
-        if 'protocol' in ud.parm:
-            ud.proto = ud.parm['protocol']
-        elif not ud.host:
-            ud.proto = 'file'
-        else:
-            ud.proto = "rsync"
-
-        ud.branch = ud.parm.get("branch", "master")
-
-        gitsrcname = '%s%s' % (ud.host, ud.path.replace('/', '.'))
-        ud.mirrortarball = 'git_%s.tar.gz' % (gitsrcname)
-        ud.clonedir = os.path.join(data.expand('${GITDIR}', d), gitsrcname)
-
-        tag = Fetch.srcrev_internal_helper(ud, d)
-        if tag is True:
-            ud.tag = self.latest_revision(url, ud, d)
-        elif tag:
-            ud.tag = tag
-
-        if not ud.tag or ud.tag == "master":
-            ud.tag = self.latest_revision(url, ud, d)
-
-        subdir = ud.parm.get("subpath", "")
-        if subdir != "":
-            if subdir.endswith("/"):
-                subdir = subdir[:-1]
-            subdirpath = os.path.join(ud.path, subdir);
-        else:
-            subdirpath = ud.path;
-
-        if 'fullclone' in ud.parm:
-            ud.localfile = ud.mirrortarball
-        else:
-            ud.localfile = data.expand('git_%s%s_%s.tar.gz' % (ud.host, subdirpath.replace('/', '.'), ud.tag), d)
-
-        ud.basecmd = data.getVar("FETCHCMD_git", d, True) or "git"
-
-        if 'noclone' in ud.parm:
-            ud.localfile = None
-            return None
-
-        return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
-
-    def forcefetch(self, url, ud, d):
-        if 'fullclone' in ud.parm:
-            return True
-        if 'noclone' in ud.parm:
-            return False
-        if os.path.exists(ud.localpath):
-            return False
-        if not self._contains_ref(ud.tag, d):
-            return True
-        return False
-
-    def try_premirror(self, u, ud, d):
-        if 'noclone' in ud.parm:
-            return False
-        if os.path.exists(ud.clonedir):
-            return False
-        if os.path.exists(ud.localpath):
-            return False
-
-        return True
-
-    def go(self, loc, ud, d):
-        """Fetch url"""
-
-        if ud.user:
-            username = ud.user + '@'
-        else:
-            username = ""
-
-        repofile = os.path.join(data.getVar("DL_DIR", d, 1), ud.mirrortarball)
-
-
-        coname = '%s' % (ud.tag)
-        codir = os.path.join(ud.clonedir, coname)
-
-        # If we have no existing clone and no mirror tarball, try and obtain one
-        if not os.path.exists(ud.clonedir) and not os.path.exists(repofile):
-            try:
-                Fetch.try_mirrors(ud.mirrortarball)
-            except:
-                pass
-
-        # If the checkout doesn't exist and the mirror tarball does, extract it
-        if not os.path.exists(ud.clonedir) and os.path.exists(repofile):
-            bb.utils.mkdirhier(ud.clonedir)
-            os.chdir(ud.clonedir)
-            runfetchcmd("tar -xzf %s" % (repofile), d)
-
-        # If the repo still doesn't exist, fallback to cloning it
-        if not os.path.exists(ud.clonedir):
-            runfetchcmd("%s clone -n %s://%s%s%s %s" % (ud.basecmd, ud.proto, username, ud.host, ud.path, ud.clonedir), d)
-
-        os.chdir(ud.clonedir)
-        # Update the checkout if needed
-        if not self._contains_ref(ud.tag, d) or 'fullclone' in ud.parm:
-            # Remove all but the .git directory
-            runfetchcmd("rm * -Rf", d)
-            if 'fullclone' in ud.parm:
-                runfetchcmd("%s fetch --all" % (ud.basecmd), d)
-            else:
-                runfetchcmd("%s fetch %s://%s%s%s %s" % (ud.basecmd, ud.proto, username, ud.host, ud.path, ud.branch), d)
-            runfetchcmd("%s fetch --tags %s://%s%s%s" % (ud.basecmd, ud.proto, username, ud.host, ud.path), d)
-            runfetchcmd("%s prune-packed" % ud.basecmd, d)
-            runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d)
-
-        # Generate a mirror tarball if needed
-        os.chdir(ud.clonedir)
-        mirror_tarballs = data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True)
-        if mirror_tarballs != "0" or 'fullclone' in ud.parm:
-            logger.info("Creating tarball of git repository")
-            runfetchcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ), d)
-
-        if 'fullclone' in ud.parm:
-            return
-
-        if os.path.exists(codir):
-            bb.utils.prunedir(codir)
-
-        subdir = ud.parm.get("subpath", "")
-        if subdir != "":
-            if subdir.endswith("/"):
-                subdirbase = os.path.basename(subdir[:-1])
-            else:
-                subdirbase = os.path.basename(subdir)
-        else:
-            subdirbase = ""
-
-        if subdir != "":
-            readpathspec = ":%s" % (subdir)
-            codir = os.path.join(codir, "git")
-            coprefix = os.path.join(codir, subdirbase, "")
-        else:
-            readpathspec = ""
-            coprefix = os.path.join(codir, "git", "")
-
-        scmdata = ud.parm.get("scmdata", "")
-        if scmdata == "keep":
-            runfetchcmd("%s clone -n %s %s" % (ud.basecmd, ud.clonedir, coprefix), d)
-            os.chdir(coprefix)
-            runfetchcmd("%s checkout -q -f %s%s" % (ud.basecmd, ud.tag, readpathspec), d)
-        else:
-            bb.utils.mkdirhier(codir)
-            os.chdir(ud.clonedir)
-            runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.tag, readpathspec), d)
-            runfetchcmd("%s checkout-index -q -f --prefix=%s -a" % (ud.basecmd, coprefix), d)
-
-        os.chdir(codir)
-        logger.info("Creating tarball of git checkout")
-        runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.join(".", "*") ), d)
-
-        os.chdir(ud.clonedir)
-        bb.utils.prunedir(codir)
-
-    def supports_srcrev(self):
-        return True
-
-    def _contains_ref(self, tag, d):
-        basecmd = data.getVar("FETCHCMD_git", d, True) or "git"
-        output = runfetchcmd("%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % (basecmd, tag), d, quiet=True)
-        return output.split()[0] != "0"
-
-    def _revision_key(self, url, ud, d, branch=False):
-        """
-        Return a unique key for the url
-        """
-        key = 'git:' + ud.host + ud.path.replace('/', '.')
-        if branch:
-            return key + ud.branch
-        else:
-            return key
-
-    def generate_revision_key(self, url, ud, d, branch=False):
-        key = self._revision_key(url, ud, d, branch)
-        return "%s-%s" % (key, d.getVar("PN", True) or "")
-
-    def _latest_revision(self, url, ud, d):
-        """
-        Compute the HEAD revision for the url
-        """
-        if ud.user:
-            username = ud.user + '@'
-        else:
-            username = ""
-
-        basecmd = data.getVar("FETCHCMD_git", d, True) or "git"
-        cmd = "%s ls-remote %s://%s%s%s %s" % (basecmd, ud.proto, username, ud.host, ud.path, ud.branch)
-        output = runfetchcmd(cmd, d, True)
-        if not output:
-            raise bb.fetch.FetchError("Fetch command %s gave empty output\n" % (cmd))
-        return output.split()[0]
-
-    def latest_revision(self, url, ud, d):
-        """
-        Look in the cache for the latest revision, if not present ask the SCM.
-        """
-        revs = bb.persist_data.persist('BB_URI_HEADREVS', d)
-
-        key = self.generate_revision_key(url, ud, d, branch=True)
-
-        try:
-            return revs[key]
-        except KeyError:
-            # Compatibility with old key format, no branch included
-            oldkey = self.generate_revision_key(url, ud, d, branch=False)
-            try:
-                rev = revs[oldkey]
-            except KeyError:
-                rev = self._latest_revision(url, ud, d)
-            else:
-                del revs[oldkey]
-            revs[key] = rev
-            return rev
-
-    def sortable_revision(self, url, ud, d):
-        """
-
-        """
-        localcounts = bb.persist_data.persist('BB_URI_LOCALCOUNT', d)
-        key = self.generate_revision_key(url, ud, d, branch=True)
-        oldkey = self.generate_revision_key(url, ud, d, branch=False)
-
-        latest_rev = self._build_revision(url, ud, d)
-        last_rev = localcounts.get(key + '_rev')
-        if last_rev is None:
-            last_rev = localcounts.get(oldkey + '_rev')
-            if last_rev is not None:
-                del localcounts[oldkey + '_rev']
-                localcounts[key + '_rev'] = last_rev
-
-        uselocalcount = d.getVar("BB_LOCALCOUNT_OVERRIDE", True) or False
-        count = None
-        if uselocalcount:
-            count = Fetch.localcount_internal_helper(ud, d)
-        if count is None:
-            count = localcounts.get(key + '_count')
-        if count is None:
-            count = localcounts.get(oldkey + '_count')
-            if count is not None:
-                del localcounts[oldkey + '_count']
-                localcounts[key + '_count'] = count
-
-        if last_rev == latest_rev:
-            return str(count + "+" + latest_rev)
-
-        buildindex_provided = hasattr(self, "_sortable_buildindex")
-        if buildindex_provided:
-            count = self._sortable_buildindex(url, ud, d, latest_rev)
-        if count is None:
-            count = "0"
-        elif uselocalcount or buildindex_provided:
-            count = str(count)
-        else:
-            count = str(int(count) + 1)
-
-        localcounts[key + '_rev'] = latest_rev
-        localcounts[key + '_count'] = count
-
-        return str(count + "+" + latest_rev)
-
-    def _build_revision(self, url, ud, d):
-        return ud.tag
-
-    def _sortable_buildindex_disabled(self, url, ud, d, rev):
-        """
-        Return a suitable buildindex for the revision specified. This is done by counting revisions
-        using "git rev-list" which may or may not work in different circumstances.
-        """
-
-        cwd = os.getcwd()
-
-        # Check if we have the rev already
-
-        if not os.path.exists(ud.clonedir):
-            print("no repo")
-            self.go(None, ud, d)
-            if not os.path.exists(ud.clonedir):
-                logger.error("GIT repository for %s doesn't exist in %s, cannot get sortable buildnumber, using old value", url, ud.clonedir)
-                return None
-
-
-        os.chdir(ud.clonedir)
-        if not self._contains_ref(rev, d):
-            self.go(None, ud, d)
-
-        output = runfetchcmd("%s rev-list %s -- 2> /dev/null | wc -l" % (ud.basecmd, rev), d, quiet=True)
-        os.chdir(cwd)
-
-        buildindex = "%s" % output.split()[0]
-        logger.debug(1, "GIT repository for %s in %s is returning %s revisions in rev-list before %s", url, ud.clonedir, buildindex, rev)
-        return buildindex

bitbake/lib/bb/fetch/hg.py

@@ -1,180 +0,0 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-"""
-BitBake 'Fetch' implementation for mercurial DRCS (hg).
-
-"""
-
-# Copyright (C) 2003, 2004  Chris Larson
-# Copyright (C) 2004        Marcin Juszkiewicz
-# Copyright (C) 2007        Robert Schuster
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Based on functions from the base bb module, Copyright 2003 Holger Schurig
-
-import os
-import sys
-import logging
-import bb
-from bb import data
-from bb.fetch import Fetch
-from bb.fetch import FetchError
-from bb.fetch import MissingParameterError
-from bb.fetch import runfetchcmd
-from bb.fetch import logger
-
-class Hg(Fetch):
-    """Class to fetch from mercurial repositories"""
-    def supports(self, url, ud, d):
-        """
-        Check to see if a given url can be fetched with mercurial.
-        """
-        return ud.type in ['hg']
-
-    def forcefetch(self, url, ud, d):
-        revTag = ud.parm.get('rev', 'tip')
-        return revTag == "tip"
-
-    def localpath(self, url, ud, d):
-        if not "module" in ud.parm:
-            raise MissingParameterError("hg method needs a 'module' parameter")
-
-        ud.module = ud.parm["module"]
-
-        # Create paths to mercurial checkouts
-        relpath = self._strip_leading_slashes(ud.path)
-        ud.pkgdir = os.path.join(data.expand('${HGDIR}', d), ud.host, relpath)
-        ud.moddir = os.path.join(ud.pkgdir, ud.module)
-
-        if 'rev' in ud.parm:
-            ud.revision = ud.parm['rev']
-        else:
-            tag = Fetch.srcrev_internal_helper(ud, d)
-            if tag is True:
-                ud.revision = self.latest_revision(url, ud, d)
-            elif tag:
-                ud.revision = tag
-            else:
-                ud.revision = self.latest_revision(url, ud, d)
-
-        ud.localfile = data.expand('%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision), d)
-
-        return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
-
-    def _buildhgcommand(self, ud, d, command):
-        """
-        Build up an hg commandline based on ud
-        command is "fetch", "update", "info"
-        """
-
-        basecmd = data.expand('${FETCHCMD_hg}', d)
-
-        proto = ud.parm.get('proto', 'http')
-
-        host = ud.host
-        if proto == "file":
-            host = "/"
-            ud.host = "localhost"
-
-        if not ud.user:
-            hgroot = host + ud.path
-        else:
-            hgroot = ud.user + "@" + host + ud.path
-
-        if command is "info":
-            return "%s identify -i %s://%s/%s" % (basecmd, proto, hgroot, ud.module)
-
-        options = [];
-        if ud.revision:
-            options.append("-r %s" % ud.revision)
-
-        if command is "fetch":
-            cmd = "%s clone %s %s://%s/%s %s" % (basecmd, " ".join(options), proto, hgroot, ud.module, ud.module)
-        elif command is "pull":
-            # do not pass options list; limiting pull to rev causes the local
-            # repo not to contain it and immediately following "update" command
-            # will crash
-            cmd = "%s pull" % (basecmd)
-        elif command is "update":
-            cmd = "%s update -C %s" % (basecmd, " ".join(options))
-        else:
-            raise FetchError("Invalid hg command %s" % command)
-
-        return cmd
-
-    def go(self, loc, ud, d):
-        """Fetch url"""
-
-        logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'")
-
-        if os.access(os.path.join(ud.moddir, '.hg'), os.R_OK):
-            updatecmd = self._buildhgcommand(ud, d, "pull")
-            logger.info("Update " + loc)
-            # update sources there
-            os.chdir(ud.moddir)
-            logger.debug(1, "Running %s", updatecmd)
-            runfetchcmd(updatecmd, d)
-
-        else:
-            fetchcmd = self._buildhgcommand(ud, d, "fetch")
-            logger.info("Fetch " + loc)
-            # check out sources there
-            bb.utils.mkdirhier(ud.pkgdir)
-            os.chdir(ud.pkgdir)
-            logger.debug(1, "Running %s", fetchcmd)
-            runfetchcmd(fetchcmd, d)
-
-        # Even when we clone (fetch), we still need to update as hg's clone
-        # won't checkout the specified revision if its on a branch
-        updatecmd = self._buildhgcommand(ud, d, "update")
-        os.chdir(ud.moddir)
-        logger.debug(1, "Running %s", updatecmd)
-        runfetchcmd(updatecmd, d)
-
-        scmdata = ud.parm.get("scmdata", "")
-        if scmdata == "keep":
-            tar_flags = ""
-        else:
-            tar_flags = "--exclude '.hg' --exclude '.hgrags'"
-
-        os.chdir(ud.pkgdir)
-        try:
-            runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, ud.module), d)
-        except:
-            t, v, tb = sys.exc_info()
-            try:
-                os.unlink(ud.localpath)
-            except OSError:
-                pass
-            raise t, v, tb
-
-    def supports_srcrev(self):
-        return True
-
-    def _latest_revision(self, url, ud, d):
-        """
-        Compute tip revision for the url
-        """
-        output = runfetchcmd(self._buildhgcommand(ud, d, "info"), d)
-        return output.strip()
-
-    def _build_revision(self, url, ud, d):
-        return ud.revision
-
-    def _revision_key(self, url, ud, d):
-        """
-        Return a unique key for the url
-        """
-        return "hg:" + ud.moddir

bitbake/lib/bb/fetch/local.py

@@ -1,73 +0,0 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-"""
-BitBake 'Fetch' implementations
-
-Classes for obtaining upstream sources for the
-BitBake build tools.
-
-"""
-
-# Copyright (C) 2003, 2004  Chris Larson
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Based on functions from the base bb module, Copyright 2003 Holger Schurig
-
-import os
-import bb
-import bb.utils
-from   bb import data
-from   bb.fetch import Fetch
-
-class Local(Fetch):
-    def supports(self, url, urldata, d):
-        """
-        Check to see if a given url represents a local fetch.
-        """
-        return urldata.type in ['file']
-
-    def localpath(self, url, urldata, d):
-        """
-        Return the local filename of a given url assuming a successful fetch.
-        """
-        path = url.split("://")[1]
-        path = path.split(";")[0]
-        newpath = path
-        if path[0] != "/":
-            filespath = data.getVar('FILESPATH', d, 1)
-            if filespath:
-                newpath = bb.utils.which(filespath, path)
-            if not newpath:
-                filesdir = data.getVar('FILESDIR', d, 1)
-                if filesdir:
-                    newpath = os.path.join(filesdir, path)
-        # We don't set localfile as for this fetcher the file is already local!
-        return newpath
-
-    def go(self, url, urldata, d):
-        """Fetch urls (no-op for Local method)"""
-        # no need to fetch local files, we'll deal with them in place.
-        return 1
-
-    def checkstatus(self, url, urldata, d):
-        """
-        Check the status of the url
-        """
-        if urldata.localpath.find("*") != -1:
-            logger.info("URL %s looks like a glob and was therefore not checked.", url)
-            return True
-        if os.path.exists(urldata.localpath):
-            return True
-        return False

bitbake/lib/bb/fetch/osc.py

@@ -1,143 +0,0 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-"""
-Bitbake "Fetch" implementation for osc (Opensuse build service client).
-Based on the svn "Fetch" implementation.
-
-"""
-
-import  os
-import  sys
-import logging
-import  bb
-from    bb       import data
-from    bb       import utils
-from    bb.fetch import Fetch
-from    bb.fetch import FetchError
-from    bb.fetch import MissingParameterError
-from    bb.fetch import runfetchcmd
-
-class Osc(Fetch):
-    """Class to fetch a module or modules from Opensuse build server
-       repositories."""
-
-    def supports(self, url, ud, d):
-        """
-        Check to see if a given url can be fetched with osc.
-        """
-        return ud.type in ['osc']
-
-    def localpath(self, url, ud, d):
-        if not "module" in ud.parm:
-            raise MissingParameterError("osc method needs a 'module' parameter.")
-
-        ud.module = ud.parm["module"]
-
-        # Create paths to osc checkouts
-        relpath = self._strip_leading_slashes(ud.path)
-        ud.pkgdir = os.path.join(data.expand('${OSCDIR}', d), ud.host)
-        ud.moddir = os.path.join(ud.pkgdir, relpath, ud.module)
-
-        if 'rev' in ud.parm:
-            ud.revision = ud.parm['rev']
-        else:
-            pv = data.getVar("PV", d, 0)
-            rev = Fetch.srcrev_internal_helper(ud, d)
-            if rev and rev != True:
-                ud.revision = rev
-            else:
-                ud.revision = ""
-
-        ud.localfile = data.expand('%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.path.replace('/', '.'), ud.revision), d)
-
-        return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
-
-    def _buildosccommand(self, ud, d, command):
-        """
-        Build up an ocs commandline based on ud
-        command is "fetch", "update", "info"
-        """
-
-        basecmd = data.expand('${FETCHCMD_osc}', d)
-
-        proto = ud.parm.get('proto', 'ocs')
-
-        options = []
-
-        config = "-c %s" % self.generate_config(ud, d)
-
-        if ud.revision:
-            options.append("-r %s" % ud.revision)
-
-        coroot = self._strip_leading_slashes(ud.path)
-
-        if command is "fetch":
-            osccmd = "%s %s co %s/%s %s" % (basecmd, config, coroot, ud.module, " ".join(options))
-        elif command is "update":
-            osccmd = "%s %s up %s" % (basecmd, config, " ".join(options))
-        else:
-            raise FetchError("Invalid osc command %s" % command)
-
-        return osccmd
-
-    def go(self, loc, ud, d):
-        """
-        Fetch url
-        """
-
-        logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'")
-
-        if os.access(os.path.join(data.expand('${OSCDIR}', d), ud.path, ud.module), os.R_OK):
-            oscupdatecmd = self._buildosccommand(ud, d, "update")
-            logger.info("Update "+ loc)
-            # update sources there
-            os.chdir(ud.moddir)
-            logger.debug(1, "Running %s", oscupdatecmd)
-            runfetchcmd(oscupdatecmd, d)
-        else:
-            oscfetchcmd = self._buildosccommand(ud, d, "fetch")
-            logger.info("Fetch " + loc)
-            # check out sources there
-            bb.utils.mkdirhier(ud.pkgdir)
-            os.chdir(ud.pkgdir)
-            logger.debug(1, "Running %s", oscfetchcmd)
-            runfetchcmd(oscfetchcmd, d)
-
-        os.chdir(os.path.join(ud.pkgdir + ud.path))
-        # tar them up to a defined filename
-        try:
-            runfetchcmd("tar -czf %s %s" % (ud.localpath, ud.module), d)
-        except:
-            t, v, tb = sys.exc_info()
-            try:
-                os.unlink(ud.localpath)
-            except OSError:
-                pass
-            raise t, v, tb
-
-    def supports_srcrev(self):
-        return False
-
-    def generate_config(self, ud, d):
-        """
-        Generate a .oscrc to be used for this run.
-        """
-
-        config_path = os.path.join(data.expand('${OSCDIR}', d), "oscrc")
-        bb.utils.remove(config_path)
-
-        f = open(config_path, 'w')
-        f.write("[general]\n")
-        f.write("apisrv = %s\n" % ud.host)
-        f.write("scheme = http\n")
-        f.write("su-wrapper = su -c\n")
-        f.write("build-root = %s\n" % data.expand('${WORKDIR}', d))
-        f.write("urllist = http://moblin-obs.jf.intel.com:8888/build/%(project)s/%(repository)s/%(buildarch)s/:full/%(name)s.rpm\n")
-        f.write("extra-pkgs = gzip\n")
-        f.write("\n")
-        f.write("[%s]\n" % ud.host)
-        f.write("user = %s\n" % ud.parm["user"])
-        f.write("pass = %s\n" % ud.parm["pswd"])
-        f.close()
-
-        return config_path

bitbake/lib/bb/fetch/perforce.py

@@ -1,206 +0,0 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-"""
-BitBake 'Fetch' implementations
-
-Classes for obtaining upstream sources for the
-BitBake build tools.
-
-"""
-
-# Copyright (C) 2003, 2004  Chris Larson
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Based on functions from the base bb module, Copyright 2003 Holger Schurig
-
-from future_builtins import zip
-import os
-import logging
-import bb
-from   bb import data
-from   bb.fetch import Fetch
-from   bb.fetch import FetchError
-from   bb.fetch import logger
-
-class Perforce(Fetch):
-    def supports(self, url, ud, d):
-        return ud.type in ['p4']
-
-    def doparse(url, d):
-        parm = {}
-        path = url.split("://")[1]
-        delim = path.find("@");
-        if delim != -1:
-            (user, pswd, host, port) = path.split('@')[0].split(":")
-            path = path.split('@')[1]
-        else:
-            (host, port) = data.getVar('P4PORT', d).split(':')
-            user = ""
-            pswd = ""
-
-        if path.find(";") != -1:
-            keys=[]
-            values=[]
-            plist = path.split(';')
-            for item in plist:
-                if item.count('='):
-                    (key, value) = item.split('=')
-                    keys.append(key)
-                    values.append(value)
-
-            parm = dict(zip(keys, values))
-        path = "//" + path.split(';')[0]
-        host += ":%s" % (port)
-        parm["cset"] = Perforce.getcset(d, path, host, user, pswd, parm)
-
-        return host, path, user, pswd, parm
-    doparse = staticmethod(doparse)
-
-    def getcset(d, depot, host, user, pswd, parm):
-        p4opt = ""
-        if "cset" in parm:
-            return parm["cset"];
-        if user:
-            p4opt += " -u %s" % (user)
-        if pswd:
-            p4opt += " -P %s" % (pswd)
-        if host:
-            p4opt += " -p %s" % (host)
-
-        p4date = data.getVar("P4DATE", d, 1)
-        if "revision" in parm:
-            depot += "#%s" % (parm["revision"])
-        elif "label" in parm:
-            depot += "@%s" % (parm["label"])
-        elif p4date:
-            depot += "@%s" % (p4date)
-
-        p4cmd = data.getVar('FETCHCOMMAND_p4', d, 1)
-        logger.debug(1, "Running %s%s changes -m 1 %s", p4cmd, p4opt, depot)
-        p4file = os.popen("%s%s changes -m 1 %s" % (p4cmd, p4opt, depot))
-        cset = p4file.readline().strip()
-        logger.debug(1, "READ %s", cset)
-        if not cset:
-            return -1
-
-        return cset.split(' ')[1]
-    getcset = staticmethod(getcset)
-
-    def localpath(self, url, ud, d):
-
-        (host, path, user, pswd, parm) = Perforce.doparse(url, d)
-
-        # If a label is specified, we use that as our filename
-
-        if "label" in parm:
-            ud.localfile = "%s.tar.gz" % (parm["label"])
-            return os.path.join(data.getVar("DL_DIR", d, 1), ud.localfile)
-
-        base = path
-        which = path.find('/...')
-        if which != -1:
-            base = path[:which]
-
-        base = self._strip_leading_slashes(base)
-
-        cset = Perforce.getcset(d, path, host, user, pswd, parm)
-
-        ud.localfile = data.expand('%s+%s+%s.tar.gz' % (host, base.replace('/', '.'), cset), d)
-
-        return os.path.join(data.getVar("DL_DIR", d, 1), ud.localfile)
-
-    def go(self, loc, ud, d):
-        """
-        Fetch urls
-        """
-
-        (host, depot, user, pswd, parm) = Perforce.doparse(loc, d)
-
-        if depot.find('/...') != -1:
-            path = depot[:depot.find('/...')]
-        else:
-            path = depot
-
-        module = parm.get('module', os.path.basename(path))
-
-        localdata = data.createCopy(d)
-        data.setVar('OVERRIDES', "p4:%s" % data.getVar('OVERRIDES', localdata), localdata)
-        data.update_data(localdata)
-
-        # Get the p4 command
-        p4opt = ""
-        if user:
-            p4opt += " -u %s" % (user)
-
-        if pswd:
-            p4opt += " -P %s" % (pswd)
-
-        if host:
-            p4opt += " -p %s" % (host)
-
-        p4cmd = data.getVar('FETCHCOMMAND', localdata, 1)
-
-        # create temp directory
-        logger.debug(2, "Fetch: creating temporary directory")
-        bb.utils.mkdirhier(data.expand('${WORKDIR}', localdata))
-        data.setVar('TMPBASE', data.expand('${WORKDIR}/oep4.XXXXXX', localdata), localdata)
-        tmppipe = os.popen(data.getVar('MKTEMPDIRCMD', localdata, 1) or "false")
-        tmpfile = tmppipe.readline().strip()
-        if not tmpfile:
-            logger.error("Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.")
-            raise FetchError(module)
-
-        if "label" in parm:
-            depot = "%s@%s" % (depot, parm["label"])
-        else:
-            cset = Perforce.getcset(d, depot, host, user, pswd, parm)
-            depot = "%s@%s" % (depot, cset)
-
-        os.chdir(tmpfile)
-        logger.info("Fetch " + loc)
-        logger.info("%s%s files %s", p4cmd, p4opt, depot)
-        p4file = os.popen("%s%s files %s" % (p4cmd, p4opt, depot))
-
-        if not p4file:
-            logger.error("Fetch: unable to get the P4 files from %s", depot)
-            raise FetchError(module)
-
-        count = 0
-
-        for file in p4file:
-            list = file.split()
-
-            if list[2] == "delete":
-                continue
-
-            dest = list[0][len(path)+1:]
-            where = dest.find("#")
-
-            os.system("%s%s print -o %s/%s %s" % (p4cmd, p4opt, module, dest[:where], list[0]))
-            count = count + 1
-
-        if count == 0:
-            logger.error("Fetch:  No files gathered from the P4 fetch")
-            raise FetchError(module)
-
-        myret = os.system("tar -czf %s %s" % (ud.localpath, module))
-        if myret != 0:
-            try:
-                os.unlink(ud.localpath)
-            except OSError:
-                pass
-            raise FetchError(module)
-        # cleanup
-        bb.utils.prunedir(tmpfile)

bitbake/lib/bb/fetch/repo.py

@@ -1,98 +0,0 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-"""
-BitBake "Fetch" repo (git) implementation
-
-"""
-
-# Copyright (C) 2009 Tom Rini <trini@embeddedalley.com>
-#
-# Based on git.py which is:
-#Copyright (C) 2005 Richard Purdie
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-import os
-import bb
-from   bb    import data
-from   bb.fetch import Fetch
-from   bb.fetch import runfetchcmd
-
-class Repo(Fetch):
-    """Class to fetch a module or modules from repo (git) repositories"""
-    def supports(self, url, ud, d):
-        """
-        Check to see if a given url can be fetched with repo.
-        """
-        return ud.type in ["repo"]
-
-    def localpath(self, url, ud, d):
-        """
-        We don"t care about the git rev of the manifests repository, but
-        we do care about the manifest to use.  The default is "default".
-        We also care about the branch or tag to be used.  The default is
-        "master".
-        """
-
-        ud.proto = ud.parm.get('protocol', 'git')
-        ud.branch = ud.parm.get('branch', 'master')
-        ud.manifest = ud.parm.get('manifest', 'default.xml')
-        if not ud.manifest.endswith('.xml'):
-            ud.manifest += '.xml'
-
-        ud.localfile = data.expand("repo_%s%s_%s_%s.tar.gz" % (ud.host, ud.path.replace("/", "."), ud.manifest, ud.branch), d)
-
-        return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
-
-    def go(self, loc, ud, d):
-        """Fetch url"""
-
-        if os.access(os.path.join(data.getVar("DL_DIR", d, True), ud.localfile), os.R_OK):
-            logger.debug(1, "%s already exists (or was stashed). Skipping repo init / sync.", ud.localpath)
-            return
-
-        gitsrcname = "%s%s" % (ud.host, ud.path.replace("/", "."))
-        repodir = data.getVar("REPODIR", d, True) or os.path.join(data.getVar("DL_DIR", d, True), "repo")
-        codir = os.path.join(repodir, gitsrcname, ud.manifest)
-
-        if ud.user:
-            username = ud.user + "@"
-        else:
-            username = ""
-
-        bb.utils.mkdirhier(os.path.join(codir, "repo"))
-        os.chdir(os.path.join(codir, "repo"))
-        if not os.path.exists(os.path.join(codir, "repo", ".repo")):
-            runfetchcmd("repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), d)
-
-        runfetchcmd("repo sync", d)
-        os.chdir(codir)
-
-        scmdata = ud.parm.get("scmdata", "")
-        if scmdata == "keep":
-            tar_flags = ""
-        else:
-            tar_flags = "--exclude '.repo' --exclude '.git'"
-
-        # Create a cache
-        runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.join(".", "*") ), d)
-
-    def supports_srcrev(self):
-        return False
-
-    def _build_revision(self, url, ud, d):
-        return ud.manifest
-
-    def _want_sortable_revision(self, url, ud, d):
-        return False

bitbake/lib/bb/fetch/ssh.py

@@ -1,118 +0,0 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-'''
-BitBake 'Fetch' implementations
-
-This implementation is for Secure Shell (SSH), and attempts to comply with the
-IETF secsh internet draft:
-    http://tools.ietf.org/wg/secsh/draft-ietf-secsh-scp-sftp-ssh-uri/
-
-    Currently does not support the sftp parameters, as this uses scp
-    Also does not support the 'fingerprint' connection parameter.
-
-'''
-
-# Copyright (C) 2006  OpenedHand Ltd.
-#
-#
-# Based in part on svk.py:
-#    Copyright (C) 2006 Holger Hans Peter Freyther
-#    Based on svn.py:
-#        Copyright (C) 2003, 2004  Chris Larson
-#        Based on functions from the base bb module:
-#            Copyright 2003 Holger Schurig
-#
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-import re, os
-from   bb import data
-from   bb.fetch import Fetch
-from   bb.fetch import FetchError
-
-
-__pattern__ = re.compile(r'''
- \s*                 # Skip leading whitespace
- ssh://              # scheme
- (                   # Optional username/password block
-  (?P<user>\S+)      # username
-  (:(?P<pass>\S+))?  # colon followed by the password (optional)
- )?
- (?P<cparam>(;[^;]+)*)?  # connection parameters block (optional)
- @
- (?P<host>\S+?)          # non-greedy match of the host
- (:(?P<port>[0-9]+))?    # colon followed by the port (optional)
- /
- (?P<path>[^;]+)         # path on the remote system, may be absolute or relative,
-                         # and may include the use of '~' to reference the remote home
-                         # directory
- (?P<sparam>(;[^;]+)*)?  # parameters block (optional)
- $
-''', re.VERBOSE)
-
-class SSH(Fetch):
-    '''Class to fetch a module or modules via Secure Shell'''
-
-    def supports(self, url, urldata, d):
-        return __pattern__.match(url) != None
-
-    def localpath(self, url, urldata, d):
-        m = __pattern__.match(url)
-        path = m.group('path')
-        host = m.group('host')
-        lpath = os.path.join(data.getVar('DL_DIR', d, True), host, os.path.basename(path))
-        return lpath
-
-    def go(self, url, urldata, d):
-        dldir = data.getVar('DL_DIR', d, 1)
-
-        m = __pattern__.match(url)
-        path = m.group('path')
-        host = m.group('host')
-        port = m.group('port')
-        user = m.group('user')
-        password = m.group('pass')
-
-        ldir = os.path.join(dldir, host)
-        lpath = os.path.join(ldir, os.path.basename(path))
-
-        if not os.path.exists(ldir):
-            os.makedirs(ldir)
-
-        if port:
-            port = '-P %s' % port
-        else:
-            port = ''
-
-        if user:
-            fr = user
-            if password:
-                fr += ':%s' % password
-            fr += '@%s' % host
-        else:
-            fr = host
-        fr += ':%s' % path
-
-
-        import commands
-        cmd = 'scp -B -r %s %s %s/' % (
-            port,
-            commands.mkarg(fr),
-            commands.mkarg(ldir)
-        )
-
-        (exitstatus, output) = commands.getstatusoutput(cmd)
-        if exitstatus != 0:
-            print(output)
-            raise FetchError('Unable to fetch %s' % url)

bitbake/lib/bb/fetch/svk.py

@@ -1,104 +0,0 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-"""
-BitBake 'Fetch' implementations
-
-This implementation is for svk. It is based on the svn implementation
-
-"""
-
-# Copyright (C) 2006 Holger Hans Peter Freyther
-# Copyright (C) 2003, 2004  Chris Larson
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Based on functions from the base bb module, Copyright 2003 Holger Schurig
-
-import os
-import logging
-import bb
-from   bb import data
-from   bb.fetch import Fetch
-from   bb.fetch import FetchError
-from   bb.fetch import MissingParameterError
-from   bb.fetch import logger
-
-class Svk(Fetch):
-    """Class to fetch a module or modules from svk repositories"""
-    def supports(self, url, ud, d):
-        """
-        Check to see if a given url can be fetched with svk.
-        """
-        return ud.type in ['svk']
-
-    def localpath(self, url, ud, d):
-        if not "module" in ud.parm:
-            raise MissingParameterError("svk method needs a 'module' parameter")
-        else:
-            ud.module = ud.parm["module"]
-
-        ud.revision = ud.parm.get('rev', "")
-
-        ud.localfile = data.expand('%s_%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision, ud.date), d)
-
-        return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
-
-    def forcefetch(self, url, ud, d):
-        return ud.date == "now"
-
-    def go(self, loc, ud, d):
-        """Fetch urls"""
-
-        svkroot = ud.host + ud.path
-
-        svkcmd = "svk co -r {%s} %s/%s" % (ud.date, svkroot, ud.module)
-
-        if ud.revision:
-            svkcmd = "svk co -r %s %s/%s" % (ud.revision, svkroot, ud.module)
-
-        # create temp directory
-        localdata = data.createCopy(d)
-        data.update_data(localdata)
-        logger.debug(2, "Fetch: creating temporary directory")
-        bb.utils.mkdirhier(data.expand('${WORKDIR}', localdata))
-        data.setVar('TMPBASE', data.expand('${WORKDIR}/oesvk.XXXXXX', localdata), localdata)
-        tmppipe = os.popen(data.getVar('MKTEMPDIRCMD', localdata, 1) or "false")
-        tmpfile = tmppipe.readline().strip()
-        if not tmpfile:
-            logger.error("Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.")
-            raise FetchError(ud.module)
-
-        # check out sources there
-        os.chdir(tmpfile)
-        logger.info("Fetch " + loc)
-        logger.debug(1, "Running %s", svkcmd)
-        myret = os.system(svkcmd)
-        if myret != 0:
-            try:
-                os.rmdir(tmpfile)
-            except OSError:
-                pass
-            raise FetchError(ud.module)
-
-        os.chdir(os.path.join(tmpfile, os.path.dirname(ud.module)))
-        # tar them up to a defined filename
-        myret = os.system("tar -czf %s %s" % (ud.localpath, os.path.basename(ud.module)))
-        if myret != 0:
-            try:
-                os.unlink(ud.localpath)
-            except OSError:
-                pass
-            raise FetchError(ud.module)
-        # cleanup
-        bb.utils.prunedir(tmpfile)

bitbake/lib/bb/fetch/svn.py

@@ -1,204 +0,0 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-"""
-BitBake 'Fetch' implementation for svn.
-
-"""
-
-# Copyright (C) 2003, 2004  Chris Larson
-# Copyright (C) 2004        Marcin Juszkiewicz
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Based on functions from the base bb module, Copyright 2003 Holger Schurig
-
-import os
-import sys
-import logging
-import bb
-from   bb import data
-from   bb.fetch import Fetch
-from   bb.fetch import FetchError
-from   bb.fetch import MissingParameterError
-from   bb.fetch import runfetchcmd
-from   bb.fetch import logger
-
-class Svn(Fetch):
-    """Class to fetch a module or modules from svn repositories"""
-    def supports(self, url, ud, d):
-        """
-        Check to see if a given url can be fetched with svn.
-        """
-        return ud.type in ['svn']
-
-    def localpath(self, url, ud, d):
-        if not "module" in ud.parm:
-            raise MissingParameterError("svn method needs a 'module' parameter")
-
-        ud.module = ud.parm["module"]
-
-        # Create paths to svn checkouts
-        relpath = self._strip_leading_slashes(ud.path)
-        ud.pkgdir = os.path.join(data.expand('${SVNDIR}', d), ud.host, relpath)
-        ud.moddir = os.path.join(ud.pkgdir, ud.module)
-
-        if 'rev' in ud.parm:
-            ud.date = ""
-            ud.revision = ud.parm['rev']
-        elif 'date' in ud.date:
-            ud.date = ud.parm['date']
-            ud.revision = ""
-        else:
-            #
-            # ***Nasty hack***
-            # If DATE in unexpanded PV, use ud.date (which is set from SRCDATE)
-            # Should warn people to switch to SRCREV here
-            #
-            pv = data.getVar("PV", d, 0)
-            if "DATE" in pv:
-                ud.revision = ""
-            else:
-                rev = Fetch.srcrev_internal_helper(ud, d)
-                if rev is True:
-                    ud.revision = self.latest_revision(url, ud, d)
-                    ud.date = ""
-                elif rev:
-                    ud.revision = rev
-                    ud.date = ""
-                else:
-                    ud.revision = ""
-
-        ud.localfile = data.expand('%s_%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision, ud.date), d)
-
-        return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
-
-    def _buildsvncommand(self, ud, d, command):
-        """
-        Build up an svn commandline based on ud
-        command is "fetch", "update", "info"
-        """
-
-        basecmd = data.expand('${FETCHCMD_svn}', d)
-
-        proto = ud.parm.get('proto', 'svn')
-
-        svn_rsh = None
-        if proto == "svn+ssh" and "rsh" in ud.parm:
-            svn_rsh = ud.parm["rsh"]
-
-        svnroot = ud.host + ud.path
-
-        # either use the revision, or SRCDATE in braces,
-        options = []
-
-        if ud.user:
-            options.append("--username %s" % ud.user)
-
-        if ud.pswd:
-            options.append("--password %s" % ud.pswd)
-
-        if command is "info":
-            svncmd = "%s info %s %s://%s/%s/" % (basecmd, " ".join(options), proto, svnroot, ud.module)
-        else:
-            suffix = ""
-            if ud.revision:
-                options.append("-r %s" % ud.revision)
-                suffix = "@%s" % (ud.revision)
-            elif ud.date:
-                options.append("-r {%s}" % ud.date)
-
-            if command is "fetch":
-                svncmd = "%s co %s %s://%s/%s%s %s" % (basecmd, " ".join(options), proto, svnroot, ud.module, suffix, ud.module)
-            elif command is "update":
-                svncmd = "%s update %s" % (basecmd, " ".join(options))
-            else:
-                raise FetchError("Invalid svn command %s" % command)
-
-        if svn_rsh:
-            svncmd = "svn_RSH=\"%s\" %s" % (svn_rsh, svncmd)
-
-        return svncmd
-
-    def go(self, loc, ud, d):
-        """Fetch url"""
-
-        logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'")
-
-        if os.access(os.path.join(ud.moddir, '.svn'), os.R_OK):
-            svnupdatecmd = self._buildsvncommand(ud, d, "update")
-            logger.info("Update " + loc)
-            # update sources there
-            os.chdir(ud.moddir)
-            logger.debug(1, "Running %s", svnupdatecmd)
-            runfetchcmd(svnupdatecmd, d)
-        else:
-            svnfetchcmd = self._buildsvncommand(ud, d, "fetch")
-            logger.info("Fetch " + loc)
-            # check out sources there
-            bb.utils.mkdirhier(ud.pkgdir)
-            os.chdir(ud.pkgdir)
-            logger.debug(1, "Running %s", svnfetchcmd)
-            runfetchcmd(svnfetchcmd, d)
-
-        scmdata = ud.parm.get("scmdata", "")
-        if scmdata == "keep":
-            tar_flags = ""
-        else:
-            tar_flags = "--exclude '.svn'"
-
-        os.chdir(ud.pkgdir)
-        # tar them up to a defined filename
-        try:
-            runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, ud.module), d)
-        except:
-            t, v, tb = sys.exc_info()
-            try:
-                os.unlink(ud.localpath)
-            except OSError:
-                pass
-            raise t, v, tb
-
-    def supports_srcrev(self):
-        return True
-
-    def _revision_key(self, url, ud, d):
-        """
-        Return a unique key for the url
-        """
-        return "svn:" + ud.moddir
-
-    def _latest_revision(self, url, ud, d):
-        """
-        Return the latest upstream revision number
-        """
-        logger.debug(2, "SVN fetcher hitting network for %s", url)
-
-        output = runfetchcmd("LANG=C LC_ALL=C " + self._buildsvncommand(ud, d, "info"), d, True)
-
-        revision = None
-        for line in output.splitlines():
-            if "Last Changed Rev" in line:
-                revision = line.split(":")[1].strip()
-
-        return revision
-
-    def _sortable_revision(self, url, ud, d):
-        """
-        Return a sortable revision number which in our case is the revision number
-        """
-
-        return self._build_revision(url, ud, d)
-
-    def _build_revision(self, url, ud, d):
-        return ud.revision

bitbake/lib/bb/fetch/wget.py

@@ -1,93 +0,0 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-"""
-BitBake 'Fetch' implementations
-
-Classes for obtaining upstream sources for the
-BitBake build tools.
-
-"""
-
-# Copyright (C) 2003, 2004  Chris Larson
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Based on functions from the base bb module, Copyright 2003 Holger Schurig
-
-import os
-import logging
-import bb
-import urllib
-from   bb import data
-from   bb.fetch import Fetch, FetchError, encodeurl, decodeurl, logger, runfetchcmd
-
-class Wget(Fetch):
-    """Class to fetch urls via 'wget'"""
-    def supports(self, url, ud, d):
-        """
-        Check to see if a given url can be fetched with wget.
-        """
-        return ud.type in ['http', 'https', 'ftp']
-
-    def localpath(self, url, ud, d):
-
-        url = encodeurl([ud.type, ud.host, ud.path, ud.user, ud.pswd, {}])
-        ud.basename = os.path.basename(ud.path)
-        ud.localfile = data.expand(urllib.unquote(ud.basename), d)
-
-        return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
-
-    def go(self, uri, ud, d, checkonly = False):
-        """Fetch urls"""
-
-        def fetch_uri(uri, ud, d):
-            if checkonly:
-                fetchcmd = data.getVar("CHECKCOMMAND", d, 1)
-            elif os.path.exists(ud.localpath):
-                # file exists, but we didnt complete it.. trying again..
-                fetchcmd = data.getVar("RESUMECOMMAND", d, 1)
-            else:
-                fetchcmd = data.getVar("FETCHCOMMAND", d, 1)
-
-            uri = uri.split(";")[0]
-            uri_decoded = list(decodeurl(uri))
-            uri_type = uri_decoded[0]
-            uri_host = uri_decoded[1]
-
-            fetchcmd = fetchcmd.replace("${URI}", uri.split(";")[0])
-            fetchcmd = fetchcmd.replace("${FILE}", ud.basename)
-            logger.info("fetch " + uri)
-            logger.debug(2, "executing " + fetchcmd)
-            runfetchcmd(fetchcmd, d)
-
-            # Sanity check since wget can pretend it succeed when it didn't
-            # Also, this used to happen if sourceforge sent us to the mirror page
-            if not os.path.exists(ud.localpath) and not checkonly:
-                logger.debug(2, "The fetch command for %s returned success but %s doesn't exist?...", uri, ud.localpath)
-                return False
-
-            return True
-
-        localdata = data.createCopy(d)
-        data.setVar('OVERRIDES', "wget:" + data.getVar('OVERRIDES', localdata), localdata)
-        data.update_data(localdata)
-
-        if fetch_uri(uri, ud, localdata):
-            return True
-
-        raise FetchError(uri)
-
-
-    def checkstatus(self, uri, ud, d):
-        return self.go(uri, ud, d, True)

bitbake/lib/bb/fetch2/__init__.py

@@ -55,7 +55,10 @@ class MalformedUrl(BBFetchException):
 class FetchError(BBFetchException):
     """General fetcher exception when something happens incorrectly"""
     def __init__(self, message, url = None):
-         msg = "Fetcher failure for URL: '%s'. %s" % (url, message)
+         if url:
+            msg = "Fetcher failure for URL: '%s'. %s" % (url, message)
+         else:
+            msg = "Fetcher failure: %s" % message
          self.url = url
          BBFetchException.__init__(self, msg)
          self.args = (message, url)
@@ -171,7 +174,8 @@ def encodeurl(decoded):
 
 def uri_replace(ud, uri_find, uri_replace, d):
     if not ud.url or not uri_find or not uri_replace:
-        logger.debug(1, "uri_replace: passed an undefined value, not replacing")
+        logger.error("uri_replace: passed an undefined value, not replacing")
+        return None
     uri_decoded = list(decodeurl(ud.url))
     uri_find_decoded = list(decodeurl(uri_find))
     uri_replace_decoded = list(decodeurl(uri_replace))
@@ -187,17 +191,20 @@ def uri_replace(ud, uri_find, uri_replace, d):
                 else:
                     result_decoded[loc] = re.sub(i, uri_replace_decoded[loc], uri_decoded[loc])
                 if uri_find_decoded.index(i) == 2:
+                    basename = None
                     if ud.mirrortarball:
-                        if result_decoded[loc].endswith("/"):
-                            result_decoded[loc] = os.path.dirname(result_decoded[loc])
-                        result_decoded[loc] = os.path.join(result_decoded[loc], os.path.basename(ud.mirrortarball))
+                        basename = os.path.basename(ud.mirrortarball)
                     elif ud.localpath:
-                        if result_decoded[loc].endswith("/"):
-                            result_decoded[loc] = os.path.dirname(result_decoded[loc])
-                        result_decoded[loc] = os.path.join(result_decoded[loc], os.path.basename(ud.localpath))
+                        basename = os.path.basename(ud.localpath)
+                    if basename and result_decoded[loc].endswith("/"):
+                        result_decoded[loc] = os.path.dirname(result_decoded[loc])
+                    if basename and not result_decoded[loc].endswith(basename):
+                        result_decoded[loc] = os.path.join(result_decoded[loc], basename)
             else:
-                return ud.url
+                return None
     result = encodeurl(result_decoded)
+    if result == ud.url:
+        return None
     logger.debug(2, "For url %s returning %s" % (ud.url, result))
     return result
 
@@ -302,10 +309,10 @@ def verify_checksum(u, ud, d):
     # it does not match.
     msg = ""
     if md5mismatch and ud.md5_expected:
-        msg = msg + "\nFile: '%s' has %s checksum %s when %s was expected (from URL: '%s')" % (ud.localpath, 'md5', md5data, ud.md5_expected, u)
+        msg = msg + "\nFile: '%s' has %s checksum %s when %s was expected" % (ud.localpath, 'md5', md5data, ud.md5_expected)
 
     if sha256mismatch and ud.sha256_expected:
-        msg = msg + "\nFile: '%s' has %s checksum %s when %s was expected (from URL: '%s')" % (ud.localpath, 'sha256', sha256data, ud.sha256_expected, u)
+        msg = msg + "\nFile: '%s' has %s checksum %s when %s was expected" % (ud.localpath, 'sha256', sha256data, ud.sha256_expected)
 
     if len(msg):
         raise FetchError('Checksum mismatch!%s' % msg, u)
@@ -389,6 +396,9 @@ def runfetchcmd(cmd, d, quiet = False, cleanup = []):
     Optionally remove the files/directories listed in cleanup upon failure
     """
 
+    import bb.process
+    import subprocess
+
     # Need to export PATH as binary could be in metadata paths
     # rather than host provided
     # Also include some other variables.
@@ -406,33 +416,27 @@ def runfetchcmd(cmd, d, quiet = False, cleanup = []):
 
     logger.debug(1, "Running %s", cmd)
 
-    # redirect stderr to stdout
-    stdout_handle = os.popen(cmd + " 2>&1", "r")
-    output = ""
+    success = False
+    error_message = ""
 
-    while True:
-        line = stdout_handle.readline()
-        if not line:
-            break
-        if not quiet:
-            print(line, end=' ')
-        output += line
+    try:
+        (output, errors) = bb.process.run(cmd, shell=True, stderr=subprocess.PIPE)
+        success = True
+    except bb.process.NotFoundError as e:
+        error_message = "Fetch command %s" % (e.command)
+    except bb.process.CmdError as e:
+        error_message = "Fetch command %s could not be run:\n%s" % (e.command, e.msg)
+    except bb.process.ExecutionError as e:
+        error_message = "Fetch command %s failed with exit code %s, output:\n%s" % (e.command, e.exitcode, e.stderr)
 
-    status = stdout_handle.close() or 0
-    signal = status >> 8
-    exitstatus = status & 0xff
-
-    if (signal or status != 0):
+    if not success:
         for f in cleanup:
             try:
                 bb.utils.remove(f, True)
             except OSError:
                 pass
 
-        if signal:
-            raise FetchError("Fetch command %s failed with signal %s, output:\n%s" % (cmd, signal, output))
-        elif status != 0:
-            raise FetchError("Fetch command %s failed with exit code %s, output:\n%s" % (cmd, status, output))
+        raise FetchError(error_message)
 
     return output
 
@@ -445,6 +449,90 @@ def check_network_access(d, info = "", url = None):
     else:
         logger.debug(1, "Fetcher accessed the network with the command %s" % info)
 
+def build_mirroruris(origud, mirrors, ld):
+    uris = []
+    uds = []
+
+    def adduri(uri, ud, uris, uds):
+        for line in mirrors:
+            try:
+                (find, replace) = line
+            except ValueError:
+                continue
+            newuri = uri_replace(ud, find, replace, ld)
+            if not newuri or newuri in uris or newuri == origud.url:
+                continue
+            try:
+                newud = FetchData(newuri, ld)
+                newud.setup_localpath(ld)
+            except bb.fetch2.BBFetchException as e:
+                logger.debug(1, "Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url))
+                logger.debug(1, str(e))
+                try:
+                    ud.method.clean(ud, ld)
+                except UnboundLocalError:
+                    pass
+                continue   
+            uris.append(newuri)
+            uds.append(newud)
+
+            adduri(newuri, newud, uris, uds)
+
+    adduri(None, origud, uris, uds)
+
+    return uris, uds
+
+def try_mirror_url(newuri, origud, ud, ld, check = False):
+    # Return of None or a value means we're finished
+    # False means try another url
+    try:
+        if check:
+            found = ud.method.checkstatus(newuri, ud, ld)
+            if found:
+                return found
+            return False
+
+        os.chdir(ld.getVar("DL_DIR", True))
+
+        if not os.path.exists(ud.donestamp) or ud.method.need_update(newuri, ud, ld):
+            ud.method.download(newuri, ud, ld)
+            if hasattr(ud.method,"build_mirror_data"):
+                ud.method.build_mirror_data(newuri, ud, ld)
+
+        if not ud.localpath or not os.path.exists(ud.localpath):
+            return False
+
+        if ud.localpath == origud.localpath:
+            return ud.localpath
+
+        # We may be obtaining a mirror tarball which needs further processing by the real fetcher
+        # If that tarball is a local file:// we need to provide a symlink to it
+        dldir = ld.getVar("DL_DIR", True)
+        if origud.mirrortarball and os.path.basename(ud.localpath) == os.path.basename(origud.mirrortarball) \
+                and os.path.basename(ud.localpath) != os.path.basename(origud.localpath):
+            open(ud.donestamp, 'w').close()
+            dest = os.path.join(dldir, os.path.basename(ud.localpath))
+            if not os.path.exists(dest):
+                os.symlink(ud.localpath, dest)
+            return None
+        # Otherwise the result is a local file:// and we symlink to it
+        if not os.path.exists(origud.localpath):
+             os.symlink(ud.localpath, origud.localpath)
+        update_stamp(newuri, origud, ld)
+        return ud.localpath
+
+    except bb.fetch2.NetworkAccess:
+        raise
+
+    except bb.fetch2.BBFetchException as e:
+        logger.debug(1, "Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url))
+        logger.debug(1, str(e))
+        try:
+            ud.method.clean(ud, ld)
+        except UnboundLocalError:
+            pass
+        return False
+
 def try_mirrors(d, origud, mirrors, check = False):
     """
     Try to use a mirrored version of the sources.
@@ -455,60 +543,13 @@ def try_mirrors(d, origud, mirrors, check = False):
     mirrors is the list of mirrors we're going to try
     """
     ld = d.createCopy()
-    for line in mirrors:
-        try:
-            (find, replace) = line
-        except ValueError:
-            continue
-        newuri = uri_replace(origud, find, replace, ld)
-        if newuri == origud.url:
-            continue
-        try:
-            ud = FetchData(newuri, ld)
-            ud.setup_localpath(ld)
 
-            if check:
-                found = ud.method.checkstatus(newuri, ud, ld)
-                if found:
-                    return found
-                continue
+    uris, uds = build_mirroruris(origud, mirrors, ld)
 
-            if ud.method.need_update(newuri, ud, ld):
-                ud.method.download(newuri, ud, ld)
-                if hasattr(ud.method,"build_mirror_data"):
-                    ud.method.build_mirror_data(newuri, ud, ld)
-
-            if not ud.localpath or not os.path.exists(ud.localpath):
-                continue
-
-            if ud.localpath == origud.localpath:
-                return ud.localpath
-
-            # We may be obtaining a mirror tarball which needs further processing by the real fetcher
-            # If that tarball is a local file:// we need to provide a symlink to it
-            dldir = ld.getVar("DL_DIR", True)
-            if os.path.basename(ud.localpath) != os.path.basename(origud.localpath):
-                dest = os.path.join(dldir, os.path.basename(ud.localpath))
-                if not os.path.exists(dest):
-                    os.symlink(ud.localpath, dest)
-                return None
-            # Otherwise the result is a local file:// and we symlink to it
-            if not os.path.exists(origud.localpath):
-                 os.symlink(ud.localpath, origud.localpath)
-            return ud.localpath
-
-        except bb.fetch2.NetworkAccess:
-            raise
-
-        except bb.fetch2.BBFetchException as e:
-            logger.debug(1, "Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url))
-            logger.debug(1, str(e))
-            try:
-                if os.path.isfile(ud.localpath):
-                    bb.utils.remove(ud.localpath)
-            except UnboundLocalError:
-                pass
-            continue
+    for index, uri in enumerate(uris):
+        ret = try_mirror_url(uri, origud, uds[index], ld, check)
+        if ret != False:
+            return ret
     return None
 
 def srcrev_internal_helper(ud, d, name):
@@ -898,14 +939,14 @@ class Fetch(object):
         self.ud = {}
 
         fn = d.getVar('FILE', True)
-        if cache and fn in urldata_cache:
+        if cache and fn and fn in urldata_cache:
             self.ud = urldata_cache[fn]
 
         for url in urls:
             if url not in self.ud:
                 self.ud[url] = FetchData(url, d)
 
-        if cache:
+        if fn and cache:
             urldata_cache[fn] = self.ud
 
     def localpath(self, url):
@@ -949,7 +990,7 @@ class Fetch(object):
             try:
                 self.d.setVar("BB_NO_NETWORK", network)
  
-                if not m.need_update(u, ud, self.d):
+                if os.path.exists(ud.donestamp) and not m.need_update(u, ud, self.d):
                     localpath = ud.localpath
                 elif m.try_premirror(u, ud, self.d):
                     logger.debug(1, "Trying PREMIRRORS")
@@ -959,7 +1000,10 @@ class Fetch(object):
                 if premirroronly:
                     self.d.setVar("BB_NO_NETWORK", "1")
 
-                if not localpath and m.need_update(u, ud, self.d):
+                os.chdir(self.d.getVar("DL_DIR", True))
+
+                firsterr = None
+                if not localpath and ((not os.path.exists(ud.donestamp)) or m.need_update(u, ud, self.d)):
                     try:
                         logger.debug(1, "Trying Upstream")
                         m.download(u, ud, self.d)
@@ -974,7 +1018,9 @@ class Fetch(object):
                         raise
 
                     except BBFetchException as e:
-                        logger.warn(str(e))
+                        logger.warn('Failed to fetch URL %s' % u)
+                        logger.debug(1, str(e))
+                        firsterr = e
                         # Remove any incomplete fetch
                         if os.path.isfile(ud.localpath):
                             bb.utils.remove(ud.localpath)
@@ -983,7 +1029,9 @@ class Fetch(object):
                         localpath = try_mirrors (self.d, ud, mirrors)
 
                 if not localpath or ((not os.path.exists(localpath)) and localpath.find("*") == -1):
-                    raise FetchError("Unable to fetch URL %s from any source." % u, u)
+                    if firsterr:
+                        logger.error(str(firsterr))
+                    raise FetchError("Unable to fetch URL from any source.", u)
 
                 update_stamp(u, ud, self.d)
 
@@ -1013,7 +1061,7 @@ class Fetch(object):
                 except:
                     # Finally, try checking uri, u, from MIRRORS
                     mirrors = mirror_from_string(self.d.getVar('MIRRORS', True))
-                    ret = try_mirrors (self.d, ud, mirrors, True)
+                    ret = try_mirrors(self.d, ud, mirrors, True)
 
             if not ret:
                 raise FetchError("URL %s doesn't work" % u, u)

bitbake/lib/bb/fetch2/git.py

@@ -38,6 +38,12 @@ Supported SRC_URI options are:
    who has its own routine to checkout code.
    The default is "0", set nocheckout=1 if needed.
 
+- bareclone
+   Create a bare clone of the source code and don't checkout the source code
+   when unpacking. Set this option for the recipe who has its own routine to
+   checkout code and tracking branch requirements.
+   The default is "0", set bareclone=1 if needed.
+
 """
 
 #Copyright (C) 2005 Richard Purdie
@@ -95,6 +101,11 @@ class Git(FetchMethod):
 
         ud.rebaseable = ud.parm.get("rebaseable","0") == "1"
 
+        # bareclone implies nocheckout
+        ud.bareclone = ud.parm.get("bareclone","0") == "1"
+        if ud.bareclone:
+            ud.nocheckout = 1
+  
         branches = ud.parm.get("branch", "master").split(',')
         if len(branches) != len(ud.names):
             raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url)
@@ -203,6 +214,7 @@ class Git(FetchMethod):
             os.chdir(ud.clonedir)
             logger.info("Creating tarball of git repository")
             runfetchcmd("tar -czf %s %s" % (ud.fullmirror, os.path.join(".") ), d)
+            runfetchcmd("touch %s.done" % (ud.fullmirror), d)
 
     def unpack(self, ud, destdir, d):
         """ unpack the downloaded src to destdir"""
@@ -220,7 +232,11 @@ class Git(FetchMethod):
         if os.path.exists(destdir):
             bb.utils.prunedir(destdir)
 
-        runfetchcmd("git clone -s -n %s %s" % (ud.clonedir, destdir), d)
+        cloneflags = "-s -n"
+        if ud.bareclone:
+            cloneflags += " --mirror"
+
+        runfetchcmd("git clone %s %s/ %s" % (cloneflags, ud.clonedir, destdir), d)
         if not ud.nocheckout:
             os.chdir(destdir)
             if subdir != "":

bitbake/lib/bb/monitordisk.py

@@ -0,0 +1,244 @@
+#!/usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# Copyright (C) 2012 Robert Yang
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import os, logging, re, sys
+import bb
+logger = logging.getLogger("BitBake.Monitor")
+
+def printErr(info):
+    logger.error("%s\n       Disk space monitor will NOT be enabled" % info)
+
+def convertGMK(unit):
+
+    """ Convert the space unit G, M, K, the unit is case-insensitive """
+
+    unitG = re.match('([1-9][0-9]*)[gG]\s?$', unit)
+    if unitG:
+        return int(unitG.group(1)) * (1024 ** 3)
+    unitM = re.match('([1-9][0-9]*)[mM]\s?$', unit)
+    if unitM:
+        return int(unitM.group(1)) * (1024 ** 2)
+    unitK = re.match('([1-9][0-9]*)[kK]\s?$', unit)
+    if unitK:
+        return int(unitK.group(1)) * 1024
+    unitN = re.match('([1-9][0-9]*)\s?$', unit)
+    if unitN:
+        return int(unitN.group(1))
+    else:
+        return None
+
+def getMountedDev(path):
+
+    """ Get the device mounted at the path, uses /proc/mounts """
+
+    # Get the mount point of the filesystem containing path
+    # st_dev is the ID of device containing file
+    parentDev = os.stat(path).st_dev
+    currentDev = parentDev
+    # When the current directory's device is different from the
+    # parrent's, then the current directory is a mount point
+    while parentDev == currentDev:
+        mountPoint = path
+        # Use dirname to get the parrent's directory
+        path = os.path.dirname(path)
+        # Reach the "/"
+        if path == mountPoint:
+            break
+        parentDev= os.stat(path).st_dev
+
+    try:
+        with open("/proc/mounts", "r") as ifp:
+            for line in ifp:
+                procLines = line.rstrip('\n').split()
+                if procLines[1] == mountPoint:
+                    return procLines[0]
+    except EnvironmentError:
+        pass
+    return None
+
+def getDiskData(BBDirs, configuration):
+
+    """Prepare disk data for disk space monitor"""
+
+    # Save the device IDs, need the ID to be unique (the dictionary's key is
+    # unique), so that when more than one directories are located in the same
+    # device, we just monitor it once
+    devDict = {}
+    for pathSpaceInode in BBDirs.split():
+        # The input format is: "dir,space,inode", dir is a must, space
+        # and inode are optional
+        pathSpaceInodeRe = re.match('([^,]*),([^,]*),([^,]*),?(.*)', pathSpaceInode)
+        if not pathSpaceInodeRe:
+            printErr("Invalid value in BB_DISKMON_DIRS: %s" % pathSpaceInode)
+            return None
+
+        action = pathSpaceInodeRe.group(1)
+        if action not in ("ABORT", "STOPTASKS", "WARN"):
+            printErr("Unknown disk space monitor action: %s" % action)
+            return None
+
+        path = os.path.realpath(pathSpaceInodeRe.group(2))
+        if not path:
+            printErr("Invalid path value in BB_DISKMON_DIRS: %s" % pathSpaceInode)
+            return None
+
+        # The disk space or inode is optional, but it should have a correct
+        # value once it is specified
+        minSpace = pathSpaceInodeRe.group(3)
+        if minSpace:
+            minSpace = convertGMK(minSpace)
+            if not minSpace:
+                printErr("Invalid disk space value in BB_DISKMON_DIRS: %s" % pathSpaceInodeRe.group(3))
+                return None
+        else:
+            # 0 means that it is not specified
+            minSpace = None
+
+        minInode = pathSpaceInodeRe.group(4)
+        if minInode:
+            minInode = convertGMK(minInode)
+            if not minInode:
+                printErr("Invalid inode value in BB_DISKMON_DIRS: %s" % pathSpaceInodeRe.group(4))
+                return None
+        else:
+            # 0 means that it is not specified
+            minInode = None
+
+        if minSpace is None and minInode is None:
+            printErr("No disk space or inode value in found BB_DISKMON_DIRS: %s" % pathSpaceInode)
+            return None
+        # mkdir for the directory since it may not exist, for example the
+        # DL_DIR may not exist at the very beginning
+        if not os.path.exists(path):
+            bb.utils.mkdirhier(path)
+        mountedDev = getMountedDev(path)
+        devDict[mountedDev] = action, path, minSpace, minInode
+
+    return devDict
+
+def getInterval(configuration):
+
+    """ Get the disk space interval """
+
+    # The default value is 50M and 5K.
+    spaceDefault = 50 * 1024 * 1024
+    inodeDefault = 5 * 1024
+
+    interval = configuration.getVar("BB_DISKMON_WARNINTERVAL", True)
+    if not interval:
+        return spaceDefault, inodeDefault
+    else:
+        # The disk space or inode interval is optional, but it should
+        # have a correct value once it is specified
+        intervalRe = re.match('([^,]*),?\s*(.*)', interval)
+        if intervalRe:
+            intervalSpace = intervalRe.group(1)
+            if intervalSpace:
+                intervalSpace = convertGMK(intervalSpace)
+                if not intervalSpace:
+                    printErr("Invalid disk space interval value in BB_DISKMON_WARNINTERVAL: %s" % intervalRe.group(1))
+                    return None, None
+            else:
+                intervalSpace = spaceDefault
+            intervalInode = intervalRe.group(2)
+            if intervalInode:
+                intervalInode = convertGMK(intervalInode)
+                if not intervalInode:
+                    printErr("Invalid disk inode interval value in BB_DISKMON_WARNINTERVAL: %s" % intervalRe.group(2))
+                    return None, None
+            else:
+                intervalInode = inodeDefault
+            return intervalSpace, intervalInode
+        else:
+            printErr("Invalid interval value in BB_DISKMON_WARNINTERVAL: %s" % interval)
+            return None, None
+
+class diskMonitor:
+
+    """Prepare the disk space monitor data"""
+
+    def __init__(self, configuration):
+
+        self.enableMonitor = False
+
+        BBDirs = configuration.getVar("BB_DISKMON_DIRS", True) or None
+        if BBDirs:
+            self.devDict = getDiskData(BBDirs, configuration)
+            if self.devDict:
+                self.spaceInterval, self.inodeInterval = getInterval(configuration)
+                if self.spaceInterval and self.inodeInterval:
+                    self.enableMonitor = True
+                    # These are for saving the previous disk free space and inode, we
+                    # use them to avoid print too many warning messages
+                    self.preFreeS = {}
+                    self.preFreeI = {}
+                    # This is for STOPTASKS and ABORT, to avoid print the message repeatly
+                    # during waiting the tasks to finish
+                    self.checked = {}
+                    for dev in self.devDict:
+                        self.preFreeS[dev] = 0
+                        self.preFreeI[dev] = 0
+                        self.checked[dev] = False
+                    if self.spaceInterval is None and self.inodeInterval is None:
+                        self.enableMonitor = False
+
+    def check(self, rq):
+
+        """ Take action for the monitor """
+
+        if self.enableMonitor:
+            for dev in self.devDict:
+                st = os.statvfs(self.devDict[dev][1])
+
+                # The free space, float point number
+                freeSpace = st.f_bavail * st.f_frsize
+
+                if self.devDict[dev][2] and freeSpace < self.devDict[dev][2]:
+                    # Always show warning, the self.checked would always be False if the action is WARN
+                    if self.preFreeS[dev] == 0 or self.preFreeS[dev] - freeSpace > self.spaceInterval and not self.checked[dev]:
+                        logger.warn("The free space of %s is running low (%.3fGB left)" % (dev, freeSpace / 1024 / 1024 / 1024.0))
+                        self.preFreeS[dev] = freeSpace
+
+                    if self.devDict[dev][0] == "STOPTASKS" and not self.checked[dev]:
+                        logger.error("No new tasks can be excuted since the disk space monitor action is \"STOPTASKS\"!")
+                        self.checked[dev] = True
+                        rq.finish_runqueue(False)
+                    elif self.devDict[dev][0] == "ABORT" and not self.checked[dev]:
+                        logger.error("Immediately abort since the disk space monitor action is \"ABORT\"!")
+                        self.checked[dev] = True
+                        rq.finish_runqueue(True)
+
+                # The free inodes, float point number
+                freeInode = st.f_favail
+
+                if self.devDict[dev][3] and freeInode < self.devDict[dev][3]:
+                    # Always show warning, the self.checked would always be False if the action is WARN
+                    if self.preFreeI[dev] == 0 or self.preFreeI[dev] - freeInode > self.inodeInterval and not self.checked[dev]:
+                        logger.warn("The free inode of %s is running low (%.3fK left)" % (dev, freeInode / 1024.0))
+                        self.preFreeI[dev] = freeInode
+
+                    if self.devDict[dev][0]  == "STOPTASKS" and not self.checked[dev]:
+                        logger.error("No new tasks can be excuted since the disk space monitor action is \"STOPTASKS\"!")
+                        self.checked[dev] = True
+                        rq.finish_runqueue(False)
+                    elif self.devDict[dev][0]  == "ABORT" and not self.checked[dev]:
+                        logger.error("Immediately abort since the disk space monitor action is \"ABORT\"!")
+                        self.checked[dev] = True
+                        rq.finish_runqueue(True)
+        return

bitbake/lib/bb/msg.py

@@ -100,6 +100,7 @@ class BBLogFilter(object):
 
 loggerDefaultDebugLevel = 0
 loggerDefaultVerbose = False
+loggerVerboseLogs = False
 loggerDefaultDomains = []
 
 def init_msgconfig(verbose, debug, debug_domains = []):
@@ -108,6 +109,8 @@ def init_msgconfig(verbose, debug, debug_domains = []):
     """
     bb.msg.loggerDefaultDebugLevel = debug
     bb.msg.loggerDefaultVerbose = verbose
+    if verbose:
+        bb.msg.loggerVerboseLogs = True
     bb.msg.loggerDefaultDomains = debug_domains
 
 def addDefaultlogFilter(handler):

bitbake/lib/bb/parse/__init__.py

@@ -37,6 +37,17 @@ logger = logging.getLogger("BitBake.Parsing")
 
 class ParseError(Exception):
     """Exception raised when parsing fails"""
+    def __init__(self, msg, filename, lineno=0):
+        self.msg = msg
+        self.filename = filename
+        self.lineno = lineno
+        Exception.__init__(self, msg, filename, lineno)
+
+    def __str__(self):
+        if self.lineno:
+            return "ParseError at %s:%d: %s" % (self.filename, self.lineno, self.msg)
+        else:
+            return "ParseError in %s: %s" % (self.filename, self.msg)
 
 class SkipPackage(Exception):
     """Exception raised to skip this package"""
@@ -78,7 +89,7 @@ def handle(fn, data, include = 0):
     for h in handlers:
         if h['supports'](fn, data):
             return h['handle'](fn, data, include)
-    raise ParseError("%s is not a BitBake file" % fn)
+    raise ParseError("not a BitBake file", fn)
 
 def init(fn, data):
     for h in handlers:
@@ -111,7 +122,7 @@ def vars_from_file(mypkg, d):
     parts = myfile[0].split('_')
     __pkgsplit_cache__[mypkg] = parts
     if len(parts) > 3:
-        raise ParseError("Unable to generate default variables from the filename: %s (too many underscores)" % mypkg)
+        raise ParseError("Unable to generate default variables from filename (too many underscores)", mypkg)
     exp = 3 - len(parts)
     tmplist = []
     while exp != 0:
@@ -120,4 +131,13 @@ def vars_from_file(mypkg, d):
     parts.extend(tmplist)
     return parts
 
+def get_file_depends(d):
+    '''Return the dependent files'''
+    dep_files = []
+    depends = d.getVar('__depends', True) or set()
+    depends = depends.union(d.getVar('__base_depends', True) or set())
+    for (fn, _) in depends:
+        dep_files.append(os.path.abspath(fn))
+    return " ".join(dep_files)
+
 from bb.parse.parse_py import __version__, ConfHandler, BBHandler

bitbake/lib/bb/parse/ast.py

@@ -59,9 +59,9 @@ class IncludeNode(AstNode):
 
         # TODO: Cache those includes... maybe not here though
         if self.force:
-            bb.parse.ConfHandler.include(self.filename, s, data, "include required")
+            bb.parse.ConfHandler.include(self.filename, s, self.lineno, data, "include required")
         else:
-            bb.parse.ConfHandler.include(self.filename, s, data, False)
+            bb.parse.ConfHandler.include(self.filename, s, self.lineno, data, False)
 
 class ExportNode(AstNode):
     def __init__(self, filename, lineno, var):
@@ -267,7 +267,7 @@ class InheritNode(AstNode):
         self.classes = classes
 
     def eval(self, data):
-        bb.parse.BBHandler.inherit(self.classes, data)
+        bb.parse.BBHandler.inherit(self.classes, self.filename, self.lineno, data)
 
 def handleInclude(statements, filename, lineno, m, force):
     statements.append(IncludeNode(filename, lineno, m.group(1), force))
@@ -304,7 +304,7 @@ def handleBBHandlers(statements, filename, lineno, m):
 
 def handleInherit(statements, filename, lineno, m):
     classes = m.group(1)
-    statements.append(InheritNode(filename, lineno, classes.split()))
+    statements.append(InheritNode(filename, lineno, classes))
 
 def finalize(fn, d, variant = None):
     all_handlers = {}
@@ -328,6 +328,8 @@ def finalize(fn, d, variant = None):
 
     bb.parse.siggen.finalise(fn, d, variant)
 
+    d.setVar('BBINCLUDED', bb.parse.get_file_depends(d))
+
     bb.event.fire(bb.event.RecipeParsed(fn), d)
 
 def _create_variants(datastores, names, function):
@@ -450,7 +452,7 @@ def multi_finalize(fn, d):
                 d.setVar("BBEXTENDVARIANT", variantmap[name])
             else:
                 d.setVar("PN", "%s-%s" % (pn, name))
-            bb.parse.BBHandler.inherit([extendedmap[name]], d)
+            bb.parse.BBHandler.inherit(extendedmap[name], fn, 0, d)
 
         safe_d.setVar("BBCLASSEXTEND", extended)
         _create_variants(datastores, extendedmap.keys(), extendfunc)

bitbake/lib/bb/parse/parse_py/BBHandler.py

@@ -68,12 +68,10 @@ def supports(fn, d):
     """Return True if fn has a supported extension"""
     return os.path.splitext(fn)[-1] in [".bb", ".bbclass", ".inc"]
 
-def inherit(files, d):
+def inherit(files, fn, lineno, d):
     __inherit_cache = data.getVar('__inherit_cache', d) or []
-    fn = ""
-    lineno = 0
+    files = d.expand(files).split()
     for file in files:
-        file = data.expand(file, d)
         if not os.path.isabs(file) and not file.endswith(".bbclass"):
             file = os.path.join('classes', '%s.bbclass' % file)
 
@@ -81,7 +79,7 @@ def inherit(files, d):
             logger.log(logging.DEBUG -1, "BB %s:%d: inheriting %s", fn, lineno, file)
             __inherit_cache.append( file )
             data.setVar('__inherit_cache', __inherit_cache, d)
-            include(fn, file, d, "inherit")
+            include(fn, file, lineno, d, "inherit")
             __inherit_cache = data.getVar('__inherit_cache', d) or []
 
 def get_statements(filename, absolute_filename, base_name):

bitbake/lib/bb/parse/parse_py/ConfHandler.py

@@ -29,11 +29,10 @@ import logging
 import bb.utils
 from bb.parse import ParseError, resolve_file, ast, logger
 
-#__config_regexp__  = re.compile( r"(?P<exp>export\s*)?(?P<var>[a-zA-Z0-9\-_+.${}]+)\s*(?P<colon>:)?(?P<ques>\?)?=\s*(?P<apo>['\"]?)(?P<value>.*)(?P=apo)$")
-__config_regexp__  = re.compile( r"(?P<exp>export\s*)?(?P<var>[a-zA-Z0-9\-_+.${}/]+)(\[(?P<flag>[a-zA-Z0-9\-_+.]+)\])?\s*((?P<colon>:=)|(?P<lazyques>\?\?=)|(?P<ques>\?=)|(?P<append>\+=)|(?P<prepend>=\+)|(?P<predot>=\.)|(?P<postdot>\.=)|=)\s*(?P<apo>['\"]?)(?P<value>.*)(?P=apo)$")
+__config_regexp__  = re.compile( r"(?P<exp>export\s*)?(?P<var>[a-zA-Z0-9\-_+.${}/]+)(\[(?P<flag>[a-zA-Z0-9\-_+.]+)\])?\s*((?P<colon>:=)|(?P<lazyques>\?\?=)|(?P<ques>\?=)|(?P<append>\+=)|(?P<prepend>=\+)|(?P<predot>=\.)|(?P<postdot>\.=)|=)\s*(?P<apo>['\"])(?P<value>.*)(?P=apo)$")
 __include_regexp__ = re.compile( r"include\s+(.+)" )
 __require_regexp__ = re.compile( r"require\s+(.+)" )
-__export_regexp__ = re.compile( r"export\s+(.+)" )
+__export_regexp__ = re.compile( r"export\s+([a-zA-Z0-9\-_+.${}/]+)$" )
 
 def init(data):
     topdir = data.getVar('TOPDIR')
@@ -44,10 +43,11 @@ def init(data):
 def supports(fn, d):
     return fn[-5:] == ".conf"
 
-def include(oldfn, fn, data, error_out):
+def include(oldfn, fn, lineno, data, error_out):
     """
-    error_out If True a ParseError will be raised if the to be included
-    config-files could not be included.
+    error_out: A string indicating the verb (e.g. "include", "inherit") to be
+    used in a ParseError that will be raised if the file to be included could
+    not be included. Specify False to avoid raising an error in this case.
     """
     if oldfn == fn: # prevent infinite recursion
         return None
@@ -58,7 +58,7 @@ def include(oldfn, fn, data, error_out):
 
     if not os.path.isabs(fn):
         dname = os.path.dirname(oldfn)
-        bbpath = "%s:%s" % (dname, data.getVar("BBPATH", 1))
+        bbpath = "%s:%s" % (dname, data.getVar("BBPATH", True))
         abs_fn = bb.utils.which(bbpath, fn)
         if abs_fn:
             fn = abs_fn
@@ -68,9 +68,17 @@ def include(oldfn, fn, data, error_out):
         ret = handle(fn, data, True)
     except IOError:
         if error_out:
-            raise ParseError("Could not %(error_out)s file %(fn)s" % vars() )
+            raise ParseError("Could not %(error_out)s file %(fn)s" % vars(), oldfn, lineno)
         logger.debug(2, "CONF file '%s' not found", fn)
 
+# We have an issue where a UI might want to enforce particular settings such as
+# an empty DISTRO variable. If configuration files do something like assigning
+# a weak default, it turns out to be very difficult to filter out these changes,
+# particularly when the weak default might appear half way though parsing a chain 
+# of configuration files. We therefore let the UIs hook into configuration file
+# parsing. This turns out to be a hard problem to solve any other way.
+confFilters = []
+
 def handle(fn, data, include):
     init(data)
 
@@ -107,6 +115,9 @@ def handle(fn, data, include):
     if oldfile:
         data.setVar('FILE', oldfile)
 
+    for f in confFilters:
+        f(fn, data)
+
     return data
 
 def feeder(lineno, s, fn, statements):
@@ -131,7 +142,7 @@ def feeder(lineno, s, fn, statements):
         ast.handleExport(statements, fn, lineno, m)
         return
 
-    raise ParseError("%s:%d: unparsed line: '%s'" % (fn, lineno, s));
+    raise ParseError("unparsed line: '%s'" % s, fn, lineno);
 
 # Add us to the handlers list
 from bb.parse import handlers

bitbake/lib/bb/persist_data.py

@@ -41,15 +41,19 @@ if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
 
 logger = logging.getLogger("BitBake.PersistData")
 if hasattr(sqlite3, 'enable_shared_cache'):
-    sqlite3.enable_shared_cache(True)
+    try:
+        sqlite3.enable_shared_cache(True)
+    except sqlite3.OperationalError:
+        pass
 
 
 @total_ordering
 class SQLTable(collections.MutableMapping):
     """Object representing a table/domain in the database"""
-    def __init__(self, cursor, table):
-        self.cursor = cursor
+    def __init__(self, cachefile, table):
+        self.cachefile = cachefile
         self.table = table
+        self.cursor = connect(self.cachefile)
 
         self._execute("CREATE TABLE IF NOT EXISTS %s(key TEXT, value TEXT);"
                       % table)
@@ -63,6 +67,8 @@ class SQLTable(collections.MutableMapping):
             except sqlite3.OperationalError as exc:
                 if 'database is locked' in str(exc) and count < 500:
                     count = count + 1
+                    self.cursor.close()
+                    self.cursor = connect(self.cachefile)
                     continue
                 raise
 
@@ -188,7 +194,7 @@ class PersistData(object):
         del self.data[domain][key]
 
 def connect(database):
-    return sqlite3.connect(database, timeout=30, isolation_level=None)
+    return sqlite3.connect(database, timeout=5, isolation_level=None)
 
 def persist(domain, d):
     """Convenience factory for SQLTable objects based upon metadata"""
@@ -201,5 +207,4 @@ def persist(domain, d):
 
     bb.utils.mkdirhier(cachedir)
     cachefile = os.path.join(cachedir, "bb_persist_data.sqlite3")
-    connection = connect(cachefile)
-    return SQLTable(connection, domain)
+    return SQLTable(cachefile, domain)

bitbake/lib/bb/providers.py

@@ -24,6 +24,7 @@
 import re
 import logging
 from bb import data, utils
+from collections import defaultdict
 import bb
 
 logger = logging.getLogger("BitBake.Provider")
@@ -35,6 +36,41 @@ class NoRProvider(bb.BBHandledException):
     """Exception raised when no provider of a runtime dependency can be found"""
 
 
+def findProviders(cfgData, dataCache, pkg_pn = None):
+    """
+    Convenience function to get latest and preferred providers in pkg_pn
+    """
+
+    if not pkg_pn:
+        pkg_pn = dataCache.pkg_pn
+
+    # Need to ensure data store is expanded
+    localdata = data.createCopy(cfgData)
+    bb.data.update_data(localdata)
+    bb.data.expandKeys(localdata)
+
+    preferred_versions = {}
+    latest_versions = {}
+
+    for pn in pkg_pn:
+        (last_ver, last_file, pref_ver, pref_file) = findBestProvider(pn, localdata, dataCache, pkg_pn)
+        preferred_versions[pn] = (pref_ver, pref_file)
+        latest_versions[pn] = (last_ver, last_file)
+
+    return (latest_versions, preferred_versions)
+
+
+def allProviders(dataCache):
+    """
+    Find all providers for each pn
+    """
+    all_providers = defaultdict(list)
+    for (fn, pn) in dataCache.pkg_fn.items():
+        ver = dataCache.pkg_pepvpr[fn]
+        all_providers[pn].append((ver, fn))
+    return all_providers
+
+
 def sortPriorities(pn, dataCache, pkg_pn = None):
     """
     Reorder pkg_pn by file priority and default preference
@@ -248,7 +284,7 @@ def filterProviders(providers, item, cfgData, dataCache):
 
     eligible = _filterProviders(providers, item, cfgData, dataCache)
 
-    prefervar = cfgData.getVar('PREFERRED_PROVIDER_%s' % item, 1)
+    prefervar = cfgData.getVar('PREFERRED_PROVIDER_%s' % item, True)
     if prefervar:
         dataCache.preferred[item] = prefervar
 
@@ -286,7 +322,7 @@ def filterProvidersRunTime(providers, item, cfgData, dataCache):
         pn = dataCache.pkg_fn[p]
         provides = dataCache.pn_provides[pn]
         for provide in provides:
-            prefervar = cfgData.getVar('PREFERRED_PROVIDER_%s' % provide, 1)
+            prefervar = cfgData.getVar('PREFERRED_PROVIDER_%s' % provide, True)
             logger.debug(1, "checking PREFERRED_PROVIDER_%s (value %s) against %s", provide, prefervar, pns.keys())
             if prefervar in pns and pns[prefervar] not in preferred:
                 var = "PREFERRED_PROVIDER_%s = %s" % (provide, prefervar)

bitbake/lib/bb/runqueue.py

@@ -31,6 +31,7 @@ import fcntl
 import logging
 import bb
 from bb import msg, data, event
+from bb import monitordisk
 
 bblogger = logging.getLogger("BitBake")
 logger = logging.getLogger("BitBake.RunQueue")
@@ -187,9 +188,10 @@ class RunQueueData:
         self.taskData = taskData
         self.targets = targets
         self.rq = rq
+        self.warn_multi_bb = False
 
-        self.stampwhitelist = cfgData.getVar("BB_STAMP_WHITELIST", 1) or ""
-        self.multi_provider_whitelist = (cfgData.getVar("MULTI_PROVIDER_WHITELIST", 1) or "").split()
+        self.stampwhitelist = cfgData.getVar("BB_STAMP_WHITELIST", True) or ""
+        self.multi_provider_whitelist = (cfgData.getVar("MULTI_PROVIDER_WHITELIST", True) or "").split()
 
         self.reset()
 
@@ -470,7 +472,7 @@ class RunQueueData:
                             dep = taskData.fn_index[depdata]
                             taskid = taskData.gettask_id(dep, idependtask, False)
                             if taskid is None:
-                                bb.msg.fatal("RunQueue", "Task %s in %s depends upon nonexistant task %s in %s" % (taskData.tasks_name[task], fn, idependtask, dep))
+                                bb.msg.fatal("RunQueue", "Task %s in %s depends upon non-existent task %s in %s" % (taskData.tasks_name[task], fn, idependtask, dep))
                             depends.append(taskid)
                             if depdata != fnid:
                                 tdepends_fnid[fnid].add(taskid)
@@ -674,11 +676,14 @@ class RunQueueData:
                     prov_list[prov] = [fn]
                 elif fn not in prov_list[prov]:
                     prov_list[prov].append(fn)
-        error = False
         for prov in prov_list:
             if len(prov_list[prov]) > 1 and prov not in self.multi_provider_whitelist:
-                error = True
-                logger.error("Multiple .bb files are due to be built which each provide %s (%s).\n This usually means one provides something the other doesn't and should.", prov, " ".join(prov_list[prov]))
+                msg = "Multiple .bb files are due to be built which each provide %s (%s)." % (prov, " ".join(prov_list[prov]))
+                if self.warn_multi_bb:
+                    logger.warn(msg)
+                else:
+                    msg += "\n This usually means one provides something the other doesn't and should."
+                    logger.error(msg)
 
 
         # Create a whitelist usable by the stamp checks
@@ -700,6 +705,12 @@ class RunQueueData:
                 continue
             self.runq_setscene.append(task)
 
+        # Invalidate task if force mode active
+        if self.cooker.configuration.force:
+            for (fn, target) in self.target_pairs:
+                logger.verbose("Invalidate task %s, %s", target, fn)
+                bb.parse.siggen.invalidate_task(target, self.dataCache, fn)
+
         # Interate over the task list and call into the siggen code
         dealtwith = set()
         todeal = set(range(len(self.runq_fnid)))
@@ -726,12 +737,6 @@ class RunQueueData:
                 deps.append(depidentifier)
             self.hash_deps[identifier] = deps
 
-        # Remove stamps for targets if force mode active
-        if self.cooker.configuration.force:
-            for (fn, target) in self.target_pairs:
-                logger.verbose("Remove stamp %s, %s", target, fn)
-                bb.build.del_stamp(target, self.dataCache, fn)
-
         return len(self.runq_fnid)
 
     def dump_data(self, taskQueue):
@@ -771,6 +776,11 @@ class RunQueue:
 
         self.state = runQueuePrepare
 
+        # For disk space monitor
+        self.dm = monitordisk.diskMonitor(cfgData)
+
+        self.rqexe = None
+
     def check_stamps(self):
         unchecked = {}
         current = []
@@ -865,7 +875,7 @@ class RunQueue:
             bb.msg.fatal("RunQueue", "check_stamps fatal internal error")
         return current
 
-    def check_stamp_task(self, task, taskname = None):
+    def check_stamp_task(self, task, taskname = None, recurse = False):
         def get_timestamp(f):
             try:
                 if not os.access(f, os.F_OK):
@@ -920,7 +930,8 @@ class RunQueue:
                     if t1 < t2:
                         logger.debug(2, 'Stampfile %s < %s', stampfile, stampfile2)
                         iscurrent = False
-
+                    if recurse and iscurrent:
+                        iscurrent = self.check_stamp_task(dep, recurse=True)
         return iscurrent
 
     def execute_runqueue(self):
@@ -945,6 +956,9 @@ class RunQueue:
             else:
                 self.rqexe = RunQueueExecuteScenequeue(self)
 
+        if self.state in [runQueueSceneRun, runQueueRunning, runQueueCleanUp]:
+            self.dm.check(self)
+
         if self.state is runQueueSceneRun:
             retval = self.rqexe.execute()
 
@@ -959,6 +973,13 @@ class RunQueue:
         if self.state is runQueueCleanUp:
            self.rqexe.finish()
 
+        if self.state is runQueueComplete or self.state is runQueueFailed:
+            if self.rqexe.stats.failed:
+                logger.info("Tasks Summary: Attempted %d tasks of which %d didn't need to be rerun and %d failed.", self.rqexe.stats.completed + self.rqexe.stats.failed, self.rqexe.stats.skipped, self.rqexe.stats.failed)
+            else:
+                # Let's avoid the word "failed" if nothing actually did
+                logger.info("Tasks Summary: Attempted %d tasks of which %d didn't need to be rerun and all succeeded.", self.rqexe.stats.completed, self.rqexe.stats.skipped)
+
         if self.state is runQueueFailed:
             if not self.rqdata.taskData.tryaltconfigs:
                 raise bb.runqueue.TaskFailure(self.rqexe.failed_fnids)
@@ -968,7 +989,6 @@ class RunQueue:
 
         if self.state is runQueueComplete:
             # All done
-            logger.info("Tasks Summary: Attempted %d tasks of which %d didn't need to be rerun and %d failed.", self.rqexe.stats.completed, self.rqexe.stats.skipped, self.rqexe.stats.failed)
             return False
 
         if self.state is runQueueChildProcess:
@@ -979,6 +999,9 @@ class RunQueue:
         return retval
 
     def finish_runqueue(self, now = False):
+        if not self.rqexe:
+            return
+
         if now:
             self.rqexe.finish_now()
         else:
@@ -1007,8 +1030,8 @@ class RunQueueExecute:
         self.cfgData = rq.cfgData
         self.rqdata = rq.rqdata
 
-        self.number_tasks = int(self.cfgData.getVar("BB_NUMBER_THREADS", 1) or 1)
-        self.scheduler = self.cfgData.getVar("BB_SCHEDULER", 1) or "speed"
+        self.number_tasks = int(self.cfgData.getVar("BB_NUMBER_THREADS", True) or 1)
+        self.scheduler = self.cfgData.getVar("BB_SCHEDULER", True) or "speed"
 
         self.runq_buildable = []
         self.runq_running = []
@@ -1045,11 +1068,19 @@ class RunQueueExecute:
             for k, v in self.build_pids.iteritems():
                 try:
                     os.kill(-k, signal.SIGTERM)
+                    os.waitpid(-1, 0)
                 except:
                     pass
         for pipe in self.build_pipes:
             self.build_pipes[pipe].read()
 
+        if len(self.failed_fnids) != 0:
+            self.rq.state = runQueueFailed
+            return
+
+        self.rq.state = runQueueComplete
+        return
+
     def finish(self):
         self.rq.state = runQueueCleanUp
 
@@ -1429,18 +1460,20 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
             sq_revdeps.append(copy.copy(self.rqdata.runq_revdeps[task]))
             sq_revdeps_new.append(set())
             if (len(self.rqdata.runq_revdeps[task]) == 0) and task not in self.rqdata.runq_setscene:
-                endpoints[task] = None
+                endpoints[task] = set()
 
         for task in self.rqdata.runq_setscene:
             for dep in self.rqdata.runq_depends[task]:
-                    endpoints[dep] = task
+                    if dep not in endpoints:
+                        endpoints[dep] = set()
+                    endpoints[dep].add(task)
 
         def process_endpoints(endpoints):
             newendpoints = {}
             for point, task in endpoints.items():
                 tasks = set()
                 if task:
-                    tasks.add(task)
+                    tasks |= task
                 if sq_revdeps_new[point]:
                     tasks |= sq_revdeps_new[point]
                 sq_revdeps_new[point] = set()
@@ -1465,8 +1498,30 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
             elif len(sq_revdeps_new[task]) != 0:
                 bb.msg.fatal("RunQueue", "Something went badly wrong during scenequeue generation, aborting. Please report this problem.")
 
+        # Resolve setscene inter-task dependencies
+        # e.g. do_sometask_setscene[depends] = "targetname:do_someothertask_setscene"
+        # Note that anything explicitly depended upon will have its reverse dependencies removed to avoid circular dependencies
+        for task in self.rqdata.runq_setscene:
+                realid = self.rqdata.taskData.gettask_id(self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]], self.rqdata.runq_task[task] + "_setscene", False)
+                idepends = self.rqdata.taskData.tasks_idepends[realid]
+                for (depid, idependtask) in idepends:
+                    if depid not in self.rqdata.taskData.build_targets:
+                        continue
+
+                    depdata = self.rqdata.taskData.build_targets[depid][0]
+                    if depdata is None:
+                         continue
+                    dep = self.rqdata.taskData.fn_index[depdata]
+                    taskid = self.rqdata.get_task_id(self.rqdata.taskData.getfn_id(dep), idependtask.replace("_setscene", ""))
+                    if taskid is None:
+                        bb.msg.fatal("RunQueue", "Task %s depends upon non-existent task %s:%s" % (self.rqdata.taskData.tasks_name[realid], dep, idependtask))
+
+                    sq_revdeps_squash[self.rqdata.runq_setscene.index(task)].add(self.rqdata.runq_setscene.index(taskid))
+                    # Have to zero this to avoid circular dependencies
+                    sq_revdeps_squash[self.rqdata.runq_setscene.index(taskid)] = set()
+
         #for task in xrange(len(sq_revdeps_squash)):
-        #    print "Task %s: %s.%s is %s " % (task, self.taskData.fn_index[self.runq_fnid[self.runq_setscene[task]]], self.runq_task[self.runq_setscene[task]] + "_setscene", sq_revdeps_squash[task])
+        #    print "Task %s: %s.%s is %s " % (task, self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[self.rqdata.runq_setscene[task]]], self.rqdata.runq_task[self.rqdata.runq_setscene[task]] + "_setscene", sq_revdeps_squash[task])
 
         self.sq_deps = []
         self.sq_revdeps = sq_revdeps_squash
@@ -1558,8 +1613,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
 
     def task_fail(self, task, result):
         self.stats.taskFailed()
-        index = self.rqdata.runq_setscene[task]
-        bb.event.fire(sceneQueueTaskFailed(index, self.stats, result, self), self.cfgData)
+        bb.event.fire(sceneQueueTaskFailed(task, self.stats, result, self), self.cfgData)
         self.scenequeue_notcovered.add(task)
         self.scenequeue_updatecounters(task)
 
@@ -1596,7 +1650,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
             fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[realtask]]
 
             taskname = self.rqdata.runq_task[realtask] + "_setscene"
-            if self.rq.check_stamp_task(realtask, self.rqdata.runq_task[realtask]):
+            if self.rq.check_stamp_task(realtask, self.rqdata.runq_task[realtask], recurse = True):
                 logger.debug(2, 'Stamp for underlying task %s(%s) is current, so skipping setscene variant',
                              task, self.rqdata.get_user_idstring(realtask))
                 self.task_failoutright(task)
@@ -1614,8 +1668,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
                 self.task_skip(task)
                 return True
 
-            logger.info("Running setscene task %d of %d (%s:%s)" % (self.stats.completed + self.stats.active + self.stats.failed + 1,
-                                                                         self.stats.total, fn, taskname))
+            startevent = sceneQueueTaskStarted(task, self.stats, self.rq)
+            bb.event.fire(startevent, self.cfgData)
 
             pid, pipein, pipeout = self.fork_off_task(fn, realtask, taskname)
 
@@ -1676,6 +1730,15 @@ class runQueueEvent(bb.event.Event):
         self.stats = stats.copy()
         bb.event.Event.__init__(self)
 
+class sceneQueueEvent(runQueueEvent):
+    """
+    Base sceneQueue event class
+    """
+    def __init__(self, task, stats, rq, noexec=False):
+        runQueueEvent.__init__(self, task, stats, rq)
+        realtask = rq.rqdata.runq_setscene[task]
+        self.taskstring = rq.rqdata.get_user_idstring(realtask, "_setscene")
+
 class runQueueTaskStarted(runQueueEvent):
     """
     Event notifing a task was started
@@ -1684,6 +1747,14 @@ class runQueueTaskStarted(runQueueEvent):
         runQueueEvent.__init__(self, task, stats, rq)
         self.noexec = noexec
 
+class sceneQueueTaskStarted(sceneQueueEvent):
+    """
+    Event notifing a setscene task was started
+    """
+    def __init__(self, task, stats, rq, noexec=False):
+        sceneQueueEvent.__init__(self, task, stats, rq)
+        self.noexec = noexec
+
 class runQueueTaskFailed(runQueueEvent):
     """
     Event notifing a task failed
@@ -1692,13 +1763,13 @@ class runQueueTaskFailed(runQueueEvent):
         runQueueEvent.__init__(self, task, stats, rq)
         self.exitcode = exitcode
 
-class sceneQueueTaskFailed(runQueueTaskFailed):
+class sceneQueueTaskFailed(sceneQueueEvent):
     """
     Event notifing a setscene task failed
     """
     def __init__(self, task, stats, exitcode, rq):
-        runQueueTaskFailed.__init__(self, task, stats, exitcode, rq)
-        self.taskstring = rq.rqdata.get_user_idstring(task, "_setscene")
+        sceneQueueEvent.__init__(self, task, stats, rq)
+        self.exitcode = exitcode
 
 class runQueueTaskCompleted(runQueueEvent):
     """

bitbake/lib/bb/server/process.py

@@ -48,7 +48,7 @@ class ServerCommunicator():
                 if self.connection.poll(.5):
                     return self.connection.recv()
                 else:
-                    return None
+                    return None, "Timeout while attempting to communicate with bitbake server"
             except KeyboardInterrupt:
                 pass
 

bitbake/lib/bb/server/xmlrpc.py

@@ -163,7 +163,7 @@ class BitBakeXMLRPCServer(SimpleXMLRPCServer):
     # remove this when you're done with debugging
     # allow_reuse_address = True
 
-    def __init__(self, interface = ("localhost", 0)):
+    def __init__(self, interface):
         """
         Constructor
         """
@@ -247,9 +247,9 @@ class BitbakeServerInfo():
         self.port = port
 
 class BitBakeServerConnection():
-    def __init__(self, serverinfo):
+    def __init__(self, serverinfo, clientinfo=("localhost", 0)):
         self.connection = _create_server(serverinfo.host, serverinfo.port)
-        self.events = uievent.BBUIEventQueue(self.connection)
+        self.events = uievent.BBUIEventQueue(self.connection, clientinfo)
         for event in bb.event.ui_queue:
             self.events.queue_event(event)
 
@@ -267,8 +267,8 @@ class BitBakeServerConnection():
             pass
 
 class BitBakeServer(object):
-    def initServer(self):
-        self.server = BitBakeXMLRPCServer()
+    def initServer(self, interface = ("localhost", 0)):
+        self.server = BitBakeXMLRPCServer(interface)
 
     def addcooker(self, cooker):
         self.cooker = cooker

bitbake/lib/bb/siggen.py

@@ -50,6 +50,10 @@ class SignatureGenerator(object):
     def dump_sigtask(self, fn, task, stampbase, runtime):
         return
 
+    def invalidate_task(self, task, d, fn):
+        bb.build.del_stamp(task, d, fn)
+
+
 class SignatureGeneratorBasic(SignatureGenerator):
     """
     """
@@ -62,9 +66,13 @@ class SignatureGeneratorBasic(SignatureGenerator):
         self.runtaskdeps = {}
         self.gendeps = {}
         self.lookupcache = {}
+        self.pkgnameextract = re.compile("(?P<fn>.*)\..*")
         self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST", True) or "").split())
-        self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST", True) or None
+        self.taskwhitelist = None
+        self.init_rundepcheck(data)
 
+    def init_rundepcheck(self, data):
+        self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST", True) or None
         if self.taskwhitelist:
             self.twl = re.compile(self.taskwhitelist)
         else:
@@ -122,7 +130,11 @@ class SignatureGeneratorBasic(SignatureGenerator):
         if variant:
             fn = "virtual:" + variant + ":" + fn
 
-        taskdeps = self._build_data(fn, d)
+        try:
+            taskdeps = self._build_data(fn, d)
+        except:
+            bb.note("Error during finalise of %s" % fn)
+            raise
 
         #Slow but can be useful for debugging mismatched basehashes
         #for task in self.taskdeps[fn]:
@@ -131,21 +143,42 @@ class SignatureGeneratorBasic(SignatureGenerator):
         for task in taskdeps:
             d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + "." + task])
 
+    def rundep_check(self, fn, recipename, task, dep, depname, dataCache):
+        # Return True if we should keep the dependency, False to drop it
+        # We only manipulate the dependencies for packages not in the whitelist
+        if self.twl and not self.twl.search(recipename):
+            # then process the actual dependencies
+            if self.twl.search(depname):
+                return False
+        return True
+
+    def read_taint(self, fn, task, stampbase):
+        taint = None
+        try:
+            with open(stampbase + '.' + task + '.taint', 'r') as taintf:
+                taint = taintf.read()
+        except IOError:
+            pass
+        return taint
+
     def get_taskhash(self, fn, task, deps, dataCache):
         k = fn + "." + task
         data = dataCache.basetaskhash[k]
         self.runtaskdeps[k] = []
+        recipename = dataCache.pkg_fn[fn]
         for dep in sorted(deps, key=clean_basepath):
-            # We only manipulate the dependencies for packages not in the whitelist
-            if self.twl and not self.twl.search(dataCache.pkg_fn[fn]):
-                # then process the actual dependencies
-                dep_fn = re.search("(?P<fn>.*)\..*", dep).group('fn')
-                if self.twl.search(dataCache.pkg_fn[dep_fn]):
-                    continue
+            depname = dataCache.pkg_fn[self.pkgnameextract.search(dep).group('fn')]
+            if not self.rundep_check(fn, recipename, task, dep, depname, dataCache):
+                continue
             if dep not in self.taskhash:
                 bb.fatal("%s is not in taskhash, caller isn't calling in dependency order?", dep)
             data = data + self.taskhash[dep]
             self.runtaskdeps[k].append(dep)
+
+        taint = self.read_taint(fn, task, dataCache.stamp[fn])
+        if taint:
+            data = data + taint
+
         h = hashlib.md5(data).hexdigest()
         self.taskhash[k] = h
         #d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task])
@@ -186,9 +219,14 @@ class SignatureGeneratorBasic(SignatureGenerator):
             for dep in data['runtaskdeps']:
                 data['runtaskhashes'][dep] = self.taskhash[dep]
 
+        taint = self.read_taint(fn, task, stampbase)
+        if taint:
+            data['taint'] = taint
+
         p = pickle.Pickler(file(sigfile, "wb"), -1)
         p.dump(data)
 
+
     def dump_sigs(self, dataCache):
         for fn in self.taskdeps:
             for task in self.taskdeps[fn]:
@@ -208,9 +246,17 @@ class SignatureGeneratorBasicHash(SignatureGeneratorBasic):
             k = fn + "." + taskname[:-9]
         else:
             k = fn + "." + taskname
-        h = self.taskhash[k]
+        if k in self.taskhash:
+            h = self.taskhash[k]
+        else:
+            # If k is not in basehash, then error
+            h = self.basehash[k]
         return ("%s.%s.%s.%s" % (stampbase, taskname, h, extrainfo)).rstrip('.')
 
+    def invalidate_task(self, task, d, fn):
+        bb.note("Tainting hash to force rebuild of task %s, %s" % (fn, task))
+        bb.build.write_taint(task, d, fn)
+
 def dump_this_task(outfile, d):
     import bb.parse
     fn = d.getVar("BB_FILENAME", True)
@@ -250,11 +296,13 @@ def compare_sigfiles(a, b):
 
     if 'basewhitelist' in a_data and a_data['basewhitelist'] != b_data['basewhitelist']:
         print "basewhitelist changed from %s to %s" % (a_data['basewhitelist'], b_data['basewhitelist'])
-        print "changed items: %s" % a_data['basewhitelist'].symmetric_difference(b_data['basewhitelist'])
+        if a_data['basewhitelist'] and b_data['basewhitelist']:
+            print "changed items: %s" % a_data['basewhitelist'].symmetric_difference(b_data['basewhitelist'])
 
     if 'taskwhitelist' in a_data and a_data['taskwhitelist'] != b_data['taskwhitelist']:
         print "taskwhitelist changed from %s to %s" % (a_data['taskwhitelist'], b_data['taskwhitelist'])
-        print "changed items: %s" % a_data['taskwhitelist'].symmetric_difference(b_data['taskwhitelist'])
+        if a_data['taskwhitelist'] and b_data['taskwhitelist']:
+            print "changed items: %s" % a_data['taskwhitelist'].symmetric_difference(b_data['taskwhitelist'])
 
     if a_data['taskdeps'] != b_data['taskdeps']:
         print "Task dependencies changed from:\n%s\nto:\n%s" % (sorted(a_data['taskdeps']), sorted(b_data['taskdeps']))
@@ -266,7 +314,8 @@ def compare_sigfiles(a, b):
     if changed:
         for dep in changed:
             print "List of dependencies for variable %s changed from %s to %s" % (dep, a_data['gendeps'][dep], b_data['gendeps'][dep])
-            print "changed items: %s" % a_data['gendeps'][dep].symmetric_difference(b_data['gendeps'][dep])
+            if a_data['gendeps'][dep] and b_data['gendeps'][dep]:
+                print "changed items: %s" % a_data['gendeps'][dep].symmetric_difference(b_data['gendeps'][dep])
     if added:
         for dep in added:
             print "Dependency on variable %s was added" % (dep)
@@ -286,16 +335,32 @@ def compare_sigfiles(a, b):
         changed, added, removed = dict_diff(a, b)
         if added:
             for dep in added:
-                print "Dependency on task %s was added" % (dep)
+	        bdep_found = False
+	        if removed:
+		    for bdep in removed:
+		    	if a[dep] == b[bdep]:
+			   #print "Dependency on task %s was replaced by %s with same hash" % (dep, bdep)
+			   bdep_found = True
+		if not bdep_found:
+                    print "Dependency on task %s was added with hash %s" % (dep, a[dep])
         if removed:
             for dep in removed:
-                print "Dependency on task %s was removed" % (dep)
+	        adep_found = False
+	        if added:
+		    for adep in added:
+		    	if a[adep] == b[dep]:
+			   #print "Dependency on task %s was replaced by %s with same hash" % (adep, dep)
+			   adep_found = True
+		if not adep_found:
+                   print "Dependency on task %s was removed with hash %s" % (dep, b[dep])
         if changed:
             for dep in changed:
                 print "Hash for dependent task %s changed from %s to %s" % (dep, a[dep], b[dep])
-    elif 'runtaskdeps' in a_data and 'runtaskdeps' in b_data and sorted(a_data['runtaskdeps']) != sorted(b_data['runtaskdeps']):
-        print "Tasks this task depends on changed from %s to %s" % (sorted(a_data['runtaskdeps']), sorted(b_data['runtaskdeps']))
-        print "changed items: %s" % a_data['runtaskdeps'].symmetric_difference(b_data['runtaskdeps'])
+
+    a_taint = a_data.get('taint', None)
+    b_taint = b_data.get('taint', None)
+    if a_taint != b_taint:
+        print "Taint (by forced/invalidated task) changed from %s to %s" % (a_taint, b_taint)
 
 def dump_sigfile(a):
     p1 = pickle.Unpickler(file(a, "rb"))
@@ -321,3 +386,6 @@ def dump_sigfile(a):
     if 'runtaskhashes' in a_data:
         for dep in a_data['runtaskhashes']:
             print "Hash for dependent task %s is %s" % (dep, a_data['runtaskhashes'][dep])
+
+    if 'taint' in a_data:
+        print "Tainted (by forced/invalidated task): %s" % a_data['taint']

bitbake/lib/bb/ui/crumbs/builddetailspage.py

@@ -0,0 +1,253 @@
+#!/usr/bin/env python
+#
+# BitBake Graphical GTK User Interface
+#
+# Copyright (C) 2012        Intel Corporation
+#
+# Authored by Dongxiao Xu <dongxiao.xu@intel.com>
+# Authored by Shane Wang <shane.wang@intel.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import gtk
+import pango
+import gobject
+from bb.ui.crumbs.progressbar import HobProgressBar
+from bb.ui.crumbs.hobwidget import hic, HobNotebook, HobAltButton, HobWarpCellRendererText
+from bb.ui.crumbs.runningbuild import RunningBuildTreeView
+from bb.ui.crumbs.runningbuild import BuildFailureTreeView
+from bb.ui.crumbs.hobpages import HobPage
+
+class BuildConfigurationTreeView(gtk.TreeView):
+    def __init__ (self):
+        gtk.TreeView.__init__(self)
+        self.set_rules_hint(False)
+        self.set_headers_visible(False)
+        self.set_property("hover-expand", True)
+        self.get_selection().set_mode(gtk.SELECTION_SINGLE)
+
+        # The icon that indicates whether we're building or failed.
+        renderer0 = gtk.CellRendererText()
+        renderer0.set_property('font-desc', pango.FontDescription('courier bold 12'))
+        col0 = gtk.TreeViewColumn ("Name", renderer0, text=0)
+        self.append_column (col0)
+
+        # The message of configuration.
+        renderer1 = HobWarpCellRendererText(col_number=1)
+        col1 = gtk.TreeViewColumn ("Values", renderer1, text=1)
+        self.append_column (col1)
+
+    def set_vars(self, key="", var=[""]):
+        d = {}
+        if type(var) == str:
+            d = {key: [var]}
+        elif type(var) == list and len(var) > 1:
+            #create the sub item line
+            l = []
+            text = ""
+            for item in var:
+                text = " - " + item
+                l.append(text)
+            d = {key: var}
+
+        return d
+
+    def set_config_model(self, show_vars):
+        listmodel = gtk.TreeStore(gobject.TYPE_STRING, gobject.TYPE_STRING)
+        parent = None
+        for var in show_vars:
+            for subitem in var.items():
+                name = subitem[0]
+                is_parent = True
+                for value in subitem[1]:
+                    if is_parent:
+                        parent = listmodel.append(parent, (name, value))
+                        is_parent = False
+                    else:
+                        listmodel.append(parent, (None, value))
+                    name = "     - "
+                parent = None
+        # renew the tree model after get the configuration messages
+        self.set_model(listmodel)
+
+    def show(self, src_config_info, src_params):
+        vars = []
+        vars.append(self.set_vars("BB version:", src_params.bb_version))
+        vars.append(self.set_vars("Target arch:", src_params.target_arch))
+        vars.append(self.set_vars("Target OS:", src_params.target_os))
+        vars.append(self.set_vars("Machine:", src_config_info.curr_mach))
+        vars.append(self.set_vars("Distro:", src_config_info.curr_distro))
+        vars.append(self.set_vars("Distro version:", src_params.distro_version))
+        vars.append(self.set_vars("SDK machine:", src_config_info.curr_sdk_machine))
+        vars.append(self.set_vars("Tune features:", src_params.tune_pkgarch))
+        vars.append(self.set_vars("Layers:", src_config_info.layers))
+
+        for path in src_config_info.layers:
+            import os, os.path
+            if os.path.exists(path):
+                f = os.popen('cd %s; git branch 2>&1 | grep "^* " | tr -d "* "' % path)
+                if f:
+                    branch = f.readline().lstrip('\n').rstrip('\n')
+                    vars.append(self.set_vars("Branch:", branch))
+                    f.close()
+                break
+
+        self.set_config_model(vars)
+
+    def reset(self):
+        self.set_model(None)
+
+#
+# BuildDetailsPage
+#
+
+class BuildDetailsPage (HobPage):
+
+    def __init__(self, builder):
+        super(BuildDetailsPage, self).__init__(builder, "Building ...")
+
+        self.num_of_issues = 0
+        self.endpath = (0,)
+        # create visual elements
+        self.create_visual_elements()
+
+    def create_visual_elements(self):
+        # create visual elements
+        self.vbox = gtk.VBox(False, 12)
+
+        self.progress_box = gtk.VBox(False, 12)
+        self.task_status = gtk.Label("\n") # to ensure layout is correct
+        self.task_status.set_alignment(0.0, 0.5)
+        self.progress_box.pack_start(self.task_status, expand=False, fill=False)
+        self.progress_hbox = gtk.HBox(False, 6)
+        self.progress_box.pack_end(self.progress_hbox, expand=True, fill=True)
+        self.progress_bar = HobProgressBar()
+        self.progress_hbox.pack_start(self.progress_bar, expand=True, fill=True)
+        self.stop_button = HobAltButton("Stop")
+        self.stop_button.connect("clicked", self.stop_button_clicked_cb)
+        self.stop_button.set_sensitive(False)
+        self.progress_hbox.pack_end(self.stop_button, expand=False, fill=False)
+
+        self.notebook = HobNotebook()
+        self.config_tv = BuildConfigurationTreeView()
+        self.scrolled_view_config = gtk.ScrolledWindow ()
+        self.scrolled_view_config.set_policy(gtk.POLICY_NEVER, gtk.POLICY_ALWAYS)
+        self.scrolled_view_config.add(self.config_tv)
+        self.notebook.append_page(self.scrolled_view_config, gtk.Label("Build configuration"))
+
+        self.failure_tv = BuildFailureTreeView()
+        self.failure_model = self.builder.handler.build.model.failure_model()
+        self.failure_tv.set_model(self.failure_model)
+        self.scrolled_view_failure = gtk.ScrolledWindow ()
+        self.scrolled_view_failure.set_policy(gtk.POLICY_NEVER, gtk.POLICY_ALWAYS)
+        self.scrolled_view_failure.add(self.failure_tv)
+        self.notebook.append_page(self.scrolled_view_failure, gtk.Label("Issues"))
+
+        self.build_tv = RunningBuildTreeView(readonly=True, hob=True)
+        self.build_tv.set_model(self.builder.handler.build.model)
+        self.scrolled_view_build = gtk.ScrolledWindow ()
+        self.scrolled_view_build.set_policy(gtk.POLICY_NEVER, gtk.POLICY_ALWAYS)
+        self.scrolled_view_build.add(self.build_tv)
+        self.notebook.append_page(self.scrolled_view_build, gtk.Label("Log"))
+
+        self.builder.handler.build.model.connect_after("row-changed", self.scroll_to_present_row, self.scrolled_view_build.get_vadjustment(), self.build_tv)
+
+        self.button_box = gtk.HBox(False, 6)
+        self.back_button = HobAltButton("<< Back to image configuration")
+        self.back_button.connect("clicked", self.back_button_clicked_cb)
+        self.button_box.pack_start(self.back_button, expand=False, fill=False)
+
+    def update_build_status(self, current, total, task):
+        recipe_path, recipe_task = task.split(", ")
+        recipe = os.path.basename(recipe_path).rstrip(".bb")
+        tsk_msg = "<b>Running task %s of %s:</b> %s\n<b>Recipe:</b> %s" % (current, total, recipe_task, recipe)
+        self.task_status.set_markup(tsk_msg)
+        self.stop_button.set_sensitive(True)
+
+    def reset_build_status(self):
+        self.task_status.set_markup("\n") # to ensure layout is correct
+        self.endpath = (0,)
+
+    def show_issues(self):
+        self.num_of_issues += 1
+        self.notebook.show_indicator_icon("Issues", self.num_of_issues)
+
+    def reset_issues(self):
+        self.num_of_issues = 0
+        self.notebook.hide_indicator_icon("Issues")
+
+    def _remove_all_widget(self):
+        children = self.vbox.get_children() or []
+        for child in children:
+            self.vbox.remove(child)
+        children = self.box_group_area.get_children() or []
+        for child in children:
+            self.box_group_area.remove(child)
+        children = self.get_children() or []
+        for child in children:
+            self.remove(child)
+
+    def show_page(self, step):
+        self._remove_all_widget()
+        if step == self.builder.PACKAGE_GENERATING or step == self.builder.FAST_IMAGE_GENERATING:
+            self.title = "Building packages ..."
+        else:
+            self.title = "Building image ..."
+        self.build_details_top = self.add_onto_top_bar(None)
+        self.pack_start(self.build_details_top, expand=False, fill=False)
+        self.pack_start(self.group_align, expand=True, fill=True)
+
+        self.box_group_area.pack_start(self.vbox, expand=True, fill=True)
+
+        self.progress_bar.reset()
+        self.config_tv.reset()
+        self.vbox.pack_start(self.progress_box, expand=False, fill=False)
+
+        self.vbox.pack_start(self.notebook, expand=True, fill=True)
+
+        self.box_group_area.pack_end(self.button_box, expand=False, fill=False)
+        self.show_all()
+        self.back_button.hide()
+
+        self.reset_build_status()
+        self.reset_issues()
+
+    def update_progress_bar(self, title, fraction, status=None):
+        self.progress_bar.update(fraction)
+        self.progress_bar.set_title(title)
+        self.progress_bar.set_rcstyle(status)
+
+    def back_button_clicked_cb(self, button):
+        self.builder.show_configuration()
+
+    def show_back_button(self):
+        self.back_button.show()
+
+    def stop_button_clicked_cb(self, button):
+        self.builder.stop_build()
+
+    def hide_stop_button(self):
+        self.stop_button.set_sensitive(False)
+        self.stop_button.hide()
+
+    def scroll_to_present_row(self, model, path, iter, v_adj, treeview):
+        if treeview and v_adj:
+            if path[0] > self.endpath[0]: # check the event is a new row append or not
+                self.endpath = path
+                # check the gtk.adjustment position is at end boundary or not
+                if (v_adj.upper <= v_adj.page_size) or (v_adj.value == v_adj.upper - v_adj.page_size):
+                    treeview.scroll_to_cell(path)
+
+    def show_configurations(self, configurations, params):
+        self.config_tv.show(configurations, params)

bitbake/lib/bb/ui/crumbs/configurator.py

@@ -1,346 +0,0 @@
-#
-# BitBake Graphical GTK User Interface
-#
-# Copyright (C) 2011        Intel Corporation
-#
-# Authored by Joshua Lock <josh@linux.intel.com>
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-import gobject
-import copy
-import re, os
-from bb import data
-
-class Configurator(gobject.GObject):
-
-    """
-    A GObject to handle writing modified configuration values back
-    to conf files.
-    """
-    __gsignals__ = {
-        "layers-loaded"  : (gobject.SIGNAL_RUN_LAST,
-                           gobject.TYPE_NONE,
-                           ()),
-        "layers-changed" : (gobject.SIGNAL_RUN_LAST,
-                            gobject.TYPE_NONE,
-                            ())
-    }
-
-    def __init__(self):
-        gobject.GObject.__init__(self)
-        self.bblayers = None
-        self.enabled_layers = {}
-        self.loaded_layers = {}
-        self.config = {}
-        self.orig_config = {}
-        self.preconf = None
-        self.postconf = None
-
-    # NOTE: cribbed from the cooker...
-    def _parse(self, f, data, include=False):
-        try:
-            return bb.parse.handle(f, data, include)
-        except (IOError, bb.parse.ParseError) as exc:
-            parselog.critical("Unable to parse %s: %s" % (f, exc))
-            sys.exit(1)
-
-    def _loadConf(self, path):
-        def getString(var):
-            return data.getVar(var, True) or ""
-
-        if self.orig_config:
-            del self.orig_config
-            self.orig_config = {}
-
-        data = bb.data.init()
-        data = self._parse(path, data)
-
-        # We only need to care about certain variables
-        mach = getString('MACHINE')
-        if mach and mach != self.config.get('MACHINE', ''):
-            self.config['MACHINE'] = mach
-        sdkmach = getString('SDKMACHINE')
-        if sdkmach and sdkmach != self.config.get('SDKMACHINE', ''):
-            self.config['SDKMACHINE'] = sdkmach
-        distro = getString('DISTRO')
-        if not distro:
-            distro = "defaultsetup"
-        if distro and distro != self.config.get('DISTRO', ''):
-            self.config['DISTRO'] = distro
-        bbnum = getString('BB_NUMBER_THREADS')
-        if bbnum and bbnum != self.config.get('BB_NUMBER_THREADS', ''):
-            self.config['BB_NUMBER_THREADS'] = bbnum
-        pmake = getString('PARALLEL_MAKE')
-        if pmake and pmake != self.config.get('PARALLEL_MAKE', ''):
-            self.config['PARALLEL_MAKE'] = pmake
-        pclass = getString('PACKAGE_CLASSES')
-        if pclass and pclass != self.config.get('PACKAGE_CLASSES', ''):
-            self.config['PACKAGE_CLASSES'] = pclass
-        fstypes = getString('IMAGE_FSTYPES')
-        if fstypes and fstypes != self.config.get('IMAGE_FSTYPES', ''):
-            self.config['IMAGE_FSTYPES'] = fstypes
-
-        # Values which aren't always set in the conf must be explicitly
-        # loaded as empty values for save to work
-        incompat = getString('INCOMPATIBLE_LICENSE')
-        if incompat and incompat != self.config.get('INCOMPATIBLE_LICENSE', ''):
-            self.config['INCOMPATIBLE_LICENSE'] = incompat
-        else:
-            self.config['INCOMPATIBLE_LICENSE'] = ""
-
-        # Non-standard, namespaces, variables for GUI preferences
-        toolchain = getString('HOB_BUILD_TOOLCHAIN')
-        if toolchain and toolchain != self.config.get('HOB_BUILD_TOOLCHAIN', ''):
-            self.config['HOB_BUILD_TOOLCHAIN'] = toolchain
-        header = getString('HOB_BUILD_TOOLCHAIN_HEADERS')
-        if header and header != self.config.get('HOB_BUILD_TOOLCHAIN_HEADERS', ''):
-            self.config['HOB_BUILD_TOOLCHAIN_HEADERS'] = header
-
-        self.orig_config = copy.deepcopy(self.config)
-
-    def setConfVar(self, var, val):
-        self.config[var] = val
-
-    def getConfVar(self, var):
-        if var in self.config:
-            return self.config[var]
-        else:
-            return ""
-
-    def _loadLayerConf(self, path):
-        self.bblayers = path
-        self.enabled_layers = {}
-        self.loaded_layers = {}
-        data = bb.data.init()
-        data = self._parse(self.bblayers, data)
-        layers = (data.getVar('BBLAYERS', True) or "").split()
-        for layer in layers:
-            # TODO: we may be better off calling the layer by its
-            # BBFILE_COLLECTIONS value?
-            name = self._getLayerName(layer)
-            self.loaded_layers[name] = layer
-
-        self.enabled_layers = copy.deepcopy(self.loaded_layers)
-        self.emit("layers-loaded")
-
-    def _addConfigFile(self, path):
-        conffiles = ["local.conf", "hob-pre.conf", "hob-post.conf"]
-        pref, sep, filename = path.rpartition("/")
-
-        if filename == "hob-pre.conf":
-            self.preconf = path
-
-        if filename == "hob-post.conf":
-            self.postconf = path
-
-        if filename in conffiles:
-            self._loadConf(path)
-        elif filename == "bblayers.conf":
-            self._loadLayerConf(path)
-
-    def _splitLayer(self, path):
-        # we only care about the path up to /conf/layer.conf
-        layerpath, conf, end = path.rpartition("/conf/")
-        return layerpath
-
-    def _getLayerName(self, path):
-        # Should this be the collection name?
-        layerpath, sep, name = path.rpartition("/")
-        return name
-
-    def disableLayer(self, layer):
-        if layer in self.enabled_layers:
-            del self.enabled_layers[layer]
-
-    def addLayerConf(self, confpath):
-        layerpath = self._splitLayer(confpath)
-        name = self._getLayerName(layerpath)
-
-        if not layerpath or not name:
-            return None, None
-        elif name not in self.enabled_layers:
-            self.addLayer(name, layerpath)
-            return name, layerpath
-        else:
-            return name, None
-
-    def addLayer(self, name, path):
-        self.enabled_layers[name] = path
-
-    def _isLayerConfDirty(self):
-        # if a different number of layers enabled to what was
-        # loaded, definitely different
-        if len(self.enabled_layers) != len(self.loaded_layers):
-            return True
-
-        for layer in self.loaded_layers:
-            # if layer loaded but no longer present, definitely dirty
-            if layer not in self.enabled_layers:
-                return True
-
-        for layer in self.enabled_layers:
-            # if this layer wasn't present at load, definitely dirty
-            if layer not in self.loaded_layers:
-                return True
-            # if this layers path has changed, definitely dirty
-            if self.enabled_layers[layer] != self.loaded_layers[layer]:
-                return True
-
-        return False
-
-    def _constructLayerEntry(self):
-        """
-        Returns a string representing the new layer selection
-        """
-        layers = self.enabled_layers.copy()
-        # Construct BBLAYERS entry
-        layer_entry = "BBLAYERS = \" \\\n"
-        if 'meta' in layers:
-            layer_entry = layer_entry + "  %s \\\n" % layers['meta']
-            del layers['meta']
-        for layer in layers:
-            layer_entry = layer_entry + "  %s \\\n" % layers[layer]
-        layer_entry = layer_entry + "  \""
-
-        return "".join(layer_entry)
-
-    def writeConfFile(self, conffile, contents):
-        """
-        Make a backup copy of conffile and write a new file in its stead with
-        the lines in the contents list.
-        """
-        # Create a backup of the conf file
-        bkup = "%s~" % conffile
-        os.rename(conffile, bkup)
-
-        # Write the contents list object to the conf file
-        with open(conffile, "w") as new:
-            new.write("".join(contents))
-
-    def updateConf(self, orig_lines, changed_values):
-        new_config_lines = []
-        for var in changed_values:
-            # Convenience function for re.subn(). If the pattern matches
-            # return a string which contains an assignment using the same
-            # assignment operator as the old assignment.
-            def replace_val(matchobj):
-                var = matchobj.group(1) # config variable
-                op = matchobj.group(2) # assignment operator
-                val = changed_values[var] # new config value
-                return "%s %s \"%s\"" % (var, op, val)
-
-            pattern = '^\s*(%s)\s*([+=?.]+)(.*)' % re.escape(var)
-            p = re.compile(pattern)
-            cnt = 0
-            replaced = False
-
-            # Iterate over the local.conf lines and if they are a match
-            # for the pattern comment out the line and append a new line
-            # with the new VAR op "value" entry
-            for line in orig_lines:
-                new_line, replacements = p.subn(replace_val, line)
-                if replacements:
-                    orig_lines[cnt] = "#%s" % line
-                    new_config_lines.append(new_line)
-                    replaced = True
-                cnt = cnt + 1
-
-            if not replaced:
-                new_config_lines.append("%s = \"%s\"\n" % (var, changed_values[var]))
-
-        # Add the modified variables
-        orig_lines.extend(new_config_lines)
-        return orig_lines
-
-    def writeConf(self):
-        pre_vars = ["MACHINE", "SDKMACHINE", "DISTRO",
-                    "INCOMPATIBLE_LICENSE"]
-        post_vars = ["BB_NUMBER_THREADS", "PARALLEL_MAKE", "PACKAGE_CLASSES",
-                     "IMAGE_FSTYPES", "HOB_BUILD_TOOLCHAIN",
-                     "HOB_BUILD_TOOLCHAIN_HEADERS"]
-        pre_values = {}
-        post_values = {}
-        changed_values = {}
-        pre_lines = None
-        post_lines = None
-
-        for var in self.config:
-            val = self.config[var]
-            if self.orig_config.get(var, None) != val:
-                changed_values[var] = val
-
-        if not len(changed_values):
-            return
-
-        for var in changed_values:
-            if var in pre_vars:
-                pre_values[var] = changed_values[var]
-            elif var in post_vars:
-                post_values[var] = changed_values[var]
-
-        with open(self.preconf, 'r') as pre:
-            pre_lines = pre.readlines()
-        pre_lines = self.updateConf(pre_lines, pre_values)
-        if len(pre_lines):
-            self.writeConfFile(self.preconf, pre_lines)
-
-        with open(self.postconf, 'r') as post:
-            post_lines = post.readlines()
-        post_lines = self.updateConf(post_lines, post_values)
-        if len(post_lines):
-            self.writeConfFile(self.postconf, post_lines)
-
-        del self.orig_config
-        self.orig_config = copy.deepcopy(self.config)
-
-    def insertTempBBPath(self, bbpath, bbfiles):
-        # read the original conf into a list
-        with open(self.postconf, 'r') as config:
-            config_lines = config.readlines()
-
-        if bbpath:
-            config_lines.append("BBPATH := \"${BBPATH}:%s\"\n" % bbpath)
-        if bbfiles:
-            config_lines.append("BBFILES := \"${BBFILES} %s\"\n" % bbfiles)
-
-        self.writeConfFile(self.postconf, config_lines)
-
-    def writeLayerConf(self):
-        # If we've not added/removed new layers don't write
-        if not self._isLayerConfDirty():
-            return
-
-        # This pattern should find the existing BBLAYERS
-        pattern = 'BBLAYERS\s=\s\".*\"'
-
-        replacement = self._constructLayerEntry()
-
-        with open(self.bblayers, "r") as f:
-            contents = f.read()
-            p = re.compile(pattern, re.DOTALL)
-            new = p.sub(replacement, contents)
-
-        self.writeConfFile(self.bblayers, new)
-
-        # set loaded_layers for dirtiness tracking
-        self.loaded_layers = copy.deepcopy(self.enabled_layers)
-
-        self.emit("layers-changed")
-
-    def configFound(self, handler, path):
-        self._addConfigFile(path)
-
-    def loadConfig(self, path):
-        self._addConfigFile(path)

bitbake/lib/bb/ui/crumbs/hobcolor.py

@@ -0,0 +1,37 @@
+#
+# BitBake Graphical GTK User Interface
+#
+# Copyright (C) 2012        Intel Corporation
+#
+# Authored by Shane Wang <shane.wang@intel.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+class HobColors:
+    WHITE        = "#ffffff"
+    PALE_GREEN   = "#aaffaa"
+    ORANGE       = "#eb8e68"
+    PALE_RED     = "#ffaaaa"
+    GRAY         = "#aaaaaa"
+    LIGHT_GRAY   = "#dddddd"
+    SLIGHT_DARK  = "#5f5f5f"
+    DARK         = "#3c3b37"
+    BLACK        = "#000000"
+    PALE_BLUE    = "#53b8ff"
+    DEEP_RED     = "#aa3e3e"
+
+    OK = WHITE
+    RUNNING = PALE_GREEN
+    WARNING = ORANGE
+    ERROR = PALE_RED

bitbake/lib/bb/ui/crumbs/hobeventhandler.py

@@ -4,6 +4,7 @@
 # Copyright (C) 2011        Intel Corporation
 #
 # Authored by Joshua Lock <josh@linux.intel.com>
+# Authored by Dongxiao Xu <dongxiao.xu@intel.com>
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -20,10 +21,8 @@
 
 import gobject
 import logging
-import tempfile
-import datetime
-
-progress_total = 0
+from bb.ui.crumbs.runningbuild import RunningBuild
+from bb.ui.crumbs.hobwidget import hcc
 
 class HobHandler(gobject.GObject):
 
@@ -31,147 +30,174 @@ class HobHandler(gobject.GObject):
     This object does BitBake event handling for the hob gui.
     """
     __gsignals__ = {
-         "machines-updated"    : (gobject.SIGNAL_RUN_LAST,
-                                  gobject.TYPE_NONE,
-                                  (gobject.TYPE_PYOBJECT,)),
-         "sdk-machines-updated": (gobject.SIGNAL_RUN_LAST,
-                                  gobject.TYPE_NONE,
-                                  (gobject.TYPE_PYOBJECT,)),
-         "distros-updated"     : (gobject.SIGNAL_RUN_LAST,
-                                  gobject.TYPE_NONE,
-                                  (gobject.TYPE_PYOBJECT,)),
-         "package-formats-found" : (gobject.SIGNAL_RUN_LAST,
-                                  gobject.TYPE_NONE,
-                                  (gobject.TYPE_PYOBJECT,)),
-         "config-found"        : (gobject.SIGNAL_RUN_LAST,
-                                  gobject.TYPE_NONE,
-                                  (gobject.TYPE_STRING,)),
-         "generating-data"     : (gobject.SIGNAL_RUN_LAST,
-                                  gobject.TYPE_NONE,
-                                  ()),
-         "data-generated"      : (gobject.SIGNAL_RUN_LAST,
-                                  gobject.TYPE_NONE,
-                                  ()),
-         "fatal-error"         : (gobject.SIGNAL_RUN_LAST,
-                                  gobject.TYPE_NONE,
-                                  (gobject.TYPE_STRING,
-                                   gobject.TYPE_STRING,)),
-         "command-failed"      : (gobject.SIGNAL_RUN_LAST,
-                                  gobject.TYPE_NONE,
-                                  (gobject.TYPE_STRING,)),
-         "reload-triggered"    : (gobject.SIGNAL_RUN_LAST,
-                                  gobject.TYPE_NONE,
-                                  (gobject.TYPE_STRING,
-                                   gobject.TYPE_STRING,)),
+         "package-formats-updated" : (gobject.SIGNAL_RUN_LAST,
+                                      gobject.TYPE_NONE,
+                                     (gobject.TYPE_PYOBJECT,)),
+         "config-updated"          : (gobject.SIGNAL_RUN_LAST,
+                                      gobject.TYPE_NONE,
+                                     (gobject.TYPE_STRING, gobject.TYPE_PYOBJECT,)),
+         "command-succeeded"       : (gobject.SIGNAL_RUN_LAST,
+                                      gobject.TYPE_NONE,
+                                     (gobject.TYPE_INT,)),
+         "command-failed"          : (gobject.SIGNAL_RUN_LAST,
+                                      gobject.TYPE_NONE,
+                                     (gobject.TYPE_STRING,)),
+         "sanity-failed"           : (gobject.SIGNAL_RUN_LAST,
+                                      gobject.TYPE_NONE,
+                                     (gobject.TYPE_STRING,)),
+         "generating-data"         : (gobject.SIGNAL_RUN_LAST,
+                                      gobject.TYPE_NONE,
+                                     ()),
+         "data-generated"          : (gobject.SIGNAL_RUN_LAST,
+                                      gobject.TYPE_NONE,
+                                     ()),
+         "parsing-started"         : (gobject.SIGNAL_RUN_LAST,
+                                      gobject.TYPE_NONE,
+                                     (gobject.TYPE_PYOBJECT,)),
+         "parsing"                 : (gobject.SIGNAL_RUN_LAST,
+                                      gobject.TYPE_NONE,
+                                     (gobject.TYPE_PYOBJECT,)),
+         "parsing-completed"       : (gobject.SIGNAL_RUN_LAST,
+                                      gobject.TYPE_NONE,
+                                     (gobject.TYPE_PYOBJECT,)),
+         "recipe-populated"        : (gobject.SIGNAL_RUN_LAST,
+                                      gobject.TYPE_NONE,
+                                     ()),
+         "package-populated"       : (gobject.SIGNAL_RUN_LAST,
+                                      gobject.TYPE_NONE,
+                                     ()),
     }
 
-    (CFG_PATH_LOCAL, CFG_PATH_PRE, CFG_PATH_POST, CFG_PATH_LAYERS, CFG_FILES_DISTRO, CFG_FILES_MACH, CFG_FILES_SDK, FILES_MATCH_CLASS, GENERATE_TGTS, REPARSE_FILES, BUILD_IMAGE) = range(11)
+    (GENERATE_CONFIGURATION, GENERATE_RECIPES, GENERATE_PACKAGES, GENERATE_IMAGE, POPULATE_PACKAGEINFO, SANITY_CHECK) = range(6)
+    (SUB_PATH_LAYERS, SUB_FILES_DISTRO, SUB_FILES_MACH, SUB_FILES_SDKMACH, SUB_MATCH_CLASS, SUB_PARSE_CONFIG, SUB_SANITY_CHECK, SUB_GNERATE_TGTS, SUB_GENERATE_PKGINFO, SUB_BUILD_RECIPES, SUB_BUILD_IMAGE) = range(11)
 
-    def __init__(self, taskmodel, server):
-        gobject.GObject.__init__(self)
+    def __init__(self, server, recipe_model, package_model):
+        super(HobHandler, self).__init__()
 
-        self.current_command = None
-        self.building = False
-        self.build_toolchain = False
-        self.build_toolchain_headers = False
+        self.build = RunningBuild(sequential=True)
+
+        self.recipe_model = recipe_model
+        self.package_model = package_model
+
+        self.commands_async = []
         self.generating = False
-        self.build_queue = []
         self.current_phase = None
-        self.bbpath_ok = False
-        self.bbfiles_ok = False
-        self.build_type = "image"
-        self.image_dir = os.path.join(tempfile.gettempdir(), 'hob-images')
+        self.building = False
+        self.recipe_queue = []
+        self.package_queue = []
 
-        self.model = taskmodel
         self.server = server
+        self.error_msg = ""
+        self.initcmd = None
 
-        deploy_dir = self.server.runCommand(["getVariable", "DEPLOY_DIR"])
-        self.image_out_dir = os.path.join(deploy_dir, "images")
-        self.image_output_types = self.server.runCommand(["getVariable", "IMAGE_FSTYPES"]).split(" ")
-        self.bbpath = self.server.runCommand(["getVariable", "BBPATH"])
-        self.bbfiles = self.server.runCommand(["getVariable", "BBFILES"])
-
-    def run_next_command(self):
-        if self.current_command and not self.generating:
+    def set_busy(self):
+        if not self.generating:
             self.emit("generating-data")
             self.generating = True
 
-        if self.current_command == self.CFG_PATH_LOCAL:
-            self.current_command = self.CFG_PATH_PRE
-            self.server.runCommand(["findConfigFilePath", "hob-pre.conf"])
-        elif self.current_command == self.CFG_PATH_PRE:
-            self.current_command = self.CFG_PATH_POST
-            self.server.runCommand(["findConfigFilePath", "hob-post.conf"])
-        elif self.current_command == self.CFG_PATH_POST:
-            self.current_command = self.CFG_PATH_LAYERS
-            self.server.runCommand(["findConfigFilePath", "bblayers.conf"])
-        elif self.current_command == self.CFG_PATH_LAYERS:
-            self.current_command = self.CFG_FILES_DISTRO
-            self.server.runCommand(["findConfigFiles", "DISTRO"])
-        elif self.current_command == self.CFG_FILES_DISTRO:
-            self.current_command = self.CFG_FILES_MACH
-            self.server.runCommand(["findConfigFiles", "MACHINE"])
-        elif self.current_command == self.CFG_FILES_MACH:
-            self.current_command = self.CFG_FILES_SDK
-            self.server.runCommand(["findConfigFiles", "MACHINE-SDK"])
-        elif self.current_command == self.CFG_FILES_SDK:
-            self.current_command = self.FILES_MATCH_CLASS
-            self.server.runCommand(["findFilesMatchingInDir", "rootfs_", "classes"])
-        elif self.current_command == self.FILES_MATCH_CLASS:
-            self.current_command = self.GENERATE_TGTS
-            self.server.runCommand(["generateTargetsTree", "classes/image.bbclass"])
-        elif self.current_command == self.GENERATE_TGTS:
-            if self.generating:
-                self.emit("data-generated")
-                self.generating = False
-            self.current_command = None
-        elif self.current_command == self.REPARSE_FILES:
-            if self.build_queue:
-                self.current_command = self.BUILD_IMAGE
-            else:
-                self.current_command = self.CFG_PATH_LAYERS
-            self.server.runCommand(["resetCooker"])
-            self.server.runCommand(["reparseFiles"])
-        elif self.current_command == self.BUILD_IMAGE:
-            if self.generating:
-                self.emit("data-generated")
-                self.generating = False
+    def clear_busy(self):
+        if self.generating:
+            self.emit("data-generated")
+            self.generating = False
+
+    def runCommand(self, commandline):
+        try:
+            result, error = self.server.runCommand(commandline)
+            if error:
+                raise Exception("Error running command '%s': %s" % (commandline, error))
+            return result
+        except Exception as e:
+            self.commands_async = []
+            self.clear_busy()
+            self.emit("command-failed", "Hob Exception - %s" % (str(e)))
+            return None
+
+    def run_next_command(self, initcmd=None):
+        if initcmd != None:
+            self.initcmd = initcmd
+
+        if self.commands_async:
+            self.set_busy()
+            next_command = self.commands_async.pop(0)
+        else:
+            self.clear_busy()
+            if self.initcmd != None:
+                self.emit("command-succeeded", self.initcmd)
+            return
+
+        if next_command == self.SUB_PATH_LAYERS:
+            self.runCommand(["findConfigFilePath", "bblayers.conf"])
+        elif next_command == self.SUB_FILES_DISTRO:
+            self.runCommand(["findConfigFiles", "DISTRO"])
+        elif next_command == self.SUB_FILES_MACH:
+            self.runCommand(["findConfigFiles", "MACHINE"])
+        elif next_command == self.SUB_FILES_SDKMACH:
+            self.runCommand(["findConfigFiles", "MACHINE-SDK"])
+        elif next_command == self.SUB_MATCH_CLASS:
+            self.runCommand(["findFilesMatchingInDir", "rootfs_", "classes"])
+        elif next_command == self.SUB_PARSE_CONFIG:
+            self.runCommand(["parseConfigurationFiles", "", ""])
+        elif next_command == self.SUB_GNERATE_TGTS:
+            self.runCommand(["generateTargetsTree", "classes/image.bbclass", []])
+        elif next_command == self.SUB_GENERATE_PKGINFO:
+            self.runCommand(["triggerEvent", "bb.event.RequestPackageInfo()"])
+        elif next_command == self.SUB_SANITY_CHECK:
+            self.runCommand(["triggerEvent", "bb.event.SanityCheck()"])
+        elif next_command == self.SUB_BUILD_RECIPES:
+            self.clear_busy()
             self.building = True
-            self.server.runCommand(["buildTargets", self.build_queue, "build"])
-            self.build_queue = []
-            self.current_command = None
+            self.runCommand(["buildTargets", self.recipe_queue, self.default_task])
+            self.recipe_queue = []
+        elif next_command == self.SUB_BUILD_IMAGE:
+            self.clear_busy()
+            self.building = True
+            targets = [self.image]
+            if self.package_queue:
+                self.runCommand(["setVariable", "LINGUAS_INSTALL", ""])
+                self.runCommand(["setVariable", "PACKAGE_INSTALL", " ".join(self.package_queue)])
+            if self.toolchain_packages:
+                self.runCommand(["setVariable", "TOOLCHAIN_TARGET_TASK", " ".join(self.toolchain_packages)])
+                targets.append(self.toolchain)
+            self.runCommand(["buildTargets", targets, self.default_task])
 
-    def handle_event(self, event, running_build, pbar):
+    def handle_event(self, event):
         if not event:
-	    return
+            return
 
-        # If we're running a build, use the RunningBuild event handler
         if self.building:
             self.current_phase = "building"
-            running_build.handle_event(event)
+            self.build.handle_event(event)
+
+        if isinstance(event, bb.event.PackageInfo):
+            self.package_model.populate(event._pkginfolist)
+            self.emit("package-populated")
+            self.run_next_command()
+
+        elif isinstance(event, bb.event.SanityCheckPassed):
+            self.run_next_command()
+
+        elif isinstance(event, bb.event.SanityCheckFailed):
+            self.emit("sanity-failed", event._msg)
+
+        elif isinstance(event, logging.LogRecord):
+            if event.levelno >= logging.ERROR:
+                formatter = bb.msg.BBLogFormatter()
+                formatter.format(event)
+                self.error_msg += event.message + '\n'
+
         elif isinstance(event, bb.event.TargetsTreeGenerated):
             self.current_phase = "data generation"
             if event._model:
-                self.model.populate(event._model)
+                self.recipe_model.populate(event._model)
+                self.emit("recipe-populated")
         elif isinstance(event, bb.event.ConfigFilesFound):
             self.current_phase = "configuration lookup"
             var = event._variable
-	    if var == "distro":
-		distros = event._values
-		distros.sort()
-		self.emit("distros-updated", distros)
-	    elif var == "machine":
-	        machines = event._values
-		machines.sort()
-		self.emit("machines-updated", machines)
-            elif var == "machine-sdk":
-                sdk_machines = event._values
-                sdk_machines.sort()
-                self.emit("sdk-machines-updated", sdk_machines)
+            values = event._values
+            values.sort()
+            self.emit("config-updated", var, values)
         elif isinstance(event, bb.event.ConfigFilePathFound):
             self.current_phase = "configuration lookup"
-            path = event._path
-            self.emit("config-found", path)
         elif isinstance(event, bb.event.FilesMatchingFound):
             self.current_phase = "configuration lookup"
             # FIXME: hard coding, should at least be a variable shared between
@@ -183,161 +209,315 @@ class HobHandler(gobject.GObject):
                     fs, sep, format = classname.rpartition("_")
                     formats.append(format)
                 formats.sort()
-                self.emit("package-formats-found", formats)
+                self.emit("package-formats-updated", formats)
         elif isinstance(event, bb.command.CommandCompleted):
             self.current_phase = None
             self.run_next_command()
         elif isinstance(event, bb.command.CommandFailed):
-            self.emit("command-failed", event.error)
-        elif isinstance(event, bb.event.CacheLoadStarted):
-            self.current_phase = "cache loading"
-            bb.ui.crumbs.hobeventhandler.progress_total = event.total
-            pbar.set_text("Loading cache: %s/%s" % (0, bb.ui.crumbs.hobeventhandler.progress_total))
-        elif isinstance(event, bb.event.CacheLoadProgress):
-            self.current_phase = "cache loading"
-            pbar.set_text("Loading cache: %s/%s" % (event.current, bb.ui.crumbs.hobeventhandler.progress_total))
-        elif isinstance(event, bb.event.CacheLoadCompleted):
-            self.current_phase = None
-            pbar.set_text("Loading...")
-        elif isinstance(event, bb.event.ParseStarted):
-            self.current_phase = "recipe parsing"
-            if event.total == 0:
-                return
-            bb.ui.crumbs.hobeventhandler.progress_total = event.total
-            pbar.set_text("Processing recipes: %s/%s" % (0, bb.ui.crumbs.hobeventhandler.progress_total))
-        elif isinstance(event, bb.event.ParseProgress):
-            self.current_phase = "recipe parsing"
-            pbar.set_text("Processing recipes: %s/%s" % (event.current, bb.ui.crumbs.hobeventhandler.progress_total))
-        elif isinstance(event, bb.event.ParseCompleted):
-            self.current_phase = None
-            pbar.set_fraction(1.0)
-            pbar.set_text("Loading...")
-        elif isinstance(event, logging.LogRecord):
-            format = bb.msg.BBLogFormatter("%(levelname)s: %(message)s")
-            if event.levelno >= format.CRITICAL:
-                self.emit("fatal-error", event.getMessage(), self.current_phase)
+            self.commands_async = []
+            self.clear_busy()
+            self.emit("command-failed", self.error_msg)
+            self.error_msg = ""
+            if self.building:
+                self.building = False
+        elif isinstance(event, (bb.event.ParseStarted,
+                 bb.event.CacheLoadStarted,
+                 bb.event.TreeDataPreparationStarted,
+                 )):
+            message = {}
+            message["eventname"] = bb.event.getName(event)
+            message["current"] = 0
+            message["total"] = None
+            message["title"] = "Parsing recipes: "
+            self.emit("parsing-started", message)
+        elif isinstance(event, (bb.event.ParseProgress,
+                bb.event.CacheLoadProgress,
+                bb.event.TreeDataPreparationProgress)):
+            message = {}
+            message["eventname"] = bb.event.getName(event)
+            message["current"] = event.current
+            message["total"] = event.total
+            message["title"] = "Parsing recipes: "
+            self.emit("parsing", message)
+        elif isinstance(event, (bb.event.ParseCompleted,
+                bb.event.CacheLoadCompleted,
+                bb.event.TreeDataPreparationCompleted)):
+            message = {}
+            message["eventname"] = bb.event.getName(event)
+            message["current"] = event.total
+            message["total"] = event.total
+            message["title"] = "Parsing recipes: "
+            self.emit("parsing-completed", message)
+
         return
 
-    def event_handle_idle_func (self, eventHandler, running_build, pbar):
-        # Consume as many messages as we can in the time available to us
-        event = eventHandler.getEvent()
-        while event:
-            self.handle_event(event, running_build, pbar)
-            event = eventHandler.getEvent()
-        return True
+    def init_cooker(self):
+        self.runCommand(["initCooker"])
+
+    def set_extra_inherit(self, bbclass):
+        inherits = self.runCommand(["getVariable", "INHERIT"]) or ""
+        inherits = inherits + " " + bbclass
+        self.runCommand(["setVariable", "INHERIT", inherits])
+
+    def set_bblayers(self, bblayers):
+        self.runCommand(["setVariable", "BBLAYERS_HOB", " ".join(bblayers)])
 
     def set_machine(self, machine):
-        self.server.runCommand(["setVariable", "MACHINE", machine])
+        if machine:
+            self.runCommand(["setVariable", "MACHINE_HOB", machine])
 
     def set_sdk_machine(self, sdk_machine):
-        self.server.runCommand(["setVariable", "SDKMACHINE", sdk_machine])
+        self.runCommand(["setVariable", "SDKMACHINE_HOB", sdk_machine])
+
+    def set_image_fstypes(self, image_fstypes):
+        self.runCommand(["setVariable", "IMAGE_FSTYPES", image_fstypes])
 
     def set_distro(self, distro):
-        self.server.runCommand(["setVariable", "DISTRO", distro])
+        self.runCommand(["setVariable", "DISTRO_HOB", distro])
 
     def set_package_format(self, format):
-        self.server.runCommand(["setVariable", "PACKAGE_CLASSES", "package_%s" % format])
-
-    def reload_data(self, config=None):
-        img = self.model.selected_image
-        selected_packages, _ = self.model.get_selected_packages()
-        self.emit("reload-triggered", img, " ".join(selected_packages))
-        self.current_command = self.REPARSE_FILES
-        self.run_next_command()
+        package_classes = ""
+        for pkgfmt in format.split():
+            package_classes += ("package_%s" % pkgfmt + " ")
+        self.runCommand(["setVariable", "PACKAGE_CLASSES_HOB", package_classes])
 
     def set_bbthreads(self, threads):
-        self.server.runCommand(["setVariable", "BB_NUMBER_THREADS", threads])
+        self.runCommand(["setVariable", "BB_NUMBER_THREADS_HOB", threads])
 
     def set_pmake(self, threads):
         pmake = "-j %s" % threads
-        self.server.runCommand(["setVariable", "BB_NUMBER_THREADS", pmake])
+        self.runCommand(["setVariable", "PARALLEL_MAKE_HOB", pmake])
 
-    def build_targets(self, tgts, configurator, build_type="image"):
-        self.build_type = build_type
+    def set_dl_dir(self, directory):
+        self.runCommand(["setVariable", "DL_DIR_HOB", directory])
+
+    def set_sstate_dir(self, directory):
+        self.runCommand(["setVariable", "SSTATE_DIR_HOB", directory])
+
+    def set_sstate_mirror(self, url):
+        self.runCommand(["setVariable", "SSTATE_MIRROR_HOB", url])
+
+    def set_extra_size(self, image_extra_size):
+        self.runCommand(["setVariable", "IMAGE_ROOTFS_EXTRA_SPACE", str(image_extra_size)])
+
+    def set_rootfs_size(self, image_rootfs_size):
+        self.runCommand(["setVariable", "IMAGE_ROOTFS_SIZE", str(image_rootfs_size)])
+
+    def set_incompatible_license(self, incompat_license):
+        self.runCommand(["setVariable", "INCOMPATIBLE_LICENSE_HOB", incompat_license])
+
+    def set_extra_config(self, extra_setting):
+        for key in extra_setting.keys():
+            value = extra_setting[key]
+            self.runCommand(["setVariable", key, value])
+
+    def set_config_filter(self, config_filter):
+        self.runCommand(["setConfFilter", config_filter])
+
+    def set_http_proxy(self, http_proxy):
+        self.runCommand(["setVariable", "http_proxy", http_proxy])
+
+    def set_https_proxy(self, https_proxy):
+        self.runCommand(["setVariable", "https_proxy", https_proxy])
+
+    def set_ftp_proxy(self, ftp_proxy):
+        self.runCommand(["setVariable", "ftp_proxy", ftp_proxy])
+
+    def set_all_proxy(self, all_proxy):
+        self.runCommand(["setVariable", "all_proxy", all_proxy])
+
+    def set_git_proxy(self, host, port):
+        self.runCommand(["setVariable", "GIT_PROXY_HOST", host])
+        self.runCommand(["setVariable", "GIT_PROXY_PORT", port])
+
+    def set_cvs_proxy(self, host, port):
+        self.runCommand(["setVariable", "CVS_PROXY_HOST", host])
+        self.runCommand(["setVariable", "CVS_PROXY_PORT", port])
+
+    def request_package_info(self):
+        self.commands_async.append(self.SUB_GENERATE_PKGINFO)
+        self.run_next_command(self.POPULATE_PACKAGEINFO)
+
+    def trigger_sanity_check(self):
+        self.commands_async.append(self.SUB_SANITY_CHECK)
+        self.run_next_command(self.SANITY_CHECK)
+
+    def generate_configuration(self):
+        self.commands_async.append(self.SUB_PARSE_CONFIG)
+        self.commands_async.append(self.SUB_PATH_LAYERS)
+        self.commands_async.append(self.SUB_FILES_DISTRO)
+        self.commands_async.append(self.SUB_FILES_MACH)
+        self.commands_async.append(self.SUB_FILES_SDKMACH)
+        self.commands_async.append(self.SUB_MATCH_CLASS)
+        self.run_next_command(self.GENERATE_CONFIGURATION)
+
+    def generate_recipes(self):
+        self.commands_async.append(self.SUB_PARSE_CONFIG)
+        self.commands_async.append(self.SUB_GNERATE_TGTS)
+        self.run_next_command(self.GENERATE_RECIPES)
+                 
+    def generate_packages(self, tgts, default_task="build"):
         targets = []
-        nbbp = None
-        nbbf = None
         targets.extend(tgts)
-        if self.build_toolchain and self.build_toolchain_headers:
-            targets.append("meta-toolchain-sdk")
-        elif self.build_toolchain:
-            targets.append("meta-toolchain")
-        self.build_queue = targets
+        self.recipe_queue = targets
+        self.default_task = default_task
+        self.commands_async.append(self.SUB_PARSE_CONFIG)
+        self.commands_async.append(self.SUB_BUILD_RECIPES)
+        self.run_next_command(self.GENERATE_PACKAGES)
 
-        if not self.bbpath_ok:
-            if self.image_dir in self.bbpath.split(":"):
-                self.bbpath_ok = True
-            else:
-                nbbp = self.image_dir
+    def generate_image(self, image, toolchain, image_packages=[], toolchain_packages=[], default_task="build"):
+        self.image = image
+        self.toolchain = toolchain
+        self.package_queue = image_packages
+        self.toolchain_packages = toolchain_packages
+        self.default_task = default_task
+        self.commands_async.append(self.SUB_PARSE_CONFIG)
+        self.commands_async.append(self.SUB_BUILD_IMAGE)
+        self.run_next_command(self.GENERATE_IMAGE)
 
-        if not self.bbfiles_ok:
-            import re
-            pattern = "%s/\*.bb" % self.image_dir
+    def build_succeeded_async(self):
+        self.building = False
 
-            for files in self.bbfiles.split(" "):
-                if re.match(pattern, files):
-                    self.bbfiles_ok = True
+    def build_failed_async(self):
+        self.initcmd = None
+        self.commands_async = []
+        self.building = False
 
-            if not self.bbfiles_ok:
-                nbbf = "%s/*.bb" % self.image_dir
-
-        if nbbp or nbbf:
-            configurator.insertTempBBPath(nbbp, nbbf)
-            self.bbpath_ok = True
-            self.bbfiles_ok = True
-
-        self.current_command = self.REPARSE_FILES
-        self.run_next_command()
+    def cancel_parse(self):
+        self.runCommand(["stateStop"])
 
     def cancel_build(self, force=False):
         if force:
             # Force the cooker to stop as quickly as possible
-            self.server.runCommand(["stateStop"])
+            self.runCommand(["stateStop"])
         else:
             # Wait for tasks to complete before shutting down, this helps
             # leave the workdir in a usable state
-            self.server.runCommand(["stateShutdown"])
+            self.runCommand(["stateShutdown"])
 
-    def set_incompatible_license(self, incompatible):
-        self.server.runCommand(["setVariable", "INCOMPATIBLE_LICENSE", incompatible])
+    def reset_build(self):
+        self.build.reset()
 
-    def toggle_toolchain(self, enabled):
-        if self.build_toolchain != enabled:
-            self.build_toolchain = enabled
+    def _remove_redundant(self, string):
+        ret = []
+        for i in string.split():
+            if i not in ret:
+                ret.append(i)
+        return " ".join(ret)
 
-    def toggle_toolchain_headers(self, enabled):
-        if self.build_toolchain_headers != enabled:
-            self.build_toolchain_headers = enabled
+    def get_parameters(self):
+        # retrieve the parameters from bitbake
+        params = {}
+        params["core_base"] = self.runCommand(["getVariable", "COREBASE"]) or ""
+        hob_layer = params["core_base"] + "/meta-hob"
+        params["layer"] = self.runCommand(["getVariable", "BBLAYERS"]) or ""
+        if hob_layer not in params["layer"].split():
+            params["layer"] += (" " + hob_layer)
+        params["dldir"] = self.runCommand(["getVariable", "DL_DIR"]) or ""
+        params["machine"] = self.runCommand(["getVariable", "MACHINE"]) or ""
+        params["distro"] = self.runCommand(["getVariable", "DISTRO"]) or "defaultsetup"
+        params["pclass"] = self.runCommand(["getVariable", "PACKAGE_CLASSES"]) or ""
+        params["sstatedir"] = self.runCommand(["getVariable", "SSTATE_DIR"]) or ""
+        params["sstatemirror"] = self.runCommand(["getVariable", "SSTATE_MIRROR"]) or ""
 
-    def set_fstypes(self, fstypes):
-        self.server.runCommand(["setVariable", "IMAGE_FSTYPES", fstypes])
+        num_threads = self.runCommand(["getCpuCount"])
+        if not num_threads:
+            num_threads = 1
+            max_threads = 65536
+        else:
+            try:
+                num_threads = int(num_threads)
+                max_threads = 16 * num_threads
+            except:
+                num_threads = 1
+                max_threads = 65536
+        params["max_threads"] = max_threads
 
-    def add_image_output_type(self, output_type):
-        if output_type not in self.image_output_types:
-            self.image_output_types.append(output_type)
-            fstypes = " ".join(self.image_output_types).lstrip(" ")
-            self.set_fstypes(fstypes)
-        return self.image_output_types
+        bbthread = self.runCommand(["getVariable", "BB_NUMBER_THREADS"])
+        if not bbthread:
+            bbthread = num_threads
+        else:
+            try:
+                bbthread = int(bbthread)
+            except:
+                bbthread = num_threads
+        params["bbthread"] = bbthread
 
-    def remove_image_output_type(self, output_type):
-        if output_type in self.image_output_types:
-            ind = self.image_output_types.index(output_type)
-            self.image_output_types.pop(ind)
-            fstypes = " ".join(self.image_output_types).lstrip(" ")
-            self.set_fstypes(fstypes)
-        return self.image_output_types
+        pmake = self.runCommand(["getVariable", "PARALLEL_MAKE"])
+        if not pmake:
+            pmake = num_threads
+        elif isinstance(pmake, int):
+            pass
+        else:
+            try:
+                pmake = int(pmake.lstrip("-j "))
+            except:
+                pmake = num_threads
+        params["pmake"] = "-j %s" % pmake
 
-    def get_image_deploy_dir(self):
-        return self.image_out_dir
+        params["image_addr"] = self.runCommand(["getVariable", "DEPLOY_DIR_IMAGE"]) or ""
 
-    def make_temp_dir(self):
-        bb.utils.mkdirhier(self.image_dir)
+        image_extra_size = self.runCommand(["getVariable", "IMAGE_ROOTFS_EXTRA_SPACE"])
+        if not image_extra_size:
+            image_extra_size = 0
+        else:
+            try:
+                image_extra_size = int(image_extra_size)
+            except:
+                image_extra_size = 0
+        params["image_extra_size"] = image_extra_size
 
-    def remove_temp_dir(self):
-        bb.utils.remove(self.image_dir, True)
+        image_rootfs_size = self.runCommand(["getVariable", "IMAGE_ROOTFS_SIZE"])
+        if not image_rootfs_size:
+            image_rootfs_size = 0
+        else:
+            try:
+                image_rootfs_size = int(image_rootfs_size)
+            except:
+                image_rootfs_size = 0
+        params["image_rootfs_size"] = image_rootfs_size
 
-    def get_temp_recipe_path(self, name):
-        timestamp = datetime.date.today().isoformat()
-        image_file = "hob-%s-variant-%s.bb" % (name, timestamp)
-        recipepath =  os.path.join(self.image_dir, image_file)
-        return recipepath
+        image_overhead_factor = self.runCommand(["getVariable", "IMAGE_OVERHEAD_FACTOR"])
+        if not image_overhead_factor:
+            image_overhead_factor = 1
+        else:
+            try:
+                image_overhead_factor = float(image_overhead_factor)
+            except:
+                image_overhead_factor = 1
+        params['image_overhead_factor'] = image_overhead_factor
+
+        params["incompat_license"] = self._remove_redundant(self.runCommand(["getVariable", "INCOMPATIBLE_LICENSE"]) or "")
+        params["sdk_machine"] = self.runCommand(["getVariable", "SDKMACHINE"]) or self.runCommand(["getVariable", "SDK_ARCH"]) or ""
+
+        params["image_fstypes"] = self._remove_redundant(self.runCommand(["getVariable", "IMAGE_FSTYPES"]) or "")
+
+        params["image_types"] = self._remove_redundant(self.runCommand(["getVariable", "IMAGE_TYPES"]) or "")
+
+        params["conf_version"] = self.runCommand(["getVariable", "CONF_VERSION"]) or ""
+        params["lconf_version"] = self.runCommand(["getVariable", "LCONF_VERSION"]) or ""
+
+        params["runnable_image_types"] = self._remove_redundant(self.runCommand(["getVariable", "RUNNABLE_IMAGE_TYPES"]) or "")
+        params["runnable_machine_patterns"] = self._remove_redundant(self.runCommand(["getVariable", "RUNNABLE_MACHINE_PATTERNS"]) or "")
+        params["deployable_image_types"] = self._remove_redundant(self.runCommand(["getVariable", "DEPLOYABLE_IMAGE_TYPES"]) or "")
+        params["tmpdir"] = self.runCommand(["getVariable", "TMPDIR"]) or ""
+        params["distro_version"] = self.runCommand(["getVariable", "DISTRO_VERSION"]) or ""
+        params["target_os"] = self.runCommand(["getVariable", "TARGET_OS"]) or ""
+        params["target_arch"] = self.runCommand(["getVariable", "TARGET_ARCH"]) or ""
+        params["tune_pkgarch"] = self.runCommand(["getVariable", "TUNE_PKGARCH"])  or ""
+        params["bb_version"] = self.runCommand(["getVariable", "BB_MIN_VERSION"]) or ""
+
+        params["default_task"] = self.runCommand(["getVariable", "BB_DEFAULT_TASK"]) or "build"
+
+        params["git_proxy_host"] = self.runCommand(["getVariable", "GIT_PROXY_HOST"]) or ""
+        params["git_proxy_port"] = self.runCommand(["getVariable", "GIT_PROXY_PORT"]) or ""
+
+        params["http_proxy"] = self.runCommand(["getVariable", "http_proxy"]) or ""
+        params["ftp_proxy"] = self.runCommand(["getVariable", "ftp_proxy"]) or ""
+        params["https_proxy"] = self.runCommand(["getVariable", "https_proxy"]) or ""
+        params["all_proxy"] = self.runCommand(["getVariable", "all_proxy"]) or ""
+
+        params["cvs_proxy_host"] = self.runCommand(["getVariable", "CVS_PROXY_HOST"]) or ""
+        params["cvs_proxy_port"] = self.runCommand(["getVariable", "CVS_PROXY_PORT"]) or ""
+
+        return params

bitbake/lib/bb/ui/crumbs/hoblistmodel.py

@@ -0,0 +1,752 @@
+#
+# BitBake Graphical GTK User Interface
+#
+# Copyright (C) 2011        Intel Corporation
+#
+# Authored by Joshua Lock <josh@linux.intel.com>
+# Authored by Dongxiao Xu <dongxiao.xu@intel.com>
+# Authored by Shane Wang <shane.wang@intel.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import gtk
+import gobject
+from bb.ui.crumbs.hobpages import HobPage
+
+#
+# PackageListModel
+#
+class PackageListModel(gtk.TreeStore):
+    """
+    This class defines an gtk.TreeStore subclass which will convert the output
+    of the bb.event.TargetsTreeGenerated event into a gtk.TreeStore whilst also
+    providing convenience functions to access gtk.TreeModel subclasses which
+    provide filtered views of the data.
+    """
+    (COL_NAME, COL_VER, COL_REV, COL_RNM, COL_SEC, COL_SUM, COL_RDEP, COL_RPROV, COL_SIZE, COL_BINB, COL_INC, COL_FADE_INC) = range(12)
+
+    __gsignals__ = {
+        "package-selection-changed" : (gobject.SIGNAL_RUN_LAST,
+                                gobject.TYPE_NONE,
+                                ()),
+    }
+
+    __toolchain_required_packages__ = ["task-core-standalone-sdk-target", "task-core-standalone-sdk-target-dbg"]
+
+    def __init__(self):
+
+        self.contents = None
+        self.images = None
+        self.pkgs_size = 0
+        self.pn_path = {}
+        self.pkg_path = {}
+        self.rprov_pkg = {}
+        
+        gtk.TreeStore.__init__ (self,
+                                gobject.TYPE_STRING,
+                                gobject.TYPE_STRING,
+                                gobject.TYPE_STRING,
+                                gobject.TYPE_STRING,
+                                gobject.TYPE_STRING,
+                                gobject.TYPE_STRING,
+                                gobject.TYPE_STRING,
+                                gobject.TYPE_STRING,
+                                gobject.TYPE_STRING,
+                                gobject.TYPE_STRING,
+                                gobject.TYPE_BOOLEAN,
+                                gobject.TYPE_BOOLEAN)
+
+
+    """
+    Find the model path for the item_name
+    Returns the path in the model or None
+    """
+    def find_path_for_item(self, item_name):
+        pkg = item_name
+        if item_name not in self.pkg_path.keys():
+            if item_name not in self.rprov_pkg.keys():
+                return None
+            pkg = self.rprov_pkg[item_name]
+            if pkg not in self.pkg_path.keys():
+                return None
+
+        return self.pkg_path[pkg]
+
+    def find_item_for_path(self, item_path):
+        return self[item_path][self.COL_NAME]
+
+    """
+    Helper function to determine whether an item is an item specified by filter
+    """
+    def tree_model_filter(self, model, it, filter):
+        for key in filter.keys():
+            if model.get_value(it, key) not in filter[key]:
+                return False
+
+        return True
+
+    """
+    Create, if required, and return a filtered gtk.TreeModelSort
+    containing only the items specified by filter
+    """
+    def tree_model(self, filter):
+        model = self.filter_new()
+        model.set_visible_func(self.tree_model_filter, filter)
+
+        sort = gtk.TreeModelSort(model)
+        sort.set_sort_column_id(PackageListModel.COL_NAME, gtk.SORT_ASCENDING)
+        sort.set_default_sort_func(None)
+        return sort
+
+    def convert_vpath_to_path(self, view_model, view_path):
+        # view_model is the model sorted
+        # get the path of the model filtered
+        filtered_model_path = view_model.convert_path_to_child_path(view_path)
+        # get the model filtered
+        filtered_model = view_model.get_model()
+        # get the path of the original model
+        path = filtered_model.convert_path_to_child_path(filtered_model_path)
+        return path
+
+    def convert_path_to_vpath(self, view_model, path):
+        name = self.find_item_for_path(path)
+        it = view_model.get_iter_first()
+        while it:
+            child_it = view_model.iter_children(it)
+            while child_it:
+                view_name = view_model.get_value(child_it, self.COL_NAME)
+                if view_name == name:
+                    view_path = view_model.get_path(child_it)
+                    return view_path
+                child_it = view_model.iter_next(child_it)
+            it = view_model.iter_next(it)
+        return None
+
+    """
+    The populate() function takes as input the data from a
+    bb.event.PackageInfo event and populates the package list.
+    """
+    def populate(self, pkginfolist):
+        self.clear()
+        self.pkgs_size = 0
+        self.pn_path = {}
+        self.pkg_path = {}
+        self.rprov_pkg = {}
+
+        for pkginfo in pkginfolist:
+            pn = pkginfo['PN']
+            pv = pkginfo['PV']
+            pr = pkginfo['PR']
+            if pn in self.pn_path.keys():
+                pniter = self.get_iter(self.pn_path[pn])
+            else:
+                pniter = self.append(None)
+                self.set(pniter, self.COL_NAME, pn + '-' + pv + '-' + pr,
+                                 self.COL_INC, False)
+                self.pn_path[pn] = self.get_path(pniter)
+
+            pkg = pkginfo['PKG']
+            pkgv = pkginfo['PKGV']
+            pkgr = pkginfo['PKGR']
+            pkgsize = pkginfo['PKGSIZE_%s' % pkg] if 'PKGSIZE_%s' % pkg in pkginfo.keys() else "0"
+            pkg_rename = pkginfo['PKG_%s' % pkg] if 'PKG_%s' % pkg in pkginfo.keys() else ""
+            section = pkginfo['SECTION_%s' % pkg] if 'SECTION_%s' % pkg in pkginfo.keys() else ""
+            summary = pkginfo['SUMMARY_%s' % pkg] if 'SUMMARY_%s' % pkg in pkginfo.keys() else ""
+            rdep = pkginfo['RDEPENDS_%s' % pkg] if 'RDEPENDS_%s' % pkg in pkginfo.keys() else ""
+            rrec = pkginfo['RRECOMMENDS_%s' % pkg] if 'RRECOMMENDS_%s' % pkg in pkginfo.keys() else ""
+            rprov = pkginfo['RPROVIDES_%s' % pkg] if 'RPROVIDES_%s' % pkg in pkginfo.keys() else ""
+            for i in rprov.split():
+                self.rprov_pkg[i] = pkg
+
+            if 'ALLOW_EMPTY_%s' % pkg in pkginfo.keys():
+                allow_empty = pkginfo['ALLOW_EMPTY_%s' % pkg]
+            elif 'ALLOW_EMPTY' in pkginfo.keys():
+                allow_empty = pkginfo['ALLOW_EMPTY']
+            else:
+                allow_empty = ""
+
+            if pkgsize == "0" and not allow_empty:
+                continue
+
+            # pkgsize is in KB
+            size = HobPage._size_to_string(HobPage._string_to_size(pkgsize + ' KB'))
+
+            it = self.append(pniter)
+            self.pkg_path[pkg] = self.get_path(it)
+            self.set(it, self.COL_NAME, pkg, self.COL_VER, pkgv,
+                     self.COL_REV, pkgr, self.COL_RNM, pkg_rename,
+                     self.COL_SEC, section, self.COL_SUM, summary,
+                     self.COL_RDEP, rdep + ' ' + rrec,
+                     self.COL_RPROV, rprov, self.COL_SIZE, size,
+                     self.COL_BINB, "", self.COL_INC, False)
+
+    """
+    Check whether the item at item_path is included or not
+    """
+    def path_included(self, item_path):
+        return self[item_path][self.COL_INC]
+
+    """
+    Update the model, send out the notification.
+    """
+    def selection_change_notification(self):
+        self.emit("package-selection-changed")
+
+    """
+    Mark a certain package as selected.
+    All its dependencies are marked as selected.
+    The recipe provides the package is marked as selected.
+    If user explicitly selects a recipe, all its providing packages are selected
+    """
+    def include_item(self, item_path, binb=""):
+        if self.path_included(item_path):
+            return
+
+        item_name = self[item_path][self.COL_NAME]
+        item_rdep = self[item_path][self.COL_RDEP]
+
+        self[item_path][self.COL_INC] = True
+
+        it = self.get_iter(item_path)
+
+        # If user explicitly selects a recipe, all its providing packages are selected.
+        if not self[item_path][self.COL_VER] and binb == "User Selected":
+            child_it = self.iter_children(it)
+            while child_it:
+                child_path = self.get_path(child_it)
+                child_included = self.path_included(child_path)
+                if not child_included:
+                    self.include_item(child_path, binb="User Selected")
+                child_it = self.iter_next(child_it)
+            return
+
+        # The recipe provides the package is also marked as selected
+        parent_it = self.iter_parent(it)
+        if parent_it:
+            parent_path = self.get_path(parent_it)
+            self[parent_path][self.COL_INC] = True
+
+        item_bin = self[item_path][self.COL_BINB].split(', ')
+        if binb and not binb in item_bin:
+            item_bin.append(binb)
+            self[item_path][self.COL_BINB] = ', '.join(item_bin).lstrip(', ')
+
+        if item_rdep:
+            # Ensure all of the items deps are included and, where appropriate,
+            # add this item to their COL_BINB
+            for dep in item_rdep.split(" "):
+                if dep.startswith('('):
+                    continue
+                # If the contents model doesn't already contain dep, add it
+                dep_path = self.find_path_for_item(dep)
+                if not dep_path:
+                    continue
+                dep_included = self.path_included(dep_path)
+
+                if dep_included and not dep in item_bin:
+                    # don't set the COL_BINB to this item if the target is an
+                    # item in our own COL_BINB
+                    dep_bin = self[dep_path][self.COL_BINB].split(', ')
+                    if not item_name in dep_bin:
+                        dep_bin.append(item_name)
+                        self[dep_path][self.COL_BINB] = ', '.join(dep_bin).lstrip(', ')
+                elif not dep_included:
+                    self.include_item(dep_path, binb=item_name)
+
+    """
+    Mark a certain package as de-selected.
+    All other packages that depends on this package are marked as de-selected.
+    If none of the packages provided by the recipe, the recipe should be marked as de-selected.
+    If user explicitly de-select a recipe, all its providing packages are de-selected.
+    """
+    def exclude_item(self, item_path):
+        if not self.path_included(item_path):
+            return
+
+        self[item_path][self.COL_INC] = False
+
+        item_name = self[item_path][self.COL_NAME]
+        item_rdep = self[item_path][self.COL_RDEP]
+        it = self.get_iter(item_path)
+
+        # If user explicitly de-select a recipe, all its providing packages are de-selected.
+        if not self[item_path][self.COL_VER]:
+            child_it = self.iter_children(it)
+            while child_it:
+                child_path = self.get_path(child_it)
+                child_included = self[child_path][self.COL_INC]
+                if child_included:
+                    self.exclude_item(child_path)
+                child_it = self.iter_next(child_it)
+            return
+
+        # If none of the packages provided by the recipe, the recipe should be marked as de-selected.
+        parent_it = self.iter_parent(it)
+        peer_iter = self.iter_children(parent_it)
+        enabled = 0
+        while peer_iter:
+            peer_path = self.get_path(peer_iter)
+            if self[peer_path][self.COL_INC]:
+                enabled = 1
+                break
+            peer_iter = self.iter_next(peer_iter)
+        if not enabled:
+            parent_path = self.get_path(parent_it)
+            self[parent_path][self.COL_INC] = False
+
+        # All packages that depends on this package are de-selected.
+        if item_rdep:
+            for dep in item_rdep.split(" "):
+                if dep.startswith('('):
+                    continue
+                dep_path = self.find_path_for_item(dep)
+                if not dep_path:
+                    continue
+                dep_bin = self[dep_path][self.COL_BINB].split(', ')
+                if item_name in dep_bin:
+                    dep_bin.remove(item_name)
+                    self[dep_path][self.COL_BINB] = ', '.join(dep_bin).lstrip(', ')
+
+        item_bin = self[item_path][self.COL_BINB].split(', ')
+        if item_bin:
+            for binb in item_bin:
+                binb_path = self.find_path_for_item(binb)
+                if not binb_path:
+                    continue
+                self.exclude_item(binb_path)
+
+    """
+    Package model may be incomplete, therefore when calling the
+    set_selected_packages(), some packages will not be set included.
+    Return the un-set packages list.
+    """
+    def set_selected_packages(self, packagelist):
+        left = []
+        for pn in packagelist:
+            if pn in self.pkg_path.keys():
+                path = self.pkg_path[pn]
+                self.include_item(item_path=path,
+                                  binb="User Selected")
+            else:
+                left.append(pn)
+
+        self.selection_change_notification()
+        return left
+
+    def get_user_selected_packages(self):
+        packagelist = []
+
+        it = self.get_iter_first()
+        while it:
+            child_it = self.iter_children(it)
+            while child_it:
+                if self.get_value(child_it, self.COL_INC):
+                    binb = self.get_value(child_it, self.COL_BINB)
+                    if not binb or binb == "User Selected":
+                        name = self.get_value(child_it, self.COL_NAME)
+                        packagelist.append(name)
+                child_it = self.iter_next(child_it)
+            it = self.iter_next(it)
+
+        return packagelist
+
+    def get_selected_packages(self):
+        packagelist = []
+
+        it = self.get_iter_first()
+        while it:
+            child_it = self.iter_children(it)
+            while child_it:
+                if self.get_value(child_it, self.COL_INC):
+                    name = self.get_value(child_it, self.COL_NAME)
+                    packagelist.append(name)
+                child_it = self.iter_next(child_it)
+            it = self.iter_next(it)
+
+        return packagelist
+
+    def get_selected_packages_toolchain(self):
+        packagelist = []
+
+        it = self.get_iter_first()
+        while it:
+            if self.get_value(it, self.COL_INC):
+                child_it = self.iter_children(it)
+                while child_it:
+                    name = self.get_value(child_it, self.COL_NAME)
+                    inc = self.get_value(child_it, self.COL_INC)
+                    if inc or name.endswith("-dev") or name.endswith("-dbg"):
+                        packagelist.append(name)
+                    child_it = self.iter_next(child_it)
+            it = self.iter_next(it)
+
+        return list(set(packagelist + self.__toolchain_required_packages__));
+    """
+    Return the selected package size, unit is B.
+    """
+    def get_packages_size(self):
+        packages_size = 0
+        it = self.get_iter_first()
+        while it:
+            child_it = self.iter_children(it)
+            while child_it:
+                if self.get_value(child_it, self.COL_INC):
+                    str_size = self.get_value(child_it, self.COL_SIZE)
+                    if not str_size:
+                        continue
+
+                    packages_size += HobPage._string_to_size(str_size)
+
+                child_it = self.iter_next(child_it)
+            it = self.iter_next(it)
+        return packages_size
+
+    """
+    Empty self.contents by setting the include of each entry to None
+    """
+    def reset(self):
+        self.pkgs_size = 0
+        it = self.get_iter_first()
+        while it:
+            self.set(it, self.COL_INC, False)
+            child_it = self.iter_children(it)
+            while child_it:
+                self.set(child_it,
+                         self.COL_INC, False,
+                         self.COL_BINB, "")
+                child_it = self.iter_next(child_it)
+            it = self.iter_next(it)
+
+        self.selection_change_notification()
+
+    """
+    Resync the state of included items to a backup column before performing the fadeout visible effect
+    """
+    def resync_fadeout_column(self, model_first_iter=None):
+        it = model_first_iter
+        while it:
+            active = self.get_value(it, self.COL_INC)
+            self.set(it, self.COL_FADE_INC, active)
+            if self.iter_has_child(it):
+                self.resync_fadeout_column(self.iter_children(it))
+
+            it = self.iter_next(it)
+
+#
+# RecipeListModel
+#
+class RecipeListModel(gtk.ListStore):
+    """
+    This class defines an gtk.ListStore subclass which will convert the output
+    of the bb.event.TargetsTreeGenerated event into a gtk.ListStore whilst also
+    providing convenience functions to access gtk.TreeModel subclasses which
+    provide filtered views of the data.
+    """
+    (COL_NAME, COL_DESC, COL_LIC, COL_GROUP, COL_DEPS, COL_BINB, COL_TYPE, COL_INC, COL_IMG, COL_INSTALL, COL_PN, COL_FADE_INC) = range(12)
+
+    __dummy_image__ = "Create your own image"
+
+    __gsignals__ = {
+        "recipe-selection-changed" : (gobject.SIGNAL_RUN_LAST,
+                                  gobject.TYPE_NONE,
+                                 ()),
+        }
+
+    """
+    """
+    def __init__(self):
+        gtk.ListStore.__init__ (self,
+                                gobject.TYPE_STRING,
+                                gobject.TYPE_STRING,
+                                gobject.TYPE_STRING,
+                                gobject.TYPE_STRING,
+                                gobject.TYPE_STRING,
+                                gobject.TYPE_STRING,
+                                gobject.TYPE_STRING,
+                                gobject.TYPE_BOOLEAN,
+                                gobject.TYPE_BOOLEAN,
+                                gobject.TYPE_STRING,
+                                gobject.TYPE_STRING,
+                                gobject.TYPE_BOOLEAN)
+
+    """
+    Find the model path for the item_name
+    Returns the path in the model or None
+    """
+    def find_path_for_item(self, item_name):
+        if self.non_target_name(item_name) or item_name not in self.pn_path.keys():
+            return None
+        else:
+            return self.pn_path[item_name]
+
+    def find_item_for_path(self, item_path):
+        return self[item_path][self.COL_NAME]
+
+    """
+    Helper method to determine whether name is a target pn
+    """
+    def non_target_name(self, name):
+        if name and ('-native' in name):
+            return True
+        return False
+
+    """
+    Helper function to determine whether an item is an item specified by filter
+    """
+    def tree_model_filter(self, model, it, filter):
+        name = model.get_value(it, self.COL_NAME)
+        if self.non_target_name(name):
+            return False
+
+        for key in filter.keys():
+            if model.get_value(it, key) not in filter[key]:
+                return False
+
+        return True
+
+    def exclude_item_sort_func(self, model, iter1, iter2):
+        val1 = model.get_value(iter1, RecipeListModel.COL_FADE_INC)
+        val2 = model.get_value(iter2, RecipeListModel.COL_INC)
+        return ((val1 == True) and (val2 == False))
+
+    """
+    Create, if required, and return a filtered gtk.TreeModelSort
+    containing only the items which are items specified by filter
+    """
+    def tree_model(self, filter, excluded_items_ahead=False):
+        model = self.filter_new()
+        model.set_visible_func(self.tree_model_filter, filter)
+
+        sort = gtk.TreeModelSort(model)
+        if excluded_items_ahead:
+            sort.set_default_sort_func(self.exclude_item_sort_func)
+        else:
+            sort.set_sort_column_id(RecipeListModel.COL_NAME, gtk.SORT_ASCENDING)
+            sort.set_default_sort_func(None)
+        return sort
+
+    def convert_vpath_to_path(self, view_model, view_path):
+        filtered_model_path = view_model.convert_path_to_child_path(view_path)
+        filtered_model = view_model.get_model()
+
+        # get the path of the original model
+        path = filtered_model.convert_path_to_child_path(filtered_model_path)
+        return path
+
+    def convert_path_to_vpath(self, view_model, path):
+        it = view_model.get_iter_first()
+        while it:
+            name = self.find_item_for_path(path)
+            view_name = view_model.get_value(it, RecipeListModel.COL_NAME)
+            if view_name == name:
+                view_path = view_model.get_path(it)
+                return view_path
+            it = view_model.iter_next(it)
+        return None
+
+    """
+    The populate() function takes as input the data from a
+    bb.event.TargetsTreeGenerated event and populates the RecipeList.
+    """
+    def populate(self, event_model):
+        # First clear the model, in case repopulating
+        self.clear()
+
+        # dummy image for prompt
+        self.set(self.append(), self.COL_NAME, self.__dummy_image__,
+                 self.COL_DESC, "Use the 'View recipes' and 'View packages' " \
+                                "options to select what you want to include " \
+                                "in your image.",
+                 self.COL_LIC, "", self.COL_GROUP, "",
+                 self.COL_DEPS, "", self.COL_BINB, "",
+                 self.COL_TYPE, "image", self.COL_INC, False,
+                 self.COL_IMG, False, self.COL_INSTALL, "", self.COL_PN, self.__dummy_image__)
+
+        for item in event_model["pn"]:
+            name = item
+            desc = event_model["pn"][item]["description"]
+            lic = event_model["pn"][item]["license"]
+            group = event_model["pn"][item]["section"]
+            inherits = event_model["pn"][item]["inherits"]
+            install = []
+
+            depends = event_model["depends"].get(item, []) + event_model["rdepends-pn"].get(item, [])
+
+            if ('task-' in name):
+                atype = 'task'
+            elif ('image.bbclass' in " ".join(inherits)):
+                if name != "hob-image":
+                    atype = 'image'
+                    install = event_model["rdepends-pkg"].get(item, []) + event_model["rrecs-pkg"].get(item, [])
+            elif ('meta-' in name):
+                atype = 'toolchain'
+            elif (name == 'dummy-image' or name == 'dummy-toolchain'):
+                atype = 'dummy'
+            else:
+                atype = 'recipe'
+
+            self.set(self.append(), self.COL_NAME, item, self.COL_DESC, desc,
+                     self.COL_LIC, lic, self.COL_GROUP, group,
+                     self.COL_DEPS, " ".join(depends), self.COL_BINB, "",
+                     self.COL_TYPE, atype, self.COL_INC, False,
+                     self.COL_IMG, False, self.COL_INSTALL, " ".join(install), self.COL_PN, item)
+
+        self.pn_path = {}
+        it = self.get_iter_first()
+        while it:
+            pn = self.get_value(it, self.COL_NAME)
+            path = self.get_path(it)
+            self.pn_path[pn] = path
+            it = self.iter_next(it)
+
+    """
+    Update the model, send out the notification.
+    """
+    def selection_change_notification(self):
+        self.emit("recipe-selection-changed")
+
+    def path_included(self, item_path):
+        return self[item_path][self.COL_INC]
+
+    """
+    Add this item, and any of its dependencies, to the image contents
+    """
+    def include_item(self, item_path, binb="", image_contents=False):
+        if self.path_included(item_path):
+            return
+
+        item_name = self[item_path][self.COL_NAME]
+        item_deps = self[item_path][self.COL_DEPS]
+
+        self[item_path][self.COL_INC] = True
+
+        item_bin = self[item_path][self.COL_BINB].split(', ')
+        if binb and not binb in item_bin:
+            item_bin.append(binb)
+            self[item_path][self.COL_BINB] = ', '.join(item_bin).lstrip(', ')
+
+        # We want to do some magic with things which are brought in by the
+        # base image so tag them as so
+        if image_contents:
+            self[item_path][self.COL_IMG] = True
+
+        if item_deps:
+            # Ensure all of the items deps are included and, where appropriate,
+            # add this item to their COL_BINB
+            for dep in item_deps.split(" "):
+                # If the contents model doesn't already contain dep, add it
+                dep_path = self.find_path_for_item(dep)
+                if not dep_path:
+                    continue
+                dep_included = self.path_included(dep_path)
+
+                if dep_included and not dep in item_bin:
+                    # don't set the COL_BINB to this item if the target is an
+                    # item in our own COL_BINB
+                    dep_bin = self[dep_path][self.COL_BINB].split(', ')
+                    if not item_name in dep_bin:
+                        dep_bin.append(item_name)
+                        self[dep_path][self.COL_BINB] = ', '.join(dep_bin).lstrip(', ')
+                elif not dep_included:
+                    self.include_item(dep_path, binb=item_name, image_contents=image_contents)
+
+    def exclude_item(self, item_path):
+        if not self.path_included(item_path):
+            return
+
+        self[item_path][self.COL_INC] = False
+
+        item_name = self[item_path][self.COL_NAME]
+        item_deps = self[item_path][self.COL_DEPS]
+        if item_deps:
+            for dep in item_deps.split(" "):
+                dep_path = self.find_path_for_item(dep)
+                if not dep_path:
+                    continue
+                dep_bin = self[dep_path][self.COL_BINB].split(', ')
+                if item_name in dep_bin:
+                    dep_bin.remove(item_name)
+                    self[dep_path][self.COL_BINB] = ', '.join(dep_bin).lstrip(', ')
+
+        item_bin = self[item_path][self.COL_BINB].split(', ')
+        if item_bin:
+            for binb in item_bin:
+                binb_path = self.find_path_for_item(binb)
+                if not binb_path:
+                    continue
+                self.exclude_item(binb_path)
+
+    def reset(self):
+        it = self.get_iter_first()
+        while it:
+            self.set(it,
+                     self.COL_INC, False,
+                     self.COL_BINB, "",
+                     self.COL_IMG, False)
+            it = self.iter_next(it)
+
+        self.selection_change_notification()
+
+    """
+    Returns two lists. One of user selected recipes and the other containing
+    all selected recipes
+    """
+    def get_selected_recipes(self):
+        allrecipes = []
+        userrecipes = []
+
+        it = self.get_iter_first()
+        while it:
+            if self.get_value(it, self.COL_INC):
+                name = self.get_value(it, self.COL_PN)
+                type = self.get_value(it, self.COL_TYPE)
+                if type != "image":
+                    allrecipes.append(name)
+                    sel = "User Selected" in self.get_value(it, self.COL_BINB)
+                    if sel:
+                        userrecipes.append(name)
+            it = self.iter_next(it)
+
+        return list(set(userrecipes)), list(set(allrecipes))
+
+    def set_selected_recipes(self, recipelist):
+        for pn in recipelist:
+            if pn in self.pn_path.keys():
+                path = self.pn_path[pn]
+                self.include_item(item_path=path,
+                                  binb="User Selected")
+        self.selection_change_notification()
+
+    def get_selected_image(self):
+        it = self.get_iter_first()
+        while it:
+            if self.get_value(it, self.COL_INC):
+                name = self.get_value(it, self.COL_PN)
+                type = self.get_value(it, self.COL_TYPE)
+                if type == "image":
+                    sel = "User Selected" in self.get_value(it, self.COL_BINB)
+                    if sel:
+                        return name
+            it = self.iter_next(it)
+        return None
+
+    def set_selected_image(self, img):
+        if not img:
+            return
+        self.reset()
+        path = self.find_path_for_item(img)
+        self.include_item(item_path=path,
+                          binb="User Selected",
+                          image_contents=True)
+        self.selection_change_notification()

bitbake/lib/bb/ui/crumbs/hobpages.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python
+#
+# BitBake Graphical GTK User Interface
+#
+# Copyright (C) 2012        Intel Corporation
+#
+# Authored by Dongxiao Xu <dongxiao.xu@intel.com>
+# Authored by Shane Wang <shane.wang@intel.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import gtk
+from bb.ui.crumbs.hobcolor import HobColors
+from bb.ui.crumbs.hobwidget import hwc
+
+#
+# HobPage: the super class for all Hob-related pages
+#    
+class HobPage (gtk.VBox):
+
+    def __init__(self, builder, title = None):
+        super(HobPage, self).__init__(False, 0)
+        self.builder = builder
+        self.builder_width, self.builder_height = self.builder.size_request()
+
+        if not title:
+            self.title = "Hob -- Image Creator"
+        else:
+            self.title = title
+
+        self.box_group_area = gtk.VBox(False, 12)
+        self.box_group_area.set_size_request(self.builder_width - 73 - 73, self.builder_height - 88 - 15 - 15)
+        self.group_align = gtk.Alignment(xalign = 0, yalign=0.5, xscale=1, yscale=1)
+        self.group_align.set_padding(15, 15, 73, 73)
+        self.group_align.add(self.box_group_area)
+        self.box_group_area.set_homogeneous(False)
+
+
+    def add_onto_top_bar(self, widget = None, padding = 0):
+        # the top button occupies 1/7 of the page height
+        # setup an event box
+        eventbox = gtk.EventBox()
+        style = eventbox.get_style().copy()
+        style.bg[gtk.STATE_NORMAL] = eventbox.get_colormap().alloc_color(HobColors.LIGHT_GRAY, False, False)
+        eventbox.set_style(style)
+        eventbox.set_size_request(-1, 88)
+
+        hbox = gtk.HBox()
+
+        label = gtk.Label()
+        label.set_markup("<span size='x-large'>%s</span>" % self.title)
+        hbox.pack_start(label, expand=False, fill=False, padding=20)
+
+        if widget:
+            # add the widget in the event box
+            hbox.pack_end(widget, expand=False, fill=False, padding=padding)
+        eventbox.add(hbox)
+
+        return eventbox
+
+    def span_tag(self, size="medium", weight="normal", forground="#1c1c1c"):
+        span_tag = "weight='%s' foreground='%s' size='%s'" % (weight, forground, size)
+        return span_tag
+
+    def append_toolbar_button(self, toolbar, buttonname, icon_disp, icon_hovor, tip, cb):
+        # Create a button and append it on the toolbar according to button name
+        icon = gtk.Image()
+        icon_display = icon_disp
+        icon_hover = icon_hovor
+        pix_buffer = gtk.gdk.pixbuf_new_from_file(icon_display)
+        icon.set_from_pixbuf(pix_buffer)
+        tip_text = tip
+        button = toolbar.append_item(buttonname, tip, None, icon, cb)
+        return button
+
+    @staticmethod
+    def _size_to_string(size):
+        try:
+            if not size:
+                size_str = "0 B"
+            else:
+                if len(str(int(size))) > 6:
+                    size_str = '%.1f' % (size*1.0/(1024*1024)) + ' MB'
+                elif len(str(int(size))) > 3:
+                    size_str = '%.1f' % (size*1.0/1024) + ' KB'
+                else:
+                    size_str = str(size) + ' B'
+        except:
+            size_str = "0 B"
+        return size_str
+
+    @staticmethod
+    def _string_to_size(str_size):
+        try:
+            if not str_size:
+                size = 0
+            else:
+                unit = str_size.split()
+                if len(unit) > 1:
+                    if unit[1] == 'MB':
+                        size = float(unit[0])*1024*1024
+                    elif unit[1] == 'KB':
+                        size = float(unit[0])*1024
+                    elif unit[1] == 'B':
+                        size = float(unit[0])
+                    else:
+                        size = 0
+                else:
+                    size = float(unit[0])
+        except:
+            size = 0
+        return size
+

bitbake/lib/bb/ui/crumbs/hobprefs.py

@@ -1,335 +0,0 @@
-#
-# BitBake Graphical GTK User Interface
-#
-# Copyright (C) 2011        Intel Corporation
-#
-# Authored by Joshua Lock <josh@linux.intel.com>
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-import gtk
-import glib
-from bb.ui.crumbs.configurator import Configurator
-
-class HobPrefs(gtk.Dialog):
-    """
-    """
-    def empty_combo_text(self, combo_text):
-        model = combo_text.get_model()
-        if model:
-            model.clear()
-
-    def output_type_toggled_cb(self, check, handler):
-        ot = check.get_label()
-        enabled = check.get_active()
-        if enabled:
-            self.selected_image_types = handler.add_image_output_type(ot)
-        else:
-            self.selected_image_types = handler.remove_image_output_type(ot)
-
-        self.configurator.setConfVar('IMAGE_FSTYPES', "%s" % " ".join(self.selected_image_types).lstrip(" "))
-        self.reload_required = True
-
-    def sdk_machine_combo_changed_cb(self, combo, handler):
-        sdk_mach = combo.get_active_text()
-	if sdk_mach != self.curr_sdk_mach:
-            self.curr_sdk_mach = sdk_mach
-            self.configurator.setConfVar('SDKMACHINE', sdk_mach)
-            handler.set_sdk_machine(sdk_mach)
-
-    def update_sdk_machines(self, handler, sdk_machines):
-        active = 0
-        # disconnect the signal handler before updating the combo model
-        if self.sdk_machine_handler_id:
-            self.sdk_machine_combo.disconnect(self.sdk_machine_handler_id)
-            self.sdk_machine_handler_id = None
-
-        self.empty_combo_text(self.sdk_machine_combo)
-        for sdk_machine in sdk_machines:
-            self.sdk_machine_combo.append_text(sdk_machine)
-            if sdk_machine == self.curr_sdk_mach:
-                self.sdk_machine_combo.set_active(active)
-            active = active + 1
-
-        self.sdk_machine_handler_id = self.sdk_machine_combo.connect("changed", self.sdk_machine_combo_changed_cb, handler)
-
-    def distro_combo_changed_cb(self, combo, handler):
-        distro = combo.get_active_text()
-	if distro != self.curr_distro:
-            self.curr_distro = distro
-            self.configurator.setConfVar('DISTRO', distro)
-            handler.set_distro(distro)
-            self.reload_required = True
-
-    def update_distros(self, handler, distros):
-        active = 0
-        # disconnect the signal handler before updating combo model
-        if self.distro_handler_id:
-            self.distro_combo.disconnect(self.distro_handler_id)
-            self.distro_handler_id = None
-
-        self.empty_combo_text(self.distro_combo)
-	for distro in distros:
-	    self.distro_combo.append_text(distro)
-	    if distro == self.curr_distro:
-                self.distro_combo.set_active(active)
-	    active = active + 1
-
-	self.distro_handler_id = self.distro_combo.connect("changed", self.distro_combo_changed_cb, handler)
-
-    def package_format_combo_changed_cb(self, combo, handler):
-        package_format = combo.get_active_text()
-        if package_format != self.curr_package_format:
-            self.curr_package_format = package_format
-            self.configurator.setConfVar('PACKAGE_CLASSES', 'package_%s' % package_format)
-            handler.set_package_format(package_format)
-            self.reload_required = True
-
-    def update_package_formats(self, handler, formats):
-        active = 0
-        # disconnect the signal handler before updating the model
-        if self.package_handler_id:
-            self.package_combo.disconnect(self.package_handler_id)
-            self.package_handler_id = None
-
-        self.empty_combo_text(self.package_combo)
-        for format in formats:
-            self.package_combo.append_text(format)
-            if format == self.curr_package_format:
-                self.package_combo.set_active(active)
-            active = active + 1
-
-        self.package_handler_id = self.package_combo.connect("changed", self.package_format_combo_changed_cb, handler)
-    
-    def include_gplv3_cb(self, toggle):
-        excluded = toggle.get_active()
-        orig_incompatible = self.configurator.getConfVar('INCOMPATIBLE_LICENSE')
-        new_incompatible = ""
-        if excluded:
-            if not orig_incompatible:
-                new_incompatible = "GPLv3"
-            elif not orig_incompatible.find('GPLv3'):
-                new_incompatible = "%s GPLv3" % orig_incompatible
-        else:
-            new_incompatible = orig_incompatible.replace('GPLv3', '')
-
-        if new_incompatible != orig_incompatible:
-            self.handler.set_incompatible_license(new_incompatible)
-            self.configurator.setConfVar('INCOMPATIBLE_LICENSE', new_incompatible)
-            self.reload_required = True
-
-    def change_bb_threads_cb(self, spinner):
-        val = spinner.get_value_as_int()
-        self.handler.set_bbthreads(val)
-        self.configurator.setConfVar('BB_NUMBER_THREADS', val)
-
-    def change_make_threads_cb(self, spinner):
-        val = spinner.get_value_as_int()
-        self.handler.set_pmake(val)
-        self.configurator.setConfVar('PARALLEL_MAKE', "-j %s" % val)
-
-    def toggle_toolchain_cb(self, check):
-        enabled = check.get_active()
-        toolchain = '0'
-        if enabled:
-            toolchain = '1'
-        self.handler.toggle_toolchain(enabled)
-        self.configurator.setConfVar('HOB_BUILD_TOOLCHAIN', toolchain)
-
-    def toggle_headers_cb(self, check):
-        enabled = check.get_active()
-        headers = '0'
-        if enabled:
-            headers = '1'
-        self.handler.toggle_toolchain_headers(enabled)
-        self.configurator.setConfVar('HOB_BUILD_TOOLCHAIN_HEADERS', headers)
-
-    def set_parent_window(self, parent):
-        self.set_transient_for(parent)
-
-    def write_changes(self):
-        self.configurator.writeConf()
-
-    def prefs_response_cb(self, dialog, response):
-        if self.reload_required:
-            glib.idle_add(self.handler.reload_data)
-            self.reload_required = False
-
-    def __init__(self, configurator, handler, curr_sdk_mach, curr_distro, pclass,
-                 pmake, bbthread, selected_image_types, all_image_types,
-                 gplv3disabled, build_toolchain, build_toolchain_headers):
-        """
-        """
-        gtk.Dialog.__init__(self, "Preferences", None,
-                            gtk.DIALOG_DESTROY_WITH_PARENT,
-                            (gtk.STOCK_CLOSE, gtk.RESPONSE_OK))
-
-        self.set_border_width(6)
-        self.vbox.set_property("spacing", 12)
-        self.action_area.set_property("spacing", 12)
-        self.action_area.set_property("border-width", 6)
-
-        self.handler = handler
-        self.configurator = configurator
-
-        self.curr_sdk_mach = curr_sdk_mach
-        self.curr_distro = curr_distro
-        self.curr_package_format = pclass
-        self.pmake = pmake
-        self.bbthread = bbthread
-        self.selected_image_types = selected_image_types.split(" ")
-        self.gplv3disabled = gplv3disabled
-        self.build_toolchain = build_toolchain
-        self.build_toolchain_headers = build_toolchain_headers
-
-        self.reload_required = False
-        self.distro_handler_id = None
-        self.sdk_machine_handler_id = None
-        self.package_handler_id = None
-
-        left = gtk.SizeGroup(gtk.SIZE_GROUP_HORIZONTAL)
-        right = gtk.SizeGroup(gtk.SIZE_GROUP_HORIZONTAL)
-
-        label = gtk.Label()
-        label.set_markup("<b>Policy</b>")
-        label.show()
-        frame = gtk.Frame()
-        frame.set_label_widget(label)
-        frame.set_shadow_type(gtk.SHADOW_NONE)
-        frame.show()
-        self.vbox.pack_start(frame)
-        pbox = gtk.VBox(False, 12)
-        pbox.show()
-        frame.add(pbox)
-        hbox = gtk.HBox(False, 12)
-        hbox.show()
-        pbox.pack_start(hbox, expand=False, fill=False, padding=6)
-        # Distro selector
-        label = gtk.Label("Distribution:")
-        label.show()
-        hbox.pack_start(label, expand=False, fill=False, padding=6)
-        self.distro_combo = gtk.combo_box_new_text()
-        self.distro_combo.set_tooltip_text("Select the Yocto distribution you would like to use")
-        self.distro_combo.show()
-        hbox.pack_start(self.distro_combo, expand=False, fill=False, padding=6)
-        # Exclude GPLv3
-        check = gtk.CheckButton("Exclude GPLv3 packages")
-        check.set_tooltip_text("Check this box to prevent GPLv3 packages from being included in your image")
-        check.show()
-        check.set_active(self.gplv3disabled)
-        check.connect("toggled", self.include_gplv3_cb)
-        hbox.pack_start(check, expand=False, fill=False, padding=6)
-        hbox = gtk.HBox(False, 12)
-        hbox.show()
-        pbox.pack_start(hbox, expand=False, fill=False, padding=6)
-        # Package format selector
-        label = gtk.Label("Package format:")
-        label.show()
-        hbox.pack_start(label, expand=False, fill=False, padding=6)
-        self.package_combo = gtk.combo_box_new_text()
-        self.package_combo.set_tooltip_text("""The package format is that used in creation
- of the root filesystem and also dictates the package manager used in your image""")
-        self.package_combo.show()
-        hbox.pack_start(self.package_combo, expand=False, fill=False, padding=6)
-        if all_image_types:
-            # Image output type selector
-            label = gtk.Label("Image output types:")
-            label.show()
-            hbox.pack_start(label, expand=False, fill=False, padding=6)
-            chk_cnt = 3
-            for it in all_image_types.split(" "):
-                chk_cnt = chk_cnt + 1
-                if chk_cnt % 6 == 0:
-                    hbox = gtk.HBox(False, 12)
-                    hbox.show()
-                    pbox.pack_start(hbox, expand=False, fill=False, padding=6)
-                chk = gtk.CheckButton(it)
-                if it in self.selected_image_types:
-                    chk.set_active(True)
-                chk.set_tooltip_text("Build an %s image" % it)
-                chk.connect("toggled", self.output_type_toggled_cb, handler)
-                chk.show()
-                hbox.pack_start(chk, expand=False, fill=False, padding=3)
-        # BitBake
-        label = gtk.Label()
-        label.set_markup("<b>BitBake</b>")
-        label.show()
-        frame = gtk.Frame()
-        frame.set_label_widget(label)
-        frame.set_shadow_type(gtk.SHADOW_NONE)
-        frame.show()
-        self.vbox.pack_start(frame)
-        pbox = gtk.VBox(False, 12)
-        pbox.show()
-        frame.add(pbox)
-        hbox = gtk.HBox(False, 12)
-        hbox.show()
-        pbox.pack_start(hbox, expand=False, fill=False, padding=6)
-        label = gtk.Label("BitBake threads:")
-        label.show()
-        # NOTE: may be a good idea in future to intelligently cap the maximum
-        # values but we need more data to make an educated decision, for now
-        # set a high maximum as a value for upper bounds is required by the
-        # gtk.Adjustment
-        spin_max = 30 # seems like a high enough arbitrary number
-        hbox.pack_start(label, expand=False, fill=False, padding=6)
-        bbadj = gtk.Adjustment(value=self.bbthread, lower=1, upper=spin_max, step_incr=1)
-        bbspinner = gtk.SpinButton(adjustment=bbadj, climb_rate=1, digits=0)
-        bbspinner.show()
-        bbspinner.connect("value-changed", self.change_bb_threads_cb)
-        hbox.pack_start(bbspinner, expand=False, fill=False, padding=6)
-        label = gtk.Label("Make threads:")
-        label.show()
-        hbox.pack_start(label, expand=False, fill=False, padding=6)
-        madj = gtk.Adjustment(value=self.pmake, lower=1, upper=spin_max, step_incr=1)
-        makespinner = gtk.SpinButton(adjustment=madj, climb_rate=1, digits=0)
-        makespinner.connect("value-changed", self.change_make_threads_cb)
-        makespinner.show()
-        hbox.pack_start(makespinner, expand=False, fill=False, padding=6)
-        # Toolchain
-        label = gtk.Label()
-        label.set_markup("<b>External Toolchain</b>")
-        label.show()
-        frame = gtk.Frame()
-        frame.set_label_widget(label)
-        frame.set_shadow_type(gtk.SHADOW_NONE)
-        frame.show()
-        self.vbox.pack_start(frame)
-        pbox = gtk.VBox(False, 12)
-        pbox.show()
-        frame.add(pbox)
-        hbox = gtk.HBox(False, 12)
-        hbox.show()
-        pbox.pack_start(hbox, expand=False, fill=False, padding=6)
-        toolcheck = gtk.CheckButton("Build external development toolchain with image")
-        toolcheck.show()
-        toolcheck.set_active(self.build_toolchain)
-        toolcheck.connect("toggled", self.toggle_toolchain_cb)
-        hbox.pack_start(toolcheck, expand=False, fill=False, padding=6)
-        hbox = gtk.HBox(False, 12)
-        hbox.show()
-        pbox.pack_start(hbox, expand=False, fill=False, padding=6)
-        label = gtk.Label("Toolchain host:")
-        label.show()
-        hbox.pack_start(label, expand=False, fill=False, padding=6)
-        self.sdk_machine_combo = gtk.combo_box_new_text()
-        self.sdk_machine_combo.set_tooltip_text("Select the host architecture of the external machine")
-        self.sdk_machine_combo.show()
-        hbox.pack_start(self.sdk_machine_combo, expand=False, fill=False, padding=6)
-        # headerscheck = gtk.CheckButton("Include development headers with toolchain")
-        # headerscheck.show()
-        # headerscheck.set_active(self.build_toolchain_headers)
-        # headerscheck.connect("toggled", self.toggle_headers_cb)
-        # hbox.pack_start(headerscheck, expand=False, fill=False, padding=6)
-        self.connect("response", self.prefs_response_cb)

bitbake/lib/bb/ui/crumbs/imageconfigurationpage.py

@@ -0,0 +1,407 @@
+#!/usr/bin/env python
+#
+# BitBake Graphical GTK User Interface
+#
+# Copyright (C) 2012        Intel Corporation
+#
+# Authored by Dongxiao Xu <dongxiao.xu@intel.com>
+# Authored by Shane Wang <shane.wang@intel.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import gtk
+import glib
+from bb.ui.crumbs.progressbar import HobProgressBar
+from bb.ui.crumbs.hobcolor import HobColors
+from bb.ui.crumbs.hobwidget import hic, HobImageButton, HobInfoButton, HobAltButton, HobButton
+from bb.ui.crumbs.hoblistmodel import RecipeListModel
+from bb.ui.crumbs.hobpages import HobPage
+
+#
+# ImageConfigurationPage
+#
+class ImageConfigurationPage (HobPage):
+
+    def __init__(self, builder):
+        super(ImageConfigurationPage, self).__init__(builder, "Image configuration")
+
+        self.image_combo_id = None
+        # we use machine_combo_changed_by_manual to identify the machine is changed by code
+        # or by manual. If by manual, all user's recipe selection and package selection are
+        # cleared.
+        self.machine_combo_changed_by_manual = True
+        self.create_visual_elements()
+
+    def create_visual_elements(self):
+        # create visual elements
+        self.toolbar = gtk.Toolbar()
+        self.toolbar.set_orientation(gtk.ORIENTATION_HORIZONTAL)
+        self.toolbar.set_style(gtk.TOOLBAR_BOTH)
+
+        template_button = self.append_toolbar_button(self.toolbar,
+            "Templates",
+            hic.ICON_TEMPLATES_DISPLAY_FILE,
+            hic.ICON_TEMPLATES_HOVER_FILE,
+            "Load a previously saved template",
+            self.template_button_clicked_cb)
+        my_images_button = self.append_toolbar_button(self.toolbar,
+            "Images",
+            hic.ICON_IMAGES_DISPLAY_FILE,
+            hic.ICON_IMAGES_HOVER_FILE,
+            "Open previously built images",
+            self.my_images_button_clicked_cb)
+        settings_button = self.append_toolbar_button(self.toolbar,
+            "Settings",
+            hic.ICON_SETTINGS_DISPLAY_FILE,
+            hic.ICON_SETTINGS_HOVER_FILE,
+            "View additional build settings",
+            self.settings_button_clicked_cb)
+
+        self.config_top_button = self.add_onto_top_bar(self.toolbar)
+
+        self.gtable = gtk.Table(40, 40, True)
+        self.create_config_machine()
+        self.create_config_baseimg()
+        self.config_build_button = self.create_config_build_button()
+
+    def _remove_all_widget(self):
+        children = self.gtable.get_children() or []
+        for child in children:
+            self.gtable.remove(child)
+        children = self.box_group_area.get_children() or []
+        for child in children:
+            self.box_group_area.remove(child)
+        children = self.get_children() or []
+        for child in children:
+            self.remove(child)
+
+    def _pack_components(self, pack_config_build_button = False):
+        self._remove_all_widget()
+        self.pack_start(self.config_top_button, expand=False, fill=False)
+        self.pack_start(self.group_align, expand=True, fill=True)
+
+        self.box_group_area.pack_start(self.gtable, expand=True, fill=True)
+        if pack_config_build_button:
+            self.box_group_area.pack_end(self.config_build_button, expand=False, fill=False)
+        else:
+            box = gtk.HBox(False, 6)
+            box.show()
+            subbox = gtk.HBox(False, 0)
+            subbox.set_size_request(205, 49)
+            subbox.show()
+            box.add(subbox)
+            self.box_group_area.pack_end(box, False, False)
+
+    def show_machine(self):
+        self.progress_bar.reset()
+        self._pack_components(pack_config_build_button = False)
+        self.set_config_machine_layout(show_progress_bar = False)
+        self.show_all()
+
+    def update_progress_bar(self, title, fraction, status=None):
+        self.progress_bar.update(fraction)
+        self.progress_bar.set_title(title)
+        self.progress_bar.set_rcstyle(status)
+
+    def show_info_populating(self):
+        self._pack_components(pack_config_build_button = False)
+        self.set_config_machine_layout(show_progress_bar = True)
+        self.show_all()
+
+    def show_info_populated(self):
+        self.progress_bar.reset()
+        self._pack_components(pack_config_build_button = False)
+        self.set_config_machine_layout(show_progress_bar = False)
+        self.set_config_baseimg_layout()
+        self.show_all()
+
+    def show_baseimg_selected(self):
+        self.progress_bar.reset()
+        self._pack_components(pack_config_build_button = True)
+        self.set_config_machine_layout(show_progress_bar = False)
+        self.set_config_baseimg_layout()
+        self.set_rcppkg_layout()
+        self.show_all()
+
+    def create_config_machine(self):
+        self.machine_title = gtk.Label()
+        self.machine_title.set_alignment(0.0, 0.5)
+        mark = "<span %s>Select a machine</span>" % self.span_tag('x-large', 'bold')
+        self.machine_title.set_markup(mark)
+
+        self.machine_title_desc = gtk.Label()
+        self.machine_title_desc.set_alignment(0.0, 0.5)
+        mark = ("<span %s>Your selection is the profile of the target machine for which you"
+        " are building the image.\n</span>") % (self.span_tag('medium'))
+        self.machine_title_desc.set_markup(mark)
+
+        self.machine_combo = gtk.combo_box_new_text()
+        self.machine_combo.set_wrap_width(1)
+        self.machine_combo.connect("changed", self.machine_combo_changed_cb)
+
+        icon_file = hic.ICON_LAYERS_DISPLAY_FILE
+        hover_file = hic.ICON_LAYERS_HOVER_FILE
+        self.layer_button = HobImageButton("Layers", "Add support for machines, software, etc.",
+                                icon_file, hover_file)
+        self.layer_button.connect("clicked", self.layer_button_clicked_cb)
+
+        markup = "Layers are a powerful mechanism to extend the Yocto Project "
+        markup += "with your own functionality.\n"
+        markup += "For more on layers, check the <a href=\""
+        markup += "http://www.yoctoproject.org/docs/current/dev-manual/"
+        markup += "dev-manual.html#understanding-and-using-layers\">reference manual</a>."
+        self.layer_info_icon = HobInfoButton(markup, self.get_parent())
+
+        self.progress_box = gtk.HBox(False, 6)
+        self.progress_bar = HobProgressBar()
+        self.progress_box.pack_start(self.progress_bar, expand=True, fill=True)
+        self.stop_button = HobAltButton("Stop")
+        self.stop_button.connect("clicked", self.stop_button_clicked_cb)
+        self.progress_box.pack_end(self.stop_button, expand=False, fill=False)
+
+        self.machine_separator = gtk.HSeparator()
+
+    def set_config_machine_layout(self, show_progress_bar = False):
+        self.gtable.attach(self.machine_title, 0, 40, 0, 4)
+        self.gtable.attach(self.machine_title_desc, 0, 40, 4, 6)
+        self.gtable.attach(self.machine_combo, 0, 12, 7, 10)
+        self.gtable.attach(self.layer_button, 14, 36, 7, 12)
+        self.gtable.attach(self.layer_info_icon, 36, 40, 7, 11)
+        if show_progress_bar:
+            self.gtable.attach(self.progress_box, 0, 40, 15, 19)
+        self.gtable.attach(self.machine_separator, 0, 40, 13, 14)
+
+    def create_config_baseimg(self):
+        self.image_title = gtk.Label()
+        self.image_title.set_alignment(0, 1.0)
+        mark = "<span %s>Select a base image</span>" % self.span_tag('x-large', 'bold')
+        self.image_title.set_markup(mark)
+
+        self.image_title_desc = gtk.Label()
+        self.image_title_desc.set_alignment(0, 0.5)
+        mark = ("<span %s>Base images are a starting point for the type of image you want. "
+                "You can build them as \n"
+                "they are or customize them to your specific needs.\n</span>") % self.span_tag('medium')
+        self.image_title_desc.set_markup(mark)
+
+        self.image_combo = gtk.combo_box_new_text()
+        self.image_combo.set_wrap_width(1)
+        self.image_combo_id = self.image_combo.connect("changed", self.image_combo_changed_cb)
+
+        self.image_desc = gtk.Label()
+        self.image_desc.set_alignment(0.0, 0.5)
+        self.image_desc.set_line_wrap(True)
+
+        # button to view recipes
+        icon_file = hic.ICON_RCIPE_DISPLAY_FILE
+        hover_file = hic.ICON_RCIPE_HOVER_FILE
+        self.view_recipes_button = HobImageButton("View recipes",
+                                        "Add/remove recipes and tasks",
+                                        icon_file, hover_file)
+        self.view_recipes_button.connect("clicked", self.view_recipes_button_clicked_cb)
+
+        # button to view packages
+        icon_file = hic.ICON_PACKAGES_DISPLAY_FILE
+        hover_file = hic.ICON_PACKAGES_HOVER_FILE
+        self.view_packages_button = HobImageButton("View packages",
+                                        "Add/remove previously built packages",
+                                        icon_file, hover_file)
+        self.view_packages_button.connect("clicked", self.view_packages_button_clicked_cb)
+
+        self.image_separator = gtk.HSeparator()
+
+    def set_config_baseimg_layout(self):
+        self.gtable.attach(self.image_title, 0, 40, 15, 17)
+        self.gtable.attach(self.image_title_desc, 0, 40, 18, 22)
+        self.gtable.attach(self.image_combo, 0, 12, 23, 26)
+        self.gtable.attach(self.image_desc, 13, 38, 23, 28)
+        self.gtable.attach(self.image_separator, 0, 40, 35, 36)
+
+    def set_rcppkg_layout(self):
+        self.gtable.attach(self.view_recipes_button, 0, 20, 28, 33)
+        self.gtable.attach(self.view_packages_button, 20, 40, 28, 33)
+
+    def create_config_build_button(self):
+        # Create the "Build packages" and "Build image" buttons at the bottom
+        button_box = gtk.HBox(False, 6)
+
+        # create button "Build image"
+        just_bake_button = HobButton("Build image")
+        just_bake_button.set_size_request(205, 49)
+        just_bake_button.set_tooltip_text("Build target image")
+        just_bake_button.connect("clicked", self.just_bake_button_clicked_cb)
+        button_box.pack_end(just_bake_button, expand=False, fill=False)
+
+        label = gtk.Label(" or ")
+        button_box.pack_end(label, expand=False, fill=False)
+
+        # create button "Build Packages"
+        build_packages_button = HobAltButton("Build packages")
+        build_packages_button.connect("clicked", self.build_packages_button_clicked_cb)
+        build_packages_button.set_tooltip_text("Build recipes into packages")
+        button_box.pack_end(build_packages_button, expand=False, fill=False)
+
+        return button_box
+
+    def stop_button_clicked_cb(self, button):
+        self.builder.cancel_parse_sync()
+
+    def machine_combo_changed_cb(self, machine_combo):
+        combo_item = machine_combo.get_active_text()
+        if not combo_item:
+            return
+
+        self.builder.configuration.curr_mach = combo_item
+        if self.machine_combo_changed_by_manual:
+            self.builder.configuration.clear_selection()
+        # reset machine_combo_changed_by_manual
+        self.machine_combo_changed_by_manual = True
+
+        # Do reparse recipes
+        self.builder.populate_recipe_package_info_async()
+
+    def update_machine_combo(self):
+        all_machines = self.builder.parameters.all_machines
+
+        model = self.machine_combo.get_model()
+        model.clear()
+        for machine in all_machines:
+            self.machine_combo.append_text(machine)
+        self.machine_combo.set_active(-1)
+
+    def switch_machine_combo(self):
+        self.machine_combo_changed_by_manual = False
+        model = self.machine_combo.get_model()
+        active = 0
+        while active < len(model):
+            if model[active][0] == self.builder.configuration.curr_mach:
+                self.machine_combo.set_active(active)
+                return
+            active += 1
+        self.machine_combo.set_active(-1)
+
+    def update_image_desc(self, selected_image):
+        desc = ""
+        if selected_image and selected_image in self.builder.recipe_model.pn_path.keys():
+            image_path = self.builder.recipe_model.pn_path[selected_image]
+            image_iter = self.builder.recipe_model.get_iter(image_path)
+            desc = self.builder.recipe_model.get_value(image_iter, self.builder.recipe_model.COL_DESC)
+
+        mark = ("<span %s>%s</span>\n") % (self.span_tag('small'), desc)
+        self.image_desc.set_markup(mark)
+
+    def image_combo_changed_idle_cb(self, selected_image, selected_recipes, selected_packages):
+        self.builder.update_recipe_model(selected_image, selected_recipes)
+        self.builder.update_package_model(selected_packages)
+        self.builder.window_sensitive(True)
+
+    def image_combo_changed_cb(self, combo):
+        self.builder.window_sensitive(False)
+        selected_image = self.image_combo.get_active_text()
+        if not selected_image:
+            return
+
+        self.builder.customized = False
+
+        selected_recipes = []
+
+        image_path = self.builder.recipe_model.pn_path[selected_image]
+        image_iter = self.builder.recipe_model.get_iter(image_path)
+        selected_packages = self.builder.recipe_model.get_value(image_iter, self.builder.recipe_model.COL_INSTALL).split()
+        self.update_image_desc(selected_image)
+
+        self.builder.recipe_model.reset()
+        self.builder.package_model.reset()
+
+        self.show_baseimg_selected()
+
+        glib.idle_add(self.image_combo_changed_idle_cb, selected_image, selected_recipes, selected_packages)
+
+    def _image_combo_connect_signal(self):
+        if not self.image_combo_id:
+            self.image_combo_id = self.image_combo.connect("changed", self.image_combo_changed_cb)
+
+    def _image_combo_disconnect_signal(self):
+        if self.image_combo_id:
+            self.image_combo.disconnect(self.image_combo_id)
+            self.image_combo_id = None
+
+    def update_image_combo(self, recipe_model, selected_image):
+        # Update the image combo according to the images in the recipe_model
+        # populate image combo
+        filter = {RecipeListModel.COL_TYPE : ['image']}
+        image_model = recipe_model.tree_model(filter)
+        active = -1
+        cnt = 0
+
+        it = image_model.get_iter_first()
+        self._image_combo_disconnect_signal()
+        model = self.image_combo.get_model()
+        model.clear()
+        # append and set active
+        while it:
+            path = image_model.get_path(it)
+            it = image_model.iter_next(it)
+            image_name = image_model[path][recipe_model.COL_NAME]
+            if image_name == self.builder.recipe_model.__dummy_image__:
+                continue
+            self.image_combo.append_text(image_name)
+            if image_name == selected_image:
+                active = cnt
+            cnt = cnt + 1
+        self.image_combo.append_text(self.builder.recipe_model.__dummy_image__)
+        if selected_image == self.builder.recipe_model.__dummy_image__:
+            active = cnt
+
+        self.image_combo.set_active(-1)
+        self.image_combo.set_active(active)
+
+        if active != -1:
+            self.show_baseimg_selected()
+
+        self._image_combo_connect_signal()
+
+    def layer_button_clicked_cb(self, button):
+        # Create a layer selection dialog
+        self.builder.show_layer_selection_dialog()
+
+    def view_recipes_button_clicked_cb(self, button):
+        self.builder.show_recipes()
+
+    def view_packages_button_clicked_cb(self, button):
+        self.builder.show_packages()
+
+    def just_bake_button_clicked_cb(self, button):
+        self.builder.just_bake()
+
+    def build_packages_button_clicked_cb(self, button):
+        self.builder.build_packages()
+
+    def template_button_clicked_cb(self, button):
+        response, path = self.builder.show_load_template_dialog()
+        if not response:
+            return
+        if path:
+            self.builder.load_template(path)
+
+    def my_images_button_clicked_cb(self, button):
+        self.builder.show_load_my_images_dialog()
+
+    def settings_button_clicked_cb(self, button):
+        # Create an advanced settings dialog
+        response, settings_changed = self.builder.show_adv_settings_dialog()
+        if not response:
+            return
+        if settings_changed:
+            self.builder.reparse_post_adv_settings()

bitbake/lib/bb/ui/crumbs/imagedetailspage.py

@@ -0,0 +1,459 @@
+#!/usr/bin/env python
+#
+# BitBake Graphical GTK User Interface
+#
+# Copyright (C) 2012        Intel Corporation
+#
+# Authored by Dongxiao Xu <dongxiao.xu@intel.com>
+# Authored by Shane Wang <shane.wang@intel.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import gobject
+import gtk
+from bb.ui.crumbs.hobcolor import HobColors
+from bb.ui.crumbs.hobwidget import hic, HobViewTable, HobAltButton, HobButton
+from bb.ui.crumbs.hobpages import HobPage
+
+#
+# ImageDetailsPage
+#
+class ImageDetailsPage (HobPage):
+
+    __columns__ = [{
+            'col_name' : 'Image name',
+            'col_id'   : 0,
+            'col_style': 'text',
+            'col_min'  : 500,
+            'col_max'  : 500
+        }, {
+            'col_name' : 'Image size',
+            'col_id'   : 1,
+            'col_style': 'text',
+            'col_min'  : 100,
+            'col_max'  : 100
+        }, {
+            'col_name' : 'Select',
+            'col_id'   : 2,
+            'col_style': 'radio toggle',
+            'col_min'  : 100,
+            'col_max'  : 100
+    }]
+
+    class DetailBox (gtk.EventBox):
+        def __init__(self, widget = None, varlist = None, vallist = None, icon = None, button = None, color = HobColors.LIGHT_GRAY):
+            gtk.EventBox.__init__(self)
+
+            # set color
+            style = self.get_style().copy()
+            style.bg[gtk.STATE_NORMAL] = self.get_colormap().alloc_color(color, False, False)
+            self.set_style(style)
+
+            self.hbox = gtk.HBox()
+            self.hbox.set_border_width(15)
+            self.add(self.hbox)
+
+            if widget:
+                row = 1
+            elif varlist and vallist:
+                # pack the icon and the text on the left
+                row = len(varlist)
+            self.table = gtk.Table(row, 20, True)
+            self.table.set_size_request(100, -1)
+            self.hbox.pack_start(self.table, expand=True, fill=True, padding=15)
+
+            colid = 0
+            self.line_widgets = {}
+            if icon:
+                self.table.attach(icon, colid, colid + 2, 0, 1)
+                colid = colid + 2
+            if widget:
+                self.table.attach(widget, colid, 20, 0, 1)
+            elif varlist and vallist:
+                for line in range(0, row):
+                    self.line_widgets[varlist[line]] = self.text2label(varlist[line], vallist[line])
+                    self.table.attach(self.line_widgets[varlist[line]], colid, 20, line, line + 1)
+
+            # pack the button on the right
+            if button:
+                self.hbox.pack_end(button, expand=False, fill=False)
+
+        def update_line_widgets(self, variable, value):
+            if len(self.line_widgets) == 0:
+                return
+            if not isinstance(self.line_widgets[variable], gtk.Label):
+                return
+            self.line_widgets[variable].set_markup(self.format_line(variable, value))
+
+        def format_line(self, variable, value):
+            markup = "<span weight=\'bold\'>%s</span>" % variable
+            markup += "<span weight=\'normal\' foreground=\'#1c1c1c\' font_desc=\'14px\'>%s</span>" % value
+            return markup
+
+        def text2label(self, variable, value):
+            # append the name:value to the left box
+            # such as "Name: hob-core-minimal-variant-2011-12-15-beagleboard"
+            label = gtk.Label()
+            label.set_alignment(0.0, 0.5)
+            label.set_markup(self.format_line(variable, value))
+            return label
+
+    def __init__(self, builder):
+        super(ImageDetailsPage, self).__init__(builder, "Image details")
+
+        self.image_store = gtk.ListStore(gobject.TYPE_STRING, gobject.TYPE_STRING, gobject.TYPE_BOOLEAN)
+        self.button_ids = {}
+        self.details_bottom_buttons = gtk.HBox(False, 6)
+        self.create_visual_elements()
+
+    def create_visual_elements(self):
+        # create visual elements
+        # create the toolbar
+        self.toolbar = gtk.Toolbar()
+        self.toolbar.set_orientation(gtk.ORIENTATION_HORIZONTAL)
+        self.toolbar.set_style(gtk.TOOLBAR_BOTH)
+
+        template_button = self.append_toolbar_button(self.toolbar,
+            "Templates",
+            hic.ICON_TEMPLATES_DISPLAY_FILE,
+            hic.ICON_TEMPLATES_HOVER_FILE,
+            "Load a previously saved template",
+            self.template_button_clicked_cb)
+        my_images_button = self.append_toolbar_button(self.toolbar,
+            "Images",
+            hic.ICON_IMAGES_DISPLAY_FILE,
+            hic.ICON_IMAGES_HOVER_FILE,
+            "Open previously built images",
+            self.my_images_button_clicked_cb)
+        settings_button = self.append_toolbar_button(self.toolbar,
+            "Settings",
+            hic.ICON_SETTINGS_DISPLAY_FILE,
+            hic.ICON_SETTINGS_HOVER_FILE,
+            "View additional build settings",
+            self.settings_button_clicked_cb)
+
+        self.details_top_buttons = self.add_onto_top_bar(self.toolbar)
+
+    def _remove_all_widget(self):
+        children = self.get_children() or []
+        for child in children:
+            self.remove(child)
+        children = self.box_group_area.get_children() or []
+        for child in children:
+            self.box_group_area.remove(child)
+        children = self.details_bottom_buttons.get_children() or []
+        for child in children:
+            self.details_bottom_buttons.remove(child)
+
+    def show_page(self, step):
+        build_succeeded = (step == self.builder.IMAGE_GENERATED)
+        image_addr = self.builder.parameters.image_addr
+        image_names = self.builder.parameters.image_names
+        if build_succeeded:
+            machine = self.builder.configuration.curr_mach
+            base_image = self.builder.recipe_model.get_selected_image()
+            layers = self.builder.configuration.layers
+            pkg_num = "%s" % len(self.builder.package_model.get_selected_packages())
+        else:
+            pkg_num = "N/A"
+
+        # remove
+        for button_id, button in self.button_ids.items():
+            button.disconnect(button_id)
+        self._remove_all_widget()
+        # repack
+        self.pack_start(self.details_top_buttons, expand=False, fill=False)
+        self.pack_start(self.group_align, expand=True, fill=True)
+
+        self.build_result = None
+        if build_succeeded:
+            # building is the previous step
+            icon = gtk.Image()
+            pixmap_path = hic.ICON_INDI_CONFIRM_FILE
+            color = HobColors.RUNNING
+            pix_buffer = gtk.gdk.pixbuf_new_from_file(pixmap_path)
+            icon.set_from_pixbuf(pix_buffer)
+            varlist = [""]
+            vallist = ["Your image is ready"]
+            self.build_result = self.DetailBox(varlist=varlist, vallist=vallist, icon=icon, color=color)
+            self.box_group_area.pack_start(self.build_result, expand=False, fill=False)
+
+        # create the buttons at the bottom first because the buttons are used in apply_button_per_image()
+        if build_succeeded:
+            self.buttonlist = ["Build new image", "Save as template", "Run image", "Deploy image"]
+        else: # get to this page from "My images"
+            self.buttonlist = ["Build new image", "Run image", "Deploy image"]
+
+        # Name
+        self.image_store.clear()
+        default_toggled = False
+        default_image_size = 0
+        i = 0
+        for image_name in image_names:
+            image_size = HobPage._size_to_string(os.stat(os.path.join(image_addr, image_name)).st_size)
+            if not default_toggled:
+                default_toggled = (self.test_type_runnable(image_name) and self.test_mach_runnable(image_name)) \
+                    or self.test_deployable(image_name)
+                if i == (len(image_names) - 1):
+                    default_toggled = True
+                self.image_store.set(self.image_store.append(), 0, image_name, 1, image_size, 2, default_toggled)
+                if default_toggled:
+                    default_image_size = image_size
+                    self.create_bottom_buttons(self.buttonlist, image_name)
+            else:
+                self.image_store.set(self.image_store.append(), 0, image_name, 1, image_size, 2, False)
+            i = i + 1
+        image_table = HobViewTable(self.__columns__)
+        image_table.set_model(self.image_store)
+        image_table.connect("toggled", self.toggled_cb)
+        view_files_button = HobAltButton("View files")
+        view_files_button.connect("clicked", self.view_files_clicked_cb, image_addr)
+        view_files_button.set_tooltip_text("Open the directory containing the image files")
+        self.image_detail = self.DetailBox(widget=image_table, button=view_files_button)
+        self.box_group_area.pack_start(self.image_detail, expand=True, fill=True)
+
+        # Machine, Base image and Layers
+        layer_num_limit = 15
+        varlist = ["Machine: ", "Base image: ", "Layers: "]
+        vallist = []
+        self.setting_detail = None
+        if build_succeeded:
+            vallist.append(machine)
+            vallist.append(base_image)
+            i = 0
+            for layer in layers:
+                varlist.append(" - ")
+                if i > layer_num_limit:
+                    break
+                i += 1
+            vallist.append("")
+            i = 0
+            for layer in layers:
+                if i > layer_num_limit:
+                    break
+                elif i == layer_num_limit:
+                    vallist.append("and more...")
+                else:
+                    vallist.append(layer)
+                i += 1
+
+            edit_config_button = HobAltButton("Edit configuration")
+            edit_config_button.set_tooltip_text("Edit machine, base image and recipes")
+            edit_config_button.connect("clicked", self.edit_config_button_clicked_cb)
+            self.setting_detail = self.DetailBox(varlist=varlist, vallist=vallist, button=edit_config_button)
+            self.box_group_area.pack_start(self.setting_detail, expand=False, fill=False)
+
+        # Packages included, and Total image size
+        varlist = ["Packages included: ", "Total image size: "]
+        vallist = []
+        vallist.append(pkg_num)
+        vallist.append(default_image_size)
+        if build_succeeded:
+            edit_packages_button = HobAltButton("Edit packages")
+            edit_packages_button.set_tooltip_text("Edit the packages included in your image")
+            edit_packages_button.connect("clicked", self.edit_packages_button_clicked_cb)
+        else: # get to this page from "My images"
+            edit_packages_button = None
+        self.package_detail = self.DetailBox(varlist=varlist, vallist=vallist, button=edit_packages_button)
+        self.box_group_area.pack_start(self.package_detail, expand=False, fill=False)
+
+        # pack the buttons at the bottom, at this time they are already created.
+        self.box_group_area.pack_end(self.details_bottom_buttons, expand=False, fill=False)
+
+        self.show_all()
+
+    def view_files_clicked_cb(self, button, image_addr):
+        os.system("xdg-open /%s" % image_addr)
+
+    def refresh_package_detail_box(self, image_size):
+        self.package_detail.update_line_widgets("Total image size: ", image_size)
+
+    def test_type_runnable(self, image_name):
+        type_runnable = False
+        for t in self.builder.parameters.runnable_image_types:
+            if image_name.endswith(t):
+                type_runnable = True
+                break
+        return type_runnable
+
+    def test_mach_runnable(self, image_name):
+        mach_runnable = False
+        for t in self.builder.parameters.runnable_machine_patterns:
+            if t in image_name:
+                mach_runnable = True
+                break
+        return mach_runnable
+
+    def test_deployable(self, image_name):
+        deployable = False
+        for t in self.builder.parameters.deployable_image_types:
+            if image_name.endswith(t):
+                deployable = True
+                break
+        return deployable
+
+    def toggled_cb(self, table, cell, path, columnid, tree):
+        model = tree.get_model()
+        if not model:
+            return
+        iter = model.get_iter_first()
+        while iter:
+            rowpath = model.get_path(iter)
+            model[rowpath][columnid] = False
+            iter = model.iter_next(iter)
+
+        model[path][columnid] = True
+        self.refresh_package_detail_box(model[path][1])
+
+        image_name = model[path][0]
+
+        # remove
+        for button_id, button in self.button_ids.items():
+            button.disconnect(button_id)
+        self._remove_all_widget()
+        # repack
+        self.pack_start(self.details_top_buttons, expand=False, fill=False)
+        self.pack_start(self.group_align, expand=True, fill=True)
+        if self.build_result:
+            self.box_group_area.pack_start(self.build_result, expand=False, fill=False)
+        self.box_group_area.pack_start(self.image_detail, expand=True, fill=True)
+        if self.setting_detail:
+            self.box_group_area.pack_start(self.setting_detail, expand=False, fill=False)
+        self.box_group_area.pack_start(self.package_detail, expand=False, fill=False)
+        self.create_bottom_buttons(self.buttonlist, image_name)
+        self.box_group_area.pack_end(self.details_bottom_buttons, expand=False, fill=False)
+        self.show_all()
+
+    def create_bottom_buttons(self, buttonlist, image_name):
+        # Create the buttons at the bottom
+        created = False
+        packed = False
+        self.button_ids = {}
+
+        # create button "Deploy image"
+        name = "Deploy image"
+        if name in buttonlist and self.test_deployable(image_name):
+            deploy_button = HobButton('Deploy image')
+            deploy_button.set_size_request(205, 49)
+            deploy_button.set_tooltip_text("Burn a live image to a USB drive or flash memory")
+            deploy_button.set_flags(gtk.CAN_DEFAULT)
+            button_id = deploy_button.connect("clicked", self.deploy_button_clicked_cb)
+            self.button_ids[button_id] = deploy_button
+            self.details_bottom_buttons.pack_end(deploy_button, expand=False, fill=False)
+            created = True
+            packed = True
+
+        name = "Run image"
+        if name in buttonlist and self.test_type_runnable(image_name) and self.test_mach_runnable(image_name):
+            if created == True:
+                # separator
+                label = gtk.Label(" or ")
+                self.details_bottom_buttons.pack_end(label, expand=False, fill=False)
+
+                # create button "Run image"
+                run_button = HobAltButton("Run image")
+            else:
+                # create button "Run image" as the primary button
+                run_button = HobButton("Run image")
+                run_button.set_size_request(205, 49)
+                run_button.set_flags(gtk.CAN_DEFAULT)
+                packed = True
+            run_button.set_tooltip_text("Start up an image with qemu emulator")
+            button_id = run_button.connect("clicked", self.run_button_clicked_cb)
+            self.button_ids[button_id] = run_button
+            self.details_bottom_buttons.pack_end(run_button, expand=False, fill=False)
+            created = True
+
+        if not packed:
+            box = gtk.HBox(False, 6)
+            box.show()
+            subbox = gtk.HBox(False, 0)
+            subbox.set_size_request(205, 49)
+            subbox.show()
+            box.add(subbox)
+            self.details_bottom_buttons.pack_end(box, False, False)
+
+        name = "Save as template"
+        if name in buttonlist:
+            if created == True:
+                # separator
+                label = gtk.Label(" or ")
+                self.details_bottom_buttons.pack_end(label, expand=False, fill=False)
+
+            # create button "Save as template"
+            save_button = HobAltButton("Save as template")
+            save_button.set_tooltip_text("Save the image configuration for reuse")
+            button_id = save_button.connect("clicked", self.save_button_clicked_cb)
+            self.button_ids[button_id] = save_button
+            self.details_bottom_buttons.pack_end(save_button, expand=False, fill=False)
+            create = True
+
+        name = "Build new image"
+        if name in buttonlist:
+            # create button "Build new image"
+            build_new_button = HobAltButton("Build new image")
+            build_new_button.set_tooltip_text("Create a new image from scratch")
+            button_id = build_new_button.connect("clicked", self.build_new_button_clicked_cb)
+            self.button_ids[button_id] = build_new_button
+            self.details_bottom_buttons.pack_start(build_new_button, expand=False, fill=False)
+
+    def _get_selected_image(self):
+        image_name = ""
+        iter = self.image_store.get_iter_first()
+        while iter:
+            path = self.image_store.get_path(iter)
+            if self.image_store[path][2]:
+                image_name = self.image_store[path][0]
+                break
+            iter = self.image_store.iter_next(iter)
+
+        return image_name
+
+    def save_button_clicked_cb(self, button):
+        self.builder.show_save_template_dialog()
+
+    def deploy_button_clicked_cb(self, button):
+        image_name = self._get_selected_image()
+        self.builder.deploy_image(image_name)
+
+    def run_button_clicked_cb(self, button):
+        image_name = self._get_selected_image()
+        self.builder.runqemu_image(image_name)
+
+    def build_new_button_clicked_cb(self, button):
+        self.builder.initiate_new_build_async()
+
+    def edit_config_button_clicked_cb(self, button):
+        self.builder.show_configuration()
+
+    def edit_packages_button_clicked_cb(self, button):
+        self.builder.show_packages(ask=False)
+
+    def template_button_clicked_cb(self, button):
+        response, path = self.builder.show_load_template_dialog()
+        if not response:
+            return
+        if path:
+            self.builder.load_template(path)
+
+    def my_images_button_clicked_cb(self, button):
+        self.builder.show_load_my_images_dialog()
+
+    def settings_button_clicked_cb(self, button):
+        # Create an advanced settings dialog
+        response, settings_changed = self.builder.show_adv_settings_dialog()
+        if not response:
+            return
+        if settings_changed:
+            self.builder.reparse_post_adv_settings()

bitbake/lib/bb/ui/crumbs/layereditor.py

@@ -1,153 +0,0 @@
-#
-# BitBake Graphical GTK User Interface
-#
-# Copyright (C) 2011        Intel Corporation
-#
-# Authored by Joshua Lock <josh@linux.intel.com>
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-import gobject
-import gtk
-from bb.ui.crumbs.configurator import Configurator
-from bb.ui.crumbs.hig import CrumbsDialog
-
-class LayerEditor(gtk.Dialog):
-    """
-    Gtk+ Widget for enabling and disabling layers.
-    Layers are added through using an open dialog to find the layer.conf
-    Disabled layers are deleted from conf/bblayers.conf
-    """
-    def __init__(self, configurator, parent=None):
-        gtk.Dialog.__init__(self, "Layers", None,
-                            gtk.DIALOG_DESTROY_WITH_PARENT,
-                            (gtk.STOCK_CLOSE, gtk.RESPONSE_OK))
-
-        # We want to show a little more of the treeview in the default,
-        # emptier, case
-        self.set_size_request(-1, 300)
-        self.set_border_width(6)
-        self.vbox.set_property("spacing", 0)
-        self.action_area.set_property("border-width", 6)
-
-        self.configurator = configurator
-        self.newly_added = {}
-
-        # Label to inform users that meta is enabled but that you can't
-        # disable it as it'd be a *bad* idea
-        msg = "As the core of the build system the <i>meta</i> layer must always be included and therefore can't be viewed or edited here."
-        lbl = gtk.Label()
-        lbl.show()
-        lbl.set_use_markup(True)
-        lbl.set_markup(msg)
-        lbl.set_line_wrap(True)
-        lbl.set_justify(gtk.JUSTIFY_FILL)
-        self.vbox.pack_start(lbl, expand=False, fill=False, padding=6)
-
-        # Create a treeview in which to list layers
-        # ListStore of Name, Path, Enabled
-        self.layer_store = gtk.ListStore(gobject.TYPE_STRING, gobject.TYPE_STRING, gobject.TYPE_BOOLEAN)
-        self.tv = gtk.TreeView(self.layer_store)
-        self.tv.set_headers_visible(True)
-
-        col0 = gtk.TreeViewColumn('Name')
-        self.tv.append_column(col0)
-        col1 = gtk.TreeViewColumn('Path')
-        self.tv.append_column(col1)
-        col2 = gtk.TreeViewColumn('Enabled')
-        self.tv.append_column(col2)
-
-        cell0 = gtk.CellRendererText()
-        col0.pack_start(cell0, True)
-        col0.set_attributes(cell0, text=0)
-        cell1 = gtk.CellRendererText()
-        col1.pack_start(cell1, True)
-        col1.set_attributes(cell1, text=1)
-        cell2 = gtk.CellRendererToggle()
-        cell2.connect("toggled", self._toggle_layer_cb)
-        col2.pack_start(cell2, True)
-        col2.set_attributes(cell2, active=2)
-
-        self.tv.show()
-        self.vbox.pack_start(self.tv, expand=True, fill=True, padding=0)
-
-        tb = gtk.Toolbar()
-        tb.set_icon_size(gtk.ICON_SIZE_SMALL_TOOLBAR)
-        tb.set_style(gtk.TOOLBAR_BOTH)
-        tb.set_tooltips(True)
-        tb.show()
-        icon = gtk.Image()
-        icon.set_from_stock(gtk.STOCK_ADD, gtk.ICON_SIZE_SMALL_TOOLBAR)
-        icon.show()
-        tb.insert_item("Add Layer", "Add new layer", None, icon,
-                       self._find_layer_cb, None, -1)
-        self.vbox.pack_start(tb, expand=False, fill=False, padding=0)
-
-    def set_parent_window(self, parent):
-        self.set_transient_for(parent)
-
-    def load_current_layers(self, data):
-        for layer, path in self.configurator.enabled_layers.items():
-            if layer != 'meta':
-                self.layer_store.append([layer, path, True])
-
-    def save_current_layers(self):
-        self.configurator.writeLayerConf()
-
-    def _toggle_layer_cb(self, cell, path):
-        name = self.layer_store[path][0]
-        toggle = not self.layer_store[path][2]
-        if toggle:
-            self.configurator.addLayer(name, path)
-        else:
-            self.configurator.disableLayer(name)
-        self.layer_store[path][2] = toggle
-
-    def _find_layer_cb(self, button):
-        self.find_layer(self)
-
-    def find_layer(self, parent):
-        def conf_error(parent, lbl):
-            dialog = CrumbsDialog(parent, lbl)
-            dialog.add_button(gtk.STOCK_OK, gtk.RESPONSE_OK)
-            response = dialog.run()
-            dialog.destroy()
-
-        dialog = gtk.FileChooserDialog("Add new layer", parent,
-                                       gtk.FILE_CHOOSER_ACTION_OPEN,
-                                       (gtk.STOCK_CANCEL, gtk.RESPONSE_NO,
-                                        gtk.STOCK_OPEN, gtk.RESPONSE_YES))
-        label = gtk.Label("Select the layer.conf of the layer you wish to add")
-        label.show()
-        dialog.set_extra_widget(label)
-        response = dialog.run()
-        path = dialog.get_filename()
-        dialog.destroy()
-
-        lbl = "<b>Error</b>\nUnable to load layer <i>%s</i> because " % path
-        if response == gtk.RESPONSE_YES:
-            # FIXME: verify we've actually got a layer conf?
-            if path.endswith("layer.conf"):
-                name, layerpath = self.configurator.addLayerConf(path)
-                if name and layerpath:
-                    self.newly_added[name] = layerpath
-                    self.layer_store.append([name, layerpath, True])
-                    return
-                elif name:
-                    return
-                else:
-                    lbl += "there was a problem parsing the layer.conf."
-            else:
-                lbl += "it is not a layer.conf file."
-            conf_error(parent, lbl)

bitbake/lib/bb/ui/crumbs/packageselectionpage.py

@@ -0,0 +1,249 @@
+#!/usr/bin/env python
+#
+# BitBake Graphical GTK User Interface
+#
+# Copyright (C) 2012        Intel Corporation
+#
+# Authored by Dongxiao Xu <dongxiao.xu@intel.com>
+# Authored by Shane Wang <shane.wang@intel.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import gtk
+import glib
+from bb.ui.crumbs.hobcolor import HobColors
+from bb.ui.crumbs.hobwidget import HobViewTable, HobNotebook, HobAltButton, HobButton
+from bb.ui.crumbs.hoblistmodel import PackageListModel
+from bb.ui.crumbs.hobpages import HobPage
+
+#
+# PackageSelectionPage
+#
+class PackageSelectionPage (HobPage):
+
+    pages = [
+        {
+         'name'    : 'Included',
+         'filter'  : { PackageListModel.COL_INC : [True] },
+         'columns' : [{
+                       'col_name' : 'Package name',
+                       'col_id'   : PackageListModel.COL_NAME,
+                       'col_style': 'text',
+                       'col_min'  : 100,
+                       'col_max'  : 300,
+                       'expand'   : 'True'
+                      }, {
+                       'col_name' : 'Brought in by',
+                       'col_id'   : PackageListModel.COL_BINB,
+                       'col_style': 'binb',
+                       'col_min'  : 100,
+                       'col_max'  : 350,
+                       'expand'   : 'True'
+                      }, {
+                       'col_name' : 'Size',
+                       'col_id'   : PackageListModel.COL_SIZE,
+                       'col_style': 'text',
+                       'col_min'  : 100,
+                       'col_max'  : 300,
+                       'expand'   : 'True'
+                      }, {
+                       'col_name' : 'Included',
+                       'col_id'   : PackageListModel.COL_INC,
+                       'col_style': 'check toggle',
+                       'col_min'  : 100,
+                       'col_max'  : 100
+                     }]
+        }, {
+         'name'    : 'All packages',
+         'filter'  : {},
+         'columns' : [{
+                       'col_name' : 'Package name',
+                       'col_id'   : PackageListModel.COL_NAME,
+                       'col_style': 'text',
+                       'col_min'  : 100,
+                       'col_max'  : 400,
+                       'expand'   : 'True'
+                      }, {
+                       'col_name' : 'Size',
+                       'col_id'   : PackageListModel.COL_SIZE,
+                       'col_style': 'text',
+                       'col_min'  : 100,
+                       'col_max'  : 500,
+                       'expand'   : 'True'
+                      }, {
+                       'col_name' : 'Included',
+                       'col_id'   : PackageListModel.COL_INC,
+                       'col_style': 'check toggle',
+                       'col_min'  : 100,
+                       'col_max'  : 100
+                      }]
+        }
+    ]
+
+    def __init__(self, builder):
+        super(PackageSelectionPage, self).__init__(builder, "Packages")
+
+        # set invisiable members
+        self.recipe_model = self.builder.recipe_model
+        self.package_model = self.builder.package_model
+
+        # create visual elements
+        self.create_visual_elements()
+
+    def create_visual_elements(self):
+        self.label = gtk.Label("Packages included: 0\nSelected packages size: 0 MB")
+        self.eventbox = self.add_onto_top_bar(self.label, 73)
+        self.pack_start(self.eventbox, expand=False, fill=False)
+        self.pack_start(self.group_align, expand=True, fill=True)
+
+        # set visiable members
+        self.ins = HobNotebook()
+        self.tables = [] # we need to modify table when the dialog is shown
+        # append the tab
+        for page in self.pages:
+            columns = page['columns']
+            tab = HobViewTable(columns)
+            filter = page['filter']
+            tab.set_model(self.package_model.tree_model(filter))
+            tab.connect("toggled", self.table_toggled_cb, page['name'])
+            if page['name'] == "Included":
+                tab.connect("button-release-event", self.button_click_cb)
+                tab.connect("cell-fadeinout-stopped", self.after_fadeout_checkin_include)
+            label = gtk.Label(page['name'])
+            self.ins.append_page(tab, label)
+            self.tables.append(tab)
+
+        self.ins.set_entry("Search packages:")
+        # set the search entry for each table
+        for tab in self.tables:
+            tab.set_search_entry(0, self.ins.search)
+
+        # add all into the dialog
+        self.box_group_area.pack_start(self.ins, expand=True, fill=True)
+
+        button_box = gtk.HBox(False, 6)
+        self.box_group_area.pack_start(button_box, expand=False, fill=False)
+
+        self.build_image_button = HobButton('Build image')
+        self.build_image_button.set_size_request(205, 49)
+        self.build_image_button.set_tooltip_text("Build target image")
+        self.build_image_button.set_flags(gtk.CAN_DEFAULT)
+        self.build_image_button.grab_default()
+        self.build_image_button.connect("clicked", self.build_image_clicked_cb)
+        button_box.pack_end(self.build_image_button, expand=False, fill=False)
+
+        self.back_button = HobAltButton("<< Back to image configuration")
+        self.back_button.connect("clicked", self.back_button_clicked_cb)
+        button_box.pack_start(self.back_button, expand=False, fill=False)
+
+    def button_click_cb(self, widget, event):
+        path, col = widget.table_tree.get_cursor()
+        tree_model = widget.table_tree.get_model()
+        if path: # else activation is likely a removal
+            binb = tree_model.get_value(tree_model.get_iter(path), PackageListModel.COL_BINB)
+            if binb:
+                self.builder.show_binb_dialog(binb)
+
+    def build_image_clicked_cb(self, button):
+        self.builder.build_image()
+
+    def back_button_clicked_cb(self, button):
+        self.builder.show_configuration()
+
+    def _expand_all(self):
+        for tab in self.tables:
+            tab.table_tree.expand_all()
+
+    def refresh_selection(self):
+        self._expand_all()
+
+        self.builder.configuration.selected_packages = self.package_model.get_selected_packages()
+        self.builder.configuration.user_selected_packages = self.package_model.get_user_selected_packages()
+        selected_packages_num = len(self.builder.configuration.selected_packages)
+        selected_packages_size = self.package_model.get_packages_size()
+        selected_packages_size_str = HobPage._size_to_string(selected_packages_size)
+
+        image_overhead_factor = self.builder.configuration.image_overhead_factor
+        image_rootfs_size = self.builder.configuration.image_rootfs_size * 1024 # image_rootfs_size is KB
+        image_extra_size = self.builder.configuration.image_extra_size * 1024 # image_extra_size is KB
+        base_size = image_overhead_factor * selected_packages_size
+        image_total_size = max(base_size, image_rootfs_size) + image_extra_size
+        if "zypper" in self.builder.configuration.selected_packages:
+            image_total_size += (51200 * 1024)
+        image_total_size_str = HobPage._size_to_string(image_total_size)
+
+        self.label.set_text("Packages included: %s\nSelected packages size: %s\nTotal image size: %s" %
+                            (selected_packages_num, selected_packages_size_str, image_total_size_str))
+        self.ins.show_indicator_icon("Included", selected_packages_num)
+
+    def toggle_item_idle_cb(self, path, view_tree, cell, pagename):
+        if not self.package_model.path_included(path):
+            self.package_model.include_item(item_path=path, binb="User Selected")
+        else:
+            if pagename == "Included":
+                self.pre_fadeout_checkout_include(view_tree)
+                self.package_model.exclude_item(item_path=path)
+                self.render_fadeout(view_tree, cell)
+            else:
+                self.package_model.exclude_item(item_path=path)
+
+        self.refresh_selection()
+        if not self.builder.customized:
+            self.builder.customized = True
+            self.builder.configuration.selected_image = self.recipe_model.__dummy_image__
+            self.builder.rcppkglist_populated()
+
+        self.builder.window_sensitive(True)
+
+    def table_toggled_cb(self, table, cell, view_path, toggled_columnid, view_tree, pagename):
+        # Click to include a package
+        self.builder.window_sensitive(False)
+        view_model = view_tree.get_model()
+        path = self.package_model.convert_vpath_to_path(view_model, view_path)
+        glib.idle_add(self.toggle_item_idle_cb, path, view_tree, cell, pagename)
+
+    def pre_fadeout_checkout_include(self, tree):
+        self.package_model.resync_fadeout_column(self.package_model.get_iter_first())
+        # Check out a model which base on the column COL_FADE_INC,
+        # it's save the prev state of column COL_INC before do exclude_item
+        filter = { PackageListModel.COL_FADE_INC  : [True]}
+        new_model = self.package_model.tree_model(filter)
+        tree.set_model(new_model)
+        tree.expand_all()
+
+    def get_excluded_rows(self, to_render_cells, model, it):
+        while it:
+            path = model.get_path(it)
+            prev_cell_is_active = model.get_value(it, PackageListModel.COL_FADE_INC)
+            curr_cell_is_active = model.get_value(it, PackageListModel.COL_INC)
+            if (prev_cell_is_active == True) and (curr_cell_is_active == False):
+                to_render_cells.append(path)
+            if model.iter_has_child(it):
+                self.get_excluded_rows(to_render_cells, model, model.iter_children(it))
+            it = model.iter_next(it)
+
+        return to_render_cells
+
+    def render_fadeout(self, tree, cell):
+        if (not cell) or (not tree):
+            return
+        to_render_cells = []
+        view_model = tree.get_model()
+        self.get_excluded_rows(to_render_cells, view_model, view_model.get_iter_first())
+
+        cell.fadeout(tree, 1000, to_render_cells)
+
+    def after_fadeout_checkin_include(self, table, ctrl, cell, tree):
+        tree.set_model(self.package_model.tree_model(self.pages[0]['filter']))
+        tree.expand_all()

bitbake/lib/bb/ui/crumbs/persistenttooltip.py

@@ -0,0 +1,186 @@
+#
+# BitBake Graphical GTK User Interface
+#
+# Copyright (C) 2012   Intel Corporation
+#
+# Authored by Joshua Lock <josh@linux.intel.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import gobject
+import gtk
+try:
+    import gconf
+except:
+    pass
+
+class PersistentTooltip(gtk.Window):
+    """
+    A tooltip which persists once shown until the user dismisses it with the Esc
+    key or by clicking the close button.
+
+    # FIXME: the PersistentTooltip should be disabled when the user clicks anywhere off
+    # it. We can't do this with focus-out-event becuase modal ensures we have focus?
+
+    markup: some Pango text markup to display in the tooltip
+    """
+    def __init__(self, markup, parent_win=None):
+        gtk.Window.__init__(self, gtk.WINDOW_POPUP)
+
+        # Inherit the system theme for a tooltip
+        style = gtk.rc_get_style_by_paths(gtk.settings_get_default(),
+                        'gtk-tooltip', 'gtk-tooltip', gobject.TYPE_NONE)
+        self.set_style(style)
+
+        # The placement of the close button on the tip should reflect how the
+        # window manager of the users system places close buttons. Try to read
+        # the metacity gconf key to determine whether the close button is on the
+        # left or the right.
+        # In the case that we can't determine the users configuration we default
+        # to close buttons being on the right.
+        __button_right = True
+        try:
+            client = gconf.client_get_default()
+            order = client.get_string("/apps/metacity/general/button_layout")
+            if order and order.endswith(":"):
+                __button_right = False
+        except NameError:
+            pass
+
+        # We need to ensure we're only shown once
+        self.shown = False
+
+        # We don't want any WM decorations
+        self.set_decorated(False)
+        # We don't want to show in the taskbar or window switcher
+        self.set_skip_pager_hint(True)
+        self.set_skip_taskbar_hint(True)
+        # We must be modal to ensure we grab focus when presented from a gtk.Dialog
+        self.set_modal(True)
+
+        self.set_border_width(0)
+        self.set_position(gtk.WIN_POS_MOUSE)
+        self.set_opacity(0.95)
+
+        # Ensure a reasonable minimum size
+        self.set_geometry_hints(self, 100, 50)
+
+        # Set this window as a transient window for parent(main window)
+        if parent_win:
+            self.set_transient_for(parent_win)
+            self.set_destroy_with_parent(True)
+        # Draw our label and close buttons
+        hbox = gtk.HBox(False, 0)
+        hbox.show()
+        self.add(hbox)
+
+        img = gtk.Image()
+        img.set_from_stock(gtk.STOCK_CLOSE, gtk.ICON_SIZE_BUTTON)
+
+        self.button = gtk.Button()
+        self.button.set_image(img)
+        self.button.connect("clicked", self._dismiss_cb)
+        self.button.set_flags(gtk.CAN_DEFAULT)
+        self.button.grab_focus()
+        self.button.show()
+        vbox = gtk.VBox(False, 0)
+        vbox.show()
+        vbox.pack_start(self.button, False, False, 0)
+        if __button_right:
+            hbox.pack_end(vbox, True, True, 0)
+        else:
+            hbox.pack_start(vbox, True, True, 0)
+
+        self.set_default(self.button)
+
+        bin = gtk.HBox(True, 6)
+        bin.set_border_width(6)
+        bin.show()
+        self.label = gtk.Label()
+        self.label.set_line_wrap(True)
+        # We want to match the colours of the normal tooltips, as dictated by
+        # the users gtk+-2.0 theme, wherever possible - on some systems this
+        # requires explicitly setting a fg_color for the label which matches the
+        # tooltip_fg_color
+        settings = gtk.settings_get_default()
+        colours = settings.get_property('gtk-color-scheme').split('\n')
+        # remove any empty lines, there's likely to be a trailing one after
+        # calling split on a dictionary-like string
+        colours = filter(None, colours)
+        for col in colours:
+            item, val = col.split(': ')
+            if item == 'tooltip_fg_color':
+                style = self.label.get_style()
+                style.fg[gtk.STATE_NORMAL] = gtk.gdk.color_parse(val)
+                self.label.set_style(style)
+                break # we only care for the tooltip_fg_color
+
+        self.label.set_markup(markup)
+        self.label.show()
+        bin.add(self.label)
+        hbox.pack_end(bin, True, True, 6)
+
+        # add the original URL display for user reference
+        if 'a href' in markup:
+            hbox.set_tooltip_text(self.get_markup_url(markup))
+        hbox.show()
+
+        self.connect("key-press-event", self._catch_esc_cb)
+
+    """
+    Callback when the PersistentTooltip's close button is clicked.
+    Hides the PersistentTooltip.
+    """
+    def _dismiss_cb(self, button):
+        self.hide()
+        return True
+
+    """
+    Callback when the Esc key is detected. Hides the PersistentTooltip.
+    """
+    def _catch_esc_cb(self, widget, event):
+        keyname = gtk.gdk.keyval_name(event.keyval)
+        if keyname == "Escape":
+            self.hide()
+        return True
+
+    """
+    Called to present the PersistentTooltip.
+    Overrides the superclasses show() method to include state tracking.
+    """
+    def show(self):
+        if not self.shown:
+            self.shown = True
+            gtk.Window.show(self)
+
+    """
+    Called to hide the PersistentTooltip.
+    Overrides the superclasses hide() method to include state tracking.
+    """
+    def hide(self):
+        self.shown = False
+        gtk.Window.hide(self)
+
+    """
+    Called to get the hyperlink URL from markup text.
+    """
+    def get_markup_url(self, markup):
+        url = "http:"
+        if markup and type(markup) == str:
+            s = markup
+            if 'http:' in s:
+                import re
+                url = re.search('(http:[^,\\ "]+)', s).group(0)
+
+        return url

bitbake/lib/bb/ui/crumbs/progressbar.py

@@ -0,0 +1,54 @@
+# BitBake Graphical GTK User Interface
+#
+# Copyright (C) 2011        Intel Corporation
+#
+# Authored by Shane Wang <shane.wang@intel.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import gtk
+from bb.ui.crumbs.hobcolor import HobColors
+
+class HobProgressBar (gtk.ProgressBar):
+    def __init__(self):
+        gtk.ProgressBar.__init__(self)
+        self.set_rcstyle(True)
+        self.percentage = 0
+
+    def set_rcstyle(self, status):
+        rcstyle = gtk.RcStyle()
+        rcstyle.fg[2] = gtk.gdk.Color(HobColors.BLACK)
+        if status == "stop":
+            rcstyle.bg[3] = gtk.gdk.Color(HobColors.WARNING)
+        elif status == "fail":
+            rcstyle.bg[3] = gtk.gdk.Color(HobColors.ERROR)
+        else:
+            rcstyle.bg[3] = gtk.gdk.Color(HobColors.RUNNING)
+        self.modify_style(rcstyle)
+
+    def set_title(self, text=None):
+        if not text:
+            text = ""
+        text += " %.0f%%" % self.percentage
+        self.set_text(text)
+
+    def reset(self):
+        self.set_fraction(0)
+        self.set_text("")
+        self.set_rcstyle(True)
+        self.percentage = 0
+
+    def update(self, fraction):
+        self.percentage = int(fraction * 100)
+        self.set_fraction(fraction)

bitbake/lib/bb/ui/crumbs/recipeselectionpage.py

@@ -0,0 +1,265 @@
+#!/usr/bin/env python
+#
+# BitBake Graphical GTK User Interface
+#
+# Copyright (C) 2012        Intel Corporation
+#
+# Authored by Dongxiao Xu <dongxiao.xu@intel.com>
+# Authored by Shane Wang <shane.wang@intel.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import gtk
+import glib
+from bb.ui.crumbs.hobcolor import HobColors
+from bb.ui.crumbs.hobwidget import HobViewTable, HobNotebook, HobAltButton, HobButton
+from bb.ui.crumbs.hoblistmodel import RecipeListModel
+from bb.ui.crumbs.hobpages import HobPage
+
+#
+# RecipeSelectionPage
+#
+class RecipeSelectionPage (HobPage):
+    pages = [
+        {
+         'name'    : 'Included',
+         'tooltip' : 'The recipes currently included for your image',
+         'filter'  : { RecipeListModel.COL_INC  : [True],
+                       RecipeListModel.COL_TYPE : ['recipe', 'task'] },
+         'columns' : [{
+                       'col_name' : 'Recipe name',
+                       'col_id'   : RecipeListModel.COL_NAME,
+                       'col_style': 'text',
+                       'col_min'  : 100,
+                       'col_max'  : 400,
+                       'expand'   : 'True'
+                      }, {
+                       'col_name' : 'Brought in by',
+                       'col_id'   : RecipeListModel.COL_BINB,
+                       'col_style': 'binb',
+                       'col_min'  : 100,
+                       'col_max'  : 500,
+                       'expand'   : 'True'
+                      }, {
+                       'col_name' : 'Group',
+                       'col_id'   : RecipeListModel.COL_GROUP,
+                       'col_style': 'text',
+                       'col_min'  : 100,
+                       'col_max'  : 300,
+                       'expand'   : 'True'
+                      }, {
+                       'col_name' : 'Included',
+                       'col_id'   : RecipeListModel.COL_INC,
+                       'col_style': 'check toggle',
+                       'col_min'  : 100,
+                       'col_max'  : 100
+                      }]
+        }, {
+         'name'    : 'All recipes',
+         'tooltip' : 'All recipes available in the Yocto Project',
+         'filter'  : { RecipeListModel.COL_TYPE : ['recipe'] },
+         'columns' : [{
+                       'col_name' : 'Recipe name',
+                       'col_id'   : RecipeListModel.COL_NAME,
+                       'col_style': 'text',
+                       'col_min'  : 100,
+                       'col_max'  : 400,
+                       'expand'   : 'True'
+                      }, {
+                       'col_name' : 'License',
+                       'col_id'   : RecipeListModel.COL_LIC,
+                       'col_style': 'text',
+                       'col_min'  : 100,
+                       'col_max'  : 400,
+                       'expand'   : 'True'
+                      }, {
+                       'col_name' : 'Group',
+                       'col_id'   : RecipeListModel.COL_GROUP,
+                       'col_style': 'text',
+                       'col_min'  : 100,
+                       'col_max'  : 400,
+                       'expand'   : 'True'
+                      }, {
+                       'col_name' : 'Included',
+                       'col_id'   : RecipeListModel.COL_INC,
+                       'col_style': 'check toggle',
+                       'col_min'  : 100,
+                       'col_max'  : 100
+                      }]
+        }, {
+         'name'    : 'Tasks',
+         'tooltip' : 'All tasks available in the Yocto Project',
+         'filter'  : { RecipeListModel.COL_TYPE : ['task'] },
+         'columns' : [{
+                       'col_name' : 'Task name',
+                       'col_id'   : RecipeListModel.COL_NAME,
+                       'col_style': 'text',
+                       'col_min'  : 100,
+                       'col_max'  : 400,
+                       'expand'   : 'True'
+                      }, {
+                       'col_name' : 'Description',
+                       'col_id'   : RecipeListModel.COL_DESC,
+                       'col_style': 'text',
+                       'col_min'  : 100,
+                       'col_max'  : 400,
+                       'expand'   : 'True'
+                      }, {
+                       'col_name' : 'Included',
+                       'col_id'   : RecipeListModel.COL_INC,
+                       'col_style': 'check toggle',
+                       'col_min'  : 100,
+                       'col_max'  : 100
+                      }]
+        }
+    ]
+
+    def __init__(self, builder = None):
+        super(RecipeSelectionPage, self).__init__(builder, "Recipes")
+
+        # set invisiable members
+        self.recipe_model = self.builder.recipe_model
+
+        # create visual elements
+        self.create_visual_elements()
+
+    def create_visual_elements(self):
+        self.label = gtk.Label()
+        self.eventbox = self.add_onto_top_bar(self.label, 73)
+        self.pack_start(self.eventbox, expand=False, fill=False)
+        self.pack_start(self.group_align, expand=True, fill=True)
+
+        # set visiable members
+        self.ins = HobNotebook()
+        self.tables = [] # we need modify table when the dialog is shown
+        # append the tabs in order
+        for page in self.pages:
+            columns = page['columns']
+            tab = HobViewTable(columns)
+            filter = page['filter']
+            tab.set_model(self.recipe_model.tree_model(filter))
+            tab.connect("toggled", self.table_toggled_cb, page['name'])
+            if page['name'] == "Included":
+                tab.connect("button-release-event", self.button_click_cb)
+                tab.connect("cell-fadeinout-stopped", self.after_fadeout_checkin_include)
+            label = gtk.Label(page['name'])
+            label.set_selectable(False)
+            label.set_tooltip_text(page['tooltip'])
+            self.ins.append_page(tab, label)
+            self.tables.append(tab)
+
+        self.ins.set_entry("Search recipes:")
+        # set the search entry for each table
+        for tab in self.tables:
+            search_tip = "Enter a recipe's or task's name to find it"
+            self.ins.search.set_tooltip_text(search_tip)
+            self.ins.search.props.has_tooltip = True
+            tab.set_search_entry(0, self.ins.search)
+
+        # add all into the window
+        self.box_group_area.pack_start(self.ins, expand=True, fill=True)
+
+        button_box = gtk.HBox(False, 6)
+        self.box_group_area.pack_end(button_box, expand=False, fill=False)
+
+        self.build_packages_button = HobButton('Build packages')
+        self.build_packages_button.set_size_request(205, 49)
+        self.build_packages_button.set_tooltip_text("Build selected recipes into packages")
+        self.build_packages_button.set_flags(gtk.CAN_DEFAULT)
+        self.build_packages_button.grab_default()
+        self.build_packages_button.connect("clicked", self.build_packages_clicked_cb)
+        button_box.pack_end(self.build_packages_button, expand=False, fill=False)
+
+        self.back_button = HobAltButton("<< Back to image configuration")
+        self.back_button.connect("clicked", self.back_button_clicked_cb)
+        button_box.pack_start(self.back_button, expand=False, fill=False)
+
+    def button_click_cb(self, widget, event):
+        path, col = widget.table_tree.get_cursor()
+        tree_model = widget.table_tree.get_model()
+        if path: # else activation is likely a removal
+            binb = tree_model.get_value(tree_model.get_iter(path), RecipeListModel.COL_BINB)
+            if binb:
+                self.builder.show_binb_dialog(binb)
+
+    def build_packages_clicked_cb(self, button):
+        self.builder.build_packages()
+
+    def back_button_clicked_cb(self, button):
+        self.builder.show_configuration()
+
+    def refresh_selection(self):
+        self.builder.configuration.selected_image = self.recipe_model.get_selected_image()
+        _, self.builder.configuration.selected_recipes = self.recipe_model.get_selected_recipes()
+        self.label.set_text("Recipes included: %s" % len(self.builder.configuration.selected_recipes))
+        self.ins.show_indicator_icon("Included", len(self.builder.configuration.selected_recipes))
+
+    def toggle_item_idle_cb(self, path, view_tree, cell, pagename):
+        if not self.recipe_model.path_included(path):
+            self.recipe_model.include_item(item_path=path, binb="User Selected", image_contents=False)
+        else:
+            if pagename == "Included":
+                self.pre_fadeout_checkout_include(view_tree)
+                self.recipe_model.exclude_item(item_path=path)
+                self.render_fadeout(view_tree, cell)
+            else:
+                self.recipe_model.exclude_item(item_path=path)
+
+        self.refresh_selection()
+        if not self.builder.customized:
+            self.builder.customized = True
+            self.builder.configuration.selected_image = self.recipe_model.__dummy_image__
+            self.builder.rcppkglist_populated()
+
+        self.builder.window_sensitive(True)
+
+    def table_toggled_cb(self, table, cell, view_path, toggled_columnid, view_tree, pagename):
+        # Click to include a recipe
+        self.builder.window_sensitive(False)
+        view_model = view_tree.get_model()
+        path = self.recipe_model.convert_vpath_to_path(view_model, view_path)
+        glib.idle_add(self.toggle_item_idle_cb, path, view_tree, cell, pagename)
+
+    def pre_fadeout_checkout_include(self, tree):
+        #resync the included items to a backup fade include column
+        it = self.recipe_model.get_iter_first()
+        while it:
+            active = self.recipe_model.get_value(it, self.recipe_model.COL_INC)
+            self.recipe_model.set(it, self.recipe_model.COL_FADE_INC, active)
+            it = self.recipe_model.iter_next(it)
+        # Check out a model which base on the column COL_FADE_INC,
+        # it's save the prev state of column COL_INC before do exclude_item
+        filter = { RecipeListModel.COL_FADE_INC  : [True],
+                   RecipeListModel.COL_TYPE      : ['recipe', 'task'] }
+        new_model = self.recipe_model.tree_model(filter, excluded_items_ahead=True)
+        tree.set_model(new_model)
+
+    def render_fadeout(self, tree, cell):
+        if (not cell) or (not tree):
+            return
+        to_render_cells = []
+        model = tree.get_model()
+        it = model.get_iter_first()
+        while it:
+            path = model.get_path(it)
+            prev_cell_is_active = model.get_value(it, RecipeListModel.COL_FADE_INC)
+            curr_cell_is_active = model.get_value(it, RecipeListModel.COL_INC)
+            if (prev_cell_is_active == True) and (curr_cell_is_active == False):
+                to_render_cells.append(path)
+            it = model.iter_next(it)
+
+        cell.fadeout(tree, 1000, to_render_cells)
+
+    def after_fadeout_checkin_include(self, table, ctrl, cell, tree):
+        tree.set_model(self.recipe_model.tree_model(self.pages[0]['filter']))

bitbake/lib/bb/ui/crumbs/runningbuild.py

@@ -25,12 +25,9 @@ import logging
 import time
 import urllib
 import urllib2
-
-class Colors(object):
-    OK = "#ffffff"
-    RUNNING = "#aaffaa"
-    WARNING ="#f88017"
-    ERROR = "#ffaaaa"
+import pango
+from bb.ui.crumbs.hobcolor import HobColors
+from bb.ui.crumbs.hobwidget import HobWarpCellRendererText, HobCellRendererPixbuf
 
 class RunningBuildModel (gtk.TreeStore):
     (COL_LOG, COL_PACKAGE, COL_TASK, COL_MESSAGE, COL_ICON, COL_COLOR, COL_NUM_ACTIVE) = range(7)
@@ -45,20 +42,49 @@ class RunningBuildModel (gtk.TreeStore):
                                 gobject.TYPE_STRING,
                                 gobject.TYPE_INT)
 
+    def failure_model_filter(self, model, it):
+        color = model.get(it, self.COL_COLOR)[0]
+        if not color:
+            return False
+        if color == HobColors.ERROR:
+            return True
+        return False
+
+    def failure_model(self):
+        model = self.filter_new()
+        model.set_visible_func(self.failure_model_filter)
+        return model
+
+    def foreach_cell_func(self, model, path, iter, usr_data=None):
+        if model.get_value(iter, self.COL_ICON) == "gtk-execute":
+            model.set(iter, self.COL_ICON, "")
+
+    def close_task_refresh(self):
+        self.foreach(self.foreach_cell_func, None)
+
 class RunningBuild (gobject.GObject):
     __gsignals__ = {
-          'build-started' : (gobject.SIGNAL_RUN_LAST,
-                               gobject.TYPE_NONE,
+          'build-started'   :  (gobject.SIGNAL_RUN_LAST,
+                                gobject.TYPE_NONE,
                                ()),
-          'build-succeeded' : (gobject.SIGNAL_RUN_LAST,
-                               gobject.TYPE_NONE,
+          'build-succeeded' :  (gobject.SIGNAL_RUN_LAST,
+                                gobject.TYPE_NONE,
                                ()),
-          'build-failed' : (gobject.SIGNAL_RUN_LAST,
-                            gobject.TYPE_NONE,
-                            ()),
-          'build-complete' : (gobject.SIGNAL_RUN_LAST,
-                              gobject.TYPE_NONE,
-                              ())
+          'build-failed'    :  (gobject.SIGNAL_RUN_LAST,
+                                gobject.TYPE_NONE,
+                               ()),
+          'build-complete'  :  (gobject.SIGNAL_RUN_LAST,
+                                gobject.TYPE_NONE,
+                               ()),
+          'task-started'    :  (gobject.SIGNAL_RUN_LAST,
+                                gobject.TYPE_NONE,
+                               (gobject.TYPE_PYOBJECT,)),
+          'log-error'       :  (gobject.SIGNAL_RUN_LAST,
+                                gobject.TYPE_NONE,
+                               ()),
+          'no-provider'     :  (gobject.SIGNAL_RUN_LAST,
+                                gobject.TYPE_NONE,
+                               (gobject.TYPE_PYOBJECT,)),
           }
     pids_to_task = {}
     tasks_to_iter = {}
@@ -108,13 +134,14 @@ class RunningBuild (gobject.GObject):
 
             if event.levelno >= logging.ERROR:
                 icon = "dialog-error"
-                color = Colors.ERROR
+                color = HobColors.ERROR
+                self.emit("log-error")
             elif event.levelno >= logging.WARNING:
                 icon = "dialog-warning"
-                color = Colors.WARNING
+                color = HobColors.WARNING
             else:
                 icon = None
-                color = Colors.OK
+                color = HobColors.OK
 
             # if we know which package we belong to, we'll append onto its list.
             # otherwise, we'll jump to the top of the master list
@@ -152,7 +179,7 @@ class RunningBuild (gobject.GObject):
                                     None,
                                     "Package: %s" % (package),
                                     None,
-                                    Colors.OK,
+                                    HobColors.OK,
                                     0))
                 self.tasks_to_iter[(package, None)] = parent
 
@@ -160,7 +187,7 @@ class RunningBuild (gobject.GObject):
             # such.
             # @todo if parent is already in error, don't mark it green
             self.model.set(parent, self.model.COL_ICON, "gtk-execute",
-                           self.model.COL_COLOR, Colors.RUNNING)
+                           self.model.COL_COLOR, HobColors.RUNNING)
 
             # Add an entry in the model for this task
             i = self.model.append (parent, (None,
@@ -168,7 +195,7 @@ class RunningBuild (gobject.GObject):
                                             task,
                                             "Task: %s" % (task),
                                             "gtk-execute",
-                                            Colors.RUNNING,
+                                            HobColors.RUNNING,
                                             0))
 
             # update the parent's active task count
@@ -179,10 +206,6 @@ class RunningBuild (gobject.GObject):
             # that we need to attach to a task.
             self.tasks_to_iter[(package, task)] = i
 
-        # If we don't handle these the GUI does not proceed
-        elif isinstance(event, bb.build.TaskInvalid):
-            return
-
         elif isinstance(event, bb.build.TaskBase):
             current = self.tasks_to_iter[(package, task)]
             parent = self.tasks_to_iter[(package, None)]
@@ -194,20 +217,20 @@ class RunningBuild (gobject.GObject):
             if isinstance(event, bb.build.TaskFailed):
                 # Mark the task and parent as failed
                 icon = "dialog-error"
-                color = Colors.ERROR
+                color = HobColors.ERROR
 
                 logfile = event.logfile
                 if logfile and os.path.exists(logfile):
                     with open(logfile) as f:
                         logdata = f.read()
-                        self.model.append(current, ('pastebin', None, None, logdata, 'gtk-error', Colors.OK, 0))
+                        self.model.append(current, ('pastebin', None, None, logdata, 'gtk-error', HobColors.OK, 0))
 
                 for i in (current, parent):
                     self.model.set(i, self.model.COL_ICON, icon,
                                    self.model.COL_COLOR, color)
             else:
                 icon = None
-                color = Colors.OK
+                color = HobColors.OK
 
                 # Mark the task as inactive
                 self.model.set(current, self.model.COL_ICON, icon,
@@ -219,7 +242,7 @@ class RunningBuild (gobject.GObject):
                 if self.model.get(parent, self.model.COL_ICON) != 'dialog-error':
                     self.model.set(parent, self.model.COL_ICON, icon)
                     if num_active == 0:
-                        self.model.set(parent, self.model.COL_COLOR, Colors.OK)
+                        self.model.set(parent, self.model.COL_COLOR, HobColors.OK)
 
             # Clear the iters and the pids since when the task goes away the
             # pid will no longer be used for messages
@@ -234,8 +257,12 @@ class RunningBuild (gobject.GObject):
                                       None,
                                       "Build Started (%s)" % time.strftime('%m/%d/%Y %H:%M:%S'),
                                       None,
-                                      Colors.OK,
+                                      HobColors.OK,
                                       0))
+            if pbar:
+                pbar.update(0, self.progress_total)
+                pbar.set_title(bb.event.getName(event))
+
         elif isinstance(event, bb.event.BuildCompleted):
             failures = int (event._failures)
             self.model.prepend(None, (None,
@@ -243,7 +270,7 @@ class RunningBuild (gobject.GObject):
                                       None,
                                       "Build Completed (%s)" % time.strftime('%m/%d/%Y %H:%M:%S'),
                                       None,
-                                      Colors.OK,
+                                      HobColors.OK,
                                       0))
 
             # Emit the appropriate signal depending on the number of failures
@@ -254,12 +281,18 @@ class RunningBuild (gobject.GObject):
             # Emit a generic "build-complete" signal for things wishing to
             # handle when the build is finished
             self.emit("build-complete")
+            # reset the all cell's icon indicator
+            self.model.close_task_refresh()
+            if pbar:
+                pbar.set_text(event.msg)
 
         elif isinstance(event, bb.command.CommandFailed):
             if event.error.startswith("Exited with"):
                 # If the command fails with an exit code we're done, emit the
                 # generic signal for the UI to notify the user
                 self.emit("build-complete")
+                # reset the all cell's icon indicator
+                self.model.close_task_refresh()
 
         elif isinstance(event, bb.event.CacheLoadStarted) and pbar:
             pbar.set_title("Loading cache")
@@ -280,6 +313,30 @@ class RunningBuild (gobject.GObject):
             pbar.update(event.current, self.progress_total)
         elif isinstance(event, bb.event.ParseCompleted) and pbar:
             pbar.hide()
+        #using runqueue events as many as possible to update the progress bar
+        elif isinstance(event, (bb.runqueue.runQueueTaskStarted, bb.runqueue.sceneQueueTaskStarted)):
+            message = {}
+            message["eventname"] = bb.event.getName(event)
+            num_of_completed = event.stats.completed + event.stats.failed
+            message["current"] = num_of_completed
+            message["total"] = event.stats.total
+            message["title"] = ""
+            message["task"] = event.taskstring
+            self.emit("task-started", message)
+        elif isinstance(event, bb.event.NoProvider):
+            msg = ""
+            if event._runtime:
+                r = "R"
+            else:
+                r = ""
+            if event._dependees:
+                msg = "Nothing %sPROVIDES '%s' (but %s %sDEPENDS on or otherwise requires it)\n" % (r, event._item, ", ".join(event._dependees), r)
+            else:
+                msg = "Nothing %sPROVIDES '%s'\n" % (r, event._item)
+            if event._reasons:
+                for reason in event._reasons:
+                    msg += ("%s\n" % reason)
+            self.emit("no-provider", msg)
 
         return
 
@@ -299,18 +356,26 @@ class RunningBuildTreeView (gtk.TreeView):
     __gsignals__ = {
         "button_press_event" : "override"
         }
-    def __init__ (self, readonly=False):
+    def __init__ (self, readonly=False, hob=False):
         gtk.TreeView.__init__ (self)
         self.readonly = readonly
 
         # The icon that indicates whether we're building or failed.
-        renderer = gtk.CellRendererPixbuf ()
+        # add 'hob' flag because there has not only hob to share this code
+        if hob:
+            renderer = HobCellRendererPixbuf ()
+        else:
+            renderer = gtk.CellRendererPixbuf()
         col = gtk.TreeViewColumn ("Status", renderer)
         col.add_attribute (renderer, "icon-name", 4)
         self.append_column (col)
 
         # The message of the build.
-        self.message_renderer = gtk.CellRendererText ()
+        # add 'hob' flag because there has not only hob to share this code
+        if hob:
+            self.message_renderer = HobWarpCellRendererText (col_number=1)
+        else:
+            self.message_renderer = gtk.CellRendererText ()
         self.message_column = gtk.TreeViewColumn ("Message", self.message_renderer, text=3)
         self.message_column.add_attribute(self.message_renderer, 'background', 5)
         self.message_renderer.set_property('editable', (not self.readonly))
@@ -367,3 +432,22 @@ class RunningBuildTreeView (gtk.TreeView):
         message = model.get(it, model.COL_MESSAGE)[0]
 
         self._add_to_clipboard(message)
+
+class BuildFailureTreeView(gtk.TreeView):
+
+    def __init__ (self):
+        gtk.TreeView.__init__(self)
+        self.set_rules_hint(False)
+        self.set_headers_visible(False)
+        self.get_selection().set_mode(gtk.SELECTION_SINGLE)
+
+        # The icon that indicates whether we're building or failed.
+        renderer = HobCellRendererPixbuf ()
+        col = gtk.TreeViewColumn ("Status", renderer)
+        col.add_attribute (renderer, "icon-name", RunningBuildModel.COL_ICON)
+        self.append_column (col)
+
+        # The message of the build.
+        self.message_renderer = HobWarpCellRendererText (col_number=1)
+        self.message_column = gtk.TreeViewColumn ("Message", self.message_renderer, text=RunningBuildModel.COL_MESSAGE, background=RunningBuildModel.COL_COLOR)
+        self.append_column (self.message_column)

bitbake/lib/bb/ui/crumbs/tasklistmodel.py

@@ -1,620 +0,0 @@
-#
-# BitBake Graphical GTK User Interface
-#
-# Copyright (C) 2011        Intel Corporation
-#
-# Authored by Joshua Lock <josh@linux.intel.com>
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-import gtk
-import gobject
-import re
-
-class BuildRep(gobject.GObject):
-
-    def __init__(self, userpkgs, allpkgs, base_image=None):
-        gobject.GObject.__init__(self)
-        self.base_image = base_image
-        self.allpkgs = allpkgs
-        self.userpkgs = userpkgs
-
-    def loadRecipe(self, pathname):
-        contents = []
-        packages = ""
-        base_image = ""
-
-        with open(pathname, 'r') as f:
-            contents = f.readlines()
-
-        pkg_pattern = "^\s*(IMAGE_INSTALL)\s*([+=.?]+)\s*(\".*?\")"
-        img_pattern = "^\s*(require)\s+(\S+.bb)"
-
-        for line in contents:
-            matchpkg = re.search(pkg_pattern, line)
-            matchimg = re.search(img_pattern, line)
-            if matchpkg:
-                packages = packages + matchpkg.group(3).strip('"')
-            if matchimg:
-                base_image = os.path.basename(matchimg.group(2)).split(".")[0]
-
-        self.base_image = base_image
-        self.userpkgs = packages
-
-    def writeRecipe(self, writepath, model):
-        template = """
-# Recipe generated by the HOB
-
-require %s
-
-IMAGE_INSTALL += "%s"
-"""
-
-        empty_template = """
-# Recipe generated by the HOB
-
-inherit core-image
-
-IMAGE_INSTALL = "%s"
-"""
-        if self.base_image and not self.base_image == "empty":
-            meta_path = model.find_image_path(self.base_image)
-            recipe = template % (meta_path, self.userpkgs)
-        else:
-            recipe = empty_template % self.allpkgs
-
-        if os.path.exists(writepath):
-            os.rename(writepath, "%s~" % writepath)
-
-        with open(writepath, 'w') as r:
-            r.write(recipe)
-
-        return writepath
-
-class TaskListModel(gtk.ListStore):
-    """
-    This class defines an gtk.ListStore subclass which will convert the output
-    of the bb.event.TargetsTreeGenerated event into a gtk.ListStore whilst also
-    providing convenience functions to access gtk.TreeModel subclasses which
-    provide filtered views of the data.
-    """
-    (COL_NAME, COL_DESC, COL_LIC, COL_GROUP, COL_DEPS, COL_BINB, COL_TYPE, COL_INC, COL_IMG, COL_PATH, COL_PN) = range(11)
-
-    __gsignals__ = {
-        "tasklist-populated" : (gobject.SIGNAL_RUN_LAST,
-                                gobject.TYPE_NONE,
-                                ()),
-        "contents-changed"   : (gobject.SIGNAL_RUN_LAST,
-                                gobject.TYPE_NONE,
-                                (gobject.TYPE_INT,)),
-        "image-changed"      : (gobject.SIGNAL_RUN_LAST,
-                                gobject.TYPE_NONE,
-                                (gobject.TYPE_STRING,)),
-        }
-
-    """
-    """
-    def __init__(self):
-        self.contents = None
-        self.tasks = None
-        self.packages = None
-        self.images = None
-        self.selected_image = None
-        
-        gtk.ListStore.__init__ (self,
-                                gobject.TYPE_STRING,
-                                gobject.TYPE_STRING,
-                                gobject.TYPE_STRING,
-                                gobject.TYPE_STRING,
-                                gobject.TYPE_STRING,
-                                gobject.TYPE_STRING,
-                                gobject.TYPE_STRING,
-                                gobject.TYPE_BOOLEAN,
-                                gobject.TYPE_BOOLEAN,
-                                gobject.TYPE_STRING,
-                                gobject.TYPE_STRING)
-
-    """
-    Helper method to determine whether name is a target pn
-    """
-    def non_target_name(self, name):
-        if ('-native' in name) or ('-cross' in name) or name.startswith('virtual/'):
-            return True
-        return False
-
-    def contents_changed_cb(self, tree_model, path, it=None):
-        pkg_cnt = self.contents.iter_n_children(None)
-        self.emit("contents-changed", pkg_cnt)
-
-    def contents_model_filter(self, model, it):
-        if not model.get_value(it, self.COL_INC) or model.get_value(it, self.COL_TYPE) == 'image':
-            return False
-        name = model.get_value(it, self.COL_NAME)
-        if self.non_target_name(name):
-            return False
-        else:
-            return True
-
-    """
-    Create, if required, and return a filtered gtk.TreeModel
-    containing only the items which are to be included in the
-    image
-    """
-    def contents_model(self):
-        if not self.contents:
-            self.contents = self.filter_new()
-            self.contents.set_visible_func(self.contents_model_filter)
-            self.contents.connect("row-inserted", self.contents_changed_cb)
-            self.contents.connect("row-deleted", self.contents_changed_cb)
-        return self.contents
-    
-    """
-    Helper function to determine whether an item is a task
-    """
-    def task_model_filter(self, model, it):
-        if model.get_value(it, self.COL_TYPE) == 'task':
-            return True
-        else:
-            return False
-
-    """
-    Create, if required, and return a filtered gtk.TreeModel
-    containing only the items which are tasks
-    """
-    def tasks_model(self):
-        if not self.tasks:
-            self.tasks = self.filter_new()
-            self.tasks.set_visible_func(self.task_model_filter)
-        return self.tasks
-
-    """
-    Helper function to determine whether an item is an image
-    """
-    def image_model_filter(self, model, it):
-        if model.get_value(it, self.COL_TYPE) == 'image':
-            return True
-        else:
-            return False
-
-    """
-    Create, if required, and return a filtered gtk.TreeModel
-    containing only the items which are images
-    """
-    def images_model(self):
-        if not self.images:
-            self.images = self.filter_new()
-            self.images.set_visible_func(self.image_model_filter)
-        return self.images
-
-    """
-    Helper function to determine whether an item is a package
-    """
-    def package_model_filter(self, model, it):
-        if model.get_value(it, self.COL_TYPE) != 'package':
-            return False
-        else:
-            name = model.get_value(it, self.COL_NAME)
-            if self.non_target_name(name):
-                return False
-            return True
-
-    """
-    Create, if required, and return a filtered gtk.TreeModel
-    containing only the items which are packages
-    """
-    def packages_model(self):
-        if not self.packages:
-            self.packages = self.filter_new()
-            self.packages.set_visible_func(self.package_model_filter)
-        return self.packages
-
-    """
-    The populate() function takes as input the data from a
-    bb.event.TargetsTreeGenerated event and populates the TaskList.
-    Once the population is done it emits gsignal tasklist-populated
-    to notify any listeners that the model is ready
-    """
-    def populate(self, event_model):
-        # First clear the model, in case repopulating
-        self.clear()
-        for item in event_model["pn"]:
-            atype = 'package'
-            name = item
-            summary = event_model["pn"][item]["summary"]
-            lic = event_model["pn"][item]["license"]
-            group = event_model["pn"][item]["section"]
-            filename = event_model["pn"][item]["filename"]
-            if ('task-' in name):
-                atype = 'task'
-            elif ('-image-' in name):
-                atype = 'image'
-
-            # Create a combined list of build and runtime dependencies and
-            # then remove any duplicate entries and any entries for -dev
-            # packages
-            depends = event_model["depends"].get(item, [])
-            rdepends = event_model["rdepends-pn"].get(item, [])
-            packages = {}
-            for pkg in event_model["packages"]:
-                if event_model["packages"][pkg]["pn"] == name:
-                    deps = []
-                    deps.extend(depends)
-                    deps.extend(event_model["rdepends-pkg"].get(pkg, []))
-                    deps.extend(rdepends)
-                    deps = self.squish(deps)
-                    # rdepends-pn includes pn-dev
-                    if ("%s-dev" % item) in deps:
-                        deps.remove("%s-dev" % item)
-                    # rdepends-on includes pn
-                    if pkg in deps:
-                        deps.remove(pkg)
-                    packages[pkg] = deps
-
-            for p in packages:
-                self.set(self.append(), self.COL_NAME, p, self.COL_DESC, summary,
-                         self.COL_LIC, lic, self.COL_GROUP, group,
-                         self.COL_DEPS, " ".join(packages[p]), self.COL_BINB, "",
-                         self.COL_TYPE, atype, self.COL_INC, False,
-                         self.COL_IMG, False, self.COL_PATH, filename,
-                         self.COL_PN, item)
-
-	self.emit("tasklist-populated")
-
-    """
-    Load a BuildRep into the model
-    """
-    def load_image_rep(self, rep):
-        # Unset everything
-        it = self.get_iter_first()
-        while it:
-            path = self.get_path(it)
-            self[path][self.COL_INC] = False
-            self[path][self.COL_IMG] = False
-            it = self.iter_next(it)
-
-        # Iterate the images and disable them all
-        it = self.images.get_iter_first()
-        while it:
-            path = self.images.convert_path_to_child_path(self.images.get_path(it))
-            name = self[path][self.COL_NAME]
-            if name == rep.base_image:
-                self.include_item(path, image_contents=True)
-            else:
-                self[path][self.COL_INC] = False
-            it = self.images.iter_next(it)
-
-        # Mark all of the additional packages for inclusion
-        packages = rep.userpkgs.split(" ")
-        it = self.get_iter_first()
-        while it:
-            path = self.get_path(it)
-            name = self[path][self.COL_NAME]
-            if name in packages:
-                self.include_item(path, binb="User Selected")
-                packages.remove(name)
-            it = self.iter_next(it)
-
-        self.emit("image-changed", rep.base_image)
-
-    """
-    squish lst so that it doesn't contain any duplicate entries
-    """
-    def squish(self, lst):
-        seen = {}
-        for l in lst:
-            seen[l] = 1
-        return seen.keys()
-
-    """
-    Mark the item at path as not included
-    NOTE:
-    path should be a gtk.TreeModelPath into self (not a filtered model)
-    """
-    def remove_item_path(self, path):
-        self[path][self.COL_BINB] = ""
-        self[path][self.COL_INC] = False
-
-    """
-    Recursively called to mark the item at opath and any package which
-    depends on it for removal.
-    NOTE: This method dumbly removes user selected packages and since we don't
-    do significant reverse dependency tracking it's easier and simpler to save
-    the items marked as user selected and re-add them once the removal sweep is
-    complete.
-    """
-    def mark(self, opath):
-        usersel = {}
-        removed = []
-
-        it = self.get_iter_first()
-        # The name of the item we're removing, so that we can use it to find
-        # other items which either depend on it, or were brought in by it
-        marked_name = self[opath][self.COL_NAME]
-
-        # Remove the passed item
-        self.remove_item_path(opath)
-
-        # Remove all dependent packages, update binb
-        while it:
-            path = self.get_path(it)
-            it = self.iter_next(it)
-
-            inc = self[path][self.COL_INC]
-            deps = self[path][self.COL_DEPS]
-            binb = self[path][self.COL_BINB].split(', ')
-            itype = self[path][self.COL_TYPE]
-            itname = self[path][self.COL_NAME]
-
-            # We ignore anything that isn't a package
-            if not itype == "package":
-                continue
-
-            # If the user added this item and it's not the item we're removing
-            # we should keep it and its dependencies, the easiest way to do so
-            # is to save its name and re-mark it for inclusion once dependency
-            # processing is complete
-            if "User Selected" in binb:
-                usersel[itname] = self[path][self.COL_IMG]
-
-            # If the iterated item is included and depends on the removed
-            # item it should also be removed.
-            # FIXME: need to ensure partial name matching doesn't happen
-            if inc and marked_name in deps and itname not in removed:
-                # found a dependency, remove it
-                removed.append(itname)
-                self.mark(path)
-
-            # If the iterated item was brought in by the removed (passed) item
-            # try and find an alternative dependee and update the binb column
-            if inc and marked_name in binb:
-                binb.remove(marked_name)
-                self[path][self.COL_BINB] = ', '.join(binb).lstrip(', ')
-
-        # Re-add any removed user selected items
-        for u in usersel:
-            npath = self.find_path_for_item(u)
-            self.include_item(item_path=npath,
-                              binb="User Selected",
-                              image_contents=usersel[u])
-    """
-    Remove items from contents if the have an empty COL_BINB (brought in by)
-    caused by all packages they are a dependency of being removed.
-    If the item isn't a package we leave it included.
-    """
-    def sweep_up(self):
-        it = self.contents.get_iter_first()
-        while it:
-            binb = self.contents.get_value(it, self.COL_BINB)
-            itype = self.contents.get_value(it, self.COL_TYPE)
-            remove = False
-
-            if itype == 'package' and not binb:
-                oit = self.contents.convert_iter_to_child_iter(it)
-                opath = self.get_path(oit)
-                self.mark(opath)
-                remove = True
-
-            # When we remove a package from the contents model we alter the
-            # model, so continuing to iterate is bad. *Furthermore* it's
-            # likely that the removal has affected an already iterated item
-            # so we should start from the beginning anyway.
-            # Only when we've managed to iterate the entire contents model
-            # without removing any items do we allow the loop to exit.
-            if remove:
-                it = self.contents.get_iter_first()
-            else:
-                it = self.contents.iter_next(it)
-
-    """
-    Check whether the item at item_path is included or not
-    """
-    def contents_includes_path(self, item_path):
-        return self[item_path][self.COL_INC]
-
-    """
-    Add this item, and any of its dependencies, to the image contents
-    """
-    def include_item(self, item_path, binb="", image_contents=False):
-        item_name = self[item_path][self.COL_NAME]
-        item_deps = self[item_path][self.COL_DEPS]
-
-        self[item_path][self.COL_INC] = True
-
-        item_bin = self[item_path][self.COL_BINB].split(', ')
-        if binb and not binb in item_bin:
-            item_bin.append(binb)
-            self[item_path][self.COL_BINB] = ', '.join(item_bin).lstrip(', ')
-
-        # We want to do some magic with things which are brought in by the
-        # base image so tag them as so
-        if image_contents:
-            self[item_path][self.COL_IMG] = True
-            if self[item_path][self.COL_TYPE] == 'image':
-                self.selected_image = item_name
-
-        if item_deps:
-            # Ensure all of the items deps are included and, where appropriate,
-            # add this item to their COL_BINB
-            for dep in item_deps.split(" "):
-                # If the contents model doesn't already contain dep, add it
-                dep_path = self.find_path_for_item(dep)
-                if not dep_path:
-                    continue
-                dep_included = self.contents_includes_path(dep_path)
-
-                if dep_included and not dep in item_bin:
-                    # don't set the COL_BINB to this item if the target is an
-                    # item in our own COL_BINB
-                    dep_bin = self[dep_path][self.COL_BINB].split(', ')
-                    if not item_name in dep_bin:
-                        dep_bin.append(item_name)
-                        self[dep_path][self.COL_BINB] = ', '.join(dep_bin).lstrip(', ')
-                elif not dep_included:
-                    self.include_item(dep_path, binb=item_name, image_contents=image_contents)
-
-    """
-    Find the model path for the item_name
-    Returns the path in the model or None
-    """
-    def find_path_for_item(self, item_name):
-        # We don't include virtual/* or *-native items in the model so save a
-        # heavy iteration loop by exiting early for these items
-        if self.non_target_name(item_name):
-            return None
-
-        it = self.get_iter_first()
-        while it:
-            if (self.get_value(it, self.COL_NAME) == item_name):
-                return self.get_path(it)
-            else:
-                it = self.iter_next(it)
-        return None
-
-    """
-    Empty self.contents by setting the include of each entry to None
-    """
-    def reset(self):
-        # Deselect images - slightly more complex logic so that we don't
-        # have to iterate all of the contents of the main model, instead
-        # just iterate the images model.
-        if self.selected_image:
-            iit = self.images.get_iter_first()
-            while iit:
-                pit = self.images.convert_iter_to_child_iter(iit)
-                self.set(pit, self.COL_INC, False)
-                iit = self.images.iter_next(iit)
-            self.selected_image = None
-
-        it = self.contents.get_iter_first()
-        while it:
-            oit = self.contents.convert_iter_to_child_iter(it)
-            self.set(oit,
-                     self.COL_INC, False,
-                     self.COL_BINB, "",
-                     self.COL_IMG, False)
-            # As we've just removed the first item...
-            it = self.contents.get_iter_first()
-
-    """
-    Returns two lists. One of user selected packages and the other containing
-    all selected packages
-    """
-    def get_selected_packages(self):
-        allpkgs = []
-        userpkgs = []
-
-        it = self.contents.get_iter_first()
-        while it:
-            sel = "User Selected" in self.contents.get_value(it, self.COL_BINB)
-            name = self.contents.get_value(it, self.COL_NAME)
-            allpkgs.append(name)
-            if sel:
-                userpkgs.append(name)
-            it = self.contents.iter_next(it)
-        return userpkgs, allpkgs
-
-    """
-    Return a squished (uniquified) list of the PN's of all selected items
-    """
-    def get_selected_pn(self):
-        pns = []
-
-        it = self.contents.get_iter_first()
-        while it:
-            if self.contents.get_value(it, self.COL_BINB):
-                pns.append(self.contents.get_value(it, self.COL_PN))
-            it = self.contents.iter_next(it)
-
-        return self.squish(pns)
-
-    def image_contents_removed(self):
-        it = self.get_iter_first()
-        while it:
-            sel = self.get_value(it, self.COL_INC)
-            img = self.get_value(it, self.COL_IMG)
-            if img and not sel:
-                return True
-            it = self.iter_next(it)
-        return False
-
-    def get_build_rep(self):
-        userpkgs, allpkgs = self.get_selected_packages()
-        # If base image contents have been removed start from an empty rootfs
-        if not self.selected_image or self.image_contents_removed():
-            image = "empty"
-        else:
-            image = self.selected_image
-
-        return BuildRep(" ".join(userpkgs), " ".join(allpkgs), image)
-
-    def find_reverse_depends(self, pn):
-        revdeps = []
-        it = self.contents.get_iter_first()
-
-        while it:
-            name = self.contents.get_value(it, self.COL_NAME)
-            itype = self.contents.get_value(it, self.COL_TYPE)
-            deps = self.contents.get_value(it, self.COL_DEPS)
-
-            it = self.contents.iter_next(it)
-
-            if not itype == 'package':
-                continue
-
-            if pn in deps:
-                revdeps.append(name)
-
-        if pn in revdeps:
-            revdeps.remove(pn)
-        return revdeps
-
-    def set_selected_image(self, img):
-        self.selected_image = img
-        path = self.find_path_for_item(img)
-        self.include_item(item_path=path,
-                          binb="User Selected",
-                          image_contents=True)
-
-        self.emit("image-changed", self.selected_image)
-
-    def set_selected_packages(self, pkglist):
-        selected = pkglist
-        it = self.get_iter_first()
-
-        while it:
-            name = self.get_value(it, self.COL_NAME)
-            if name in pkglist:
-                pkglist.remove(name)
-                path = self.get_path(it)
-                self.include_item(item_path=path,
-                                  binb="User Selected")
-                if len(pkglist) == 0:
-                    return
-            it = self.iter_next(it)
-
-    def find_image_path(self, image):
-        it = self.images.get_iter_first()
-
-        while it:
-            image_name = self.images.get_value(it, self.COL_NAME)
-            if image_name == image:
-                path = self.images.get_value(it, self.COL_PATH)
-                meta_pattern = "(\S*)/(meta*/)(\S*)"
-                meta_match = re.search(meta_pattern, path)
-                if meta_match:
-                    _, lyr, bbrel = path.partition(meta_match.group(2))
-                    if bbrel:
-                        path = bbrel
-                return path
-            it = self.images.iter_next(it)

bitbake/lib/bb/ui/crumbs/template.py

@@ -0,0 +1,196 @@
+#
+# BitBake Graphical GTK User Interface
+#
+# Copyright (C) 2011        Intel Corporation
+#
+# Authored by Shane Wang <shane.wang@intel.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import gobject
+import os
+import re
+
+class File(gobject.GObject):
+
+    def __init__(self, pathfilename, suffix):
+        if not pathfilename.endswith(suffix):
+            pathfilename = "%s%s" % (pathfilename, suffix)
+        gobject.GObject.__init__(self)
+        self.pathfilename = pathfilename
+
+    def readFile(self):
+        if not os.path.isfile(self.pathfilename):
+            return None
+        if not os.path.exists(self.pathfilename):
+            return None
+
+        with open(self.pathfilename, 'r') as f:
+            contents = f.readlines()
+        f.close()
+
+        return contents
+
+    def writeFile(self, contents):
+        if os.path.exists(self.pathfilename):
+            orig = "%s.orig" % self.pathfilename
+            if os.path.exists(orig):
+                os.remove(orig)
+            os.rename(self.pathfilename, orig)
+
+        with open(self.pathfilename, 'w') as f:
+            f.write(contents)
+        f.close()
+
+class ConfigFile(File):
+    """
+    This object does save general config file. (say bblayers.conf, or local.conf). Again, it is the base class for other template files and image bb files.
+    """
+    def __init__(self, pathfilename, suffix = None, header = None):
+        if suffix:
+            File.__init__(self, pathfilename, suffix)
+        else:
+            File.__init__(self, pathfilename, ".conf")
+        if header:
+            self.header = header
+        else:
+            self.header = "# Config generated by Hob\n\n"
+        self.dictionary = {}
+
+    def setVar(self, var, val):
+        if isinstance(val, list):
+            liststr = ""
+            if val:
+                i = 0
+                for value in val:
+                    if i < len(val) - 1:
+                        liststr += "%s " % value
+                    else:
+                        liststr += "%s" % value
+                    i += 1
+            self.dictionary[var] = liststr
+        else:
+            self.dictionary[var] = val
+
+    def save(self):
+        contents = self.header
+        for var, val in self.dictionary.items():
+            contents += "%s = \"%s\"\n" % (var, val)
+        File.writeFile(self, contents)
+
+class HobTemplateFile(ConfigFile):
+    """
+    This object does save or load hob specific file.
+    """
+    def __init__(self, pathfilename):
+        ConfigFile.__init__(self, pathfilename, ".hob", "# Hob Template generated by Hob\n\n")
+
+    def getVar(self, var):
+        if var in self.dictionary: 
+            return self.dictionary[var]
+        else:
+            return ""
+ 
+    def load(self):
+        contents = ConfigFile.readFile(self)
+        self.dictionary.clear()
+
+        pattern = "^\s*(\S+)\s*=\s*(\".*?\")"
+
+        for line in contents:
+            match = re.search(pattern, line)
+            if match:
+                var = match.group(1)
+                val = match.group(2).strip('"')
+                self.dictionary[var] = val
+        return self.dictionary
+
+class RecipeFile(ConfigFile):
+    """
+    This object is for image bb file.
+    """
+    def __init__(self, pathfilename):
+        ConfigFile.__init__(self, pathfilename, ".bb", "# Recipe generated by Hob\n\ninherit core-image\n")
+
+class TemplateMgr(gobject.GObject):
+
+    __gLocalVars__ = ["MACHINE", "PACKAGE_CLASSES", "DISTRO", "DL_DIR", "SSTATE_DIR", "SSTATE_MIRROR", "PARALLEL_MAKE", "BB_NUMBER_THREADS", "CONF_VERSION"]
+    __gBBLayersVars__ = ["BBLAYERS", "LCONF_VERSION"]
+    __gRecipeVars__ = ["DEPENDS", "IMAGE_INSTALL"]
+
+    def __init__(self):
+        gobject.GObject.__init__(self)
+        self.template_hob = None
+        self.bblayers_conf = None
+        self.local_conf = None
+        self.image_bb = None
+
+    @classmethod
+    def convert_to_template_pathfilename(cls, filename, path):
+        return "%s/%s%s%s" % (path, "template-", filename, ".hob")
+
+    @classmethod
+    def convert_to_bblayers_pathfilename(cls, filename, path):
+        return "%s/%s%s%s" % (path, "bblayers-", filename, ".conf")
+
+    @classmethod
+    def convert_to_local_pathfilename(cls, filename, path):
+        return "%s/%s%s%s" % (path, "local-", filename, ".conf")
+
+    @classmethod
+    def convert_to_image_pathfilename(cls, filename, path):
+        return "%s/%s%s%s" % (path, "hob-image-", filename, ".bb")
+
+    def open(self, filename, path):
+        self.template_hob = HobTemplateFile(TemplateMgr.convert_to_template_pathfilename(filename, path))
+        self.bblayers_conf = ConfigFile(TemplateMgr.convert_to_bblayers_pathfilename(filename, path))
+        self.local_conf = ConfigFile(TemplateMgr.convert_to_local_pathfilename(filename, path))
+        self.image_bb = RecipeFile(TemplateMgr.convert_to_image_pathfilename(filename, path))
+
+    def setVar(self, var, val):
+        if var in TemplateMgr.__gLocalVars__:
+            self.local_conf.setVar(var, val)
+        if var in TemplateMgr.__gBBLayersVars__:
+            self.bblayers_conf.setVar(var, val)
+        if var in TemplateMgr.__gRecipeVars__:
+            self.image_bb.setVar(var, val)
+
+        self.template_hob.setVar(var, val)
+
+    def save(self):
+        self.local_conf.save()
+        self.bblayers_conf.save()
+        self.image_bb.save()
+        self.template_hob.save()
+
+    def load(self, path):
+        self.template_hob = HobTemplateFile(path)
+        self.dictionary = self.template_hob.load()
+
+    def getVar(self, var):
+        return self.template_hob.getVar(var)
+
+    def destroy(self):
+        if self.template_hob:
+            del self.template_hob
+            template_hob = None
+        if self.bblayers_conf:
+            del self.bblayers_conf
+            self.bblayers_conf = None
+        if self.local_conf:
+            del self.local_conf
+            self.local_conf = None
+        if self.image_bb:
+            del self.image_bb
+            self.image_bb = None

bitbake/lib/bb/ui/crumbs/utils.py

@@ -0,0 +1,33 @@
+#
+# BitBake UI Utils 
+#
+# Copyright (C) 2012 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+# This utility method looks for xterm or vte and return the 
+# frist to exist, currently we are keeping this simple, but 
+# we will likely move the oe.terminal implementation into 
+# bitbake which will allow more flexibility.
+
+import os
+
+def which_terminal():
+    term = bb.utils.which(os.environ["PATH"], "xterm")
+    if term:
+        return term + " -e "
+    term = bb.utils.which(os.environ["PATH"], "vte")
+    if term:
+        return term + " -c "
+    return None

bitbake/lib/bb/ui/depexp.py

@@ -198,17 +198,23 @@ class gtkthread(threading.Thread):
 
 def main(server, eventHandler):
     try:
-        cmdline = server.runCommand(["getCmdLineAction"])
-        if cmdline and not cmdline['action']:
-            print(cmdline['msg'])
-            return
-        elif not cmdline or (cmdline['action'] and cmdline['action'][0] != "generateDotGraph"):
+        cmdline, error = server.runCommand(["getCmdLineAction"])
+        if error:
+            print("Error getting bitbake commandline: %s" % error)
+            return 1
+        elif not cmdline:
+            print("Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.")
+            return 1
+        elif not cmdline or cmdline[0] != "generateDotGraph":
             print("This UI is only compatible with the -g option")
-            return
-        ret = server.runCommand(["generateDepTreeEvent", cmdline['action'][1], cmdline['action'][2]])
-        if ret != True:
-            print("Couldn't run command! %s" % ret)
-            return
+            return 1
+        ret, error = server.runCommand(["generateDepTreeEvent", cmdline[1], cmdline[2]])
+        if error:
+            print("Error running command '%s': %s" % (cmdline, error))
+            return 1
+        elif ret != True:
+            print("Error running command '%s': returned %s" % (cmdline, ret))
+            return 1
     except xmlrpclib.Fault as x:
         print("XMLRPC Fault getting commandline:\n %s" % x)
         return
@@ -229,7 +235,9 @@ def main(server, eventHandler):
         try:
             event = eventHandler.waitEvent(0.25)
             if gtkthread.quit.isSet():
-                server.runCommand(["stateStop"])
+                _, error = server.runCommand(["stateStop"])
+                if error:
+                    print('Unable to cleanly stop: %s' % error)
                 break
 
             if event is None:
@@ -302,9 +310,13 @@ def main(server, eventHandler):
                 break
             if shutdown == 1:
                 print("\nSecond Keyboard Interrupt, stopping...\n")
-                server.runCommand(["stateStop"])
+                _, error = server.runCommand(["stateStop"])
+                if error:
+                    print('Unable to cleanly stop: %s' % error)
             if shutdown == 0:
                 print("\nKeyboard Interrupt, closing down...\n")
-                server.runCommand(["stateShutdown"])
+                _, error = server.runCommand(["stateShutdown"])
+                if error:
+                    print('Unable to cleanly shutdown: %s' % error)
             shutdown = shutdown + 1
             pass

bitbake/lib/bb/ui/goggle.py

@@ -80,16 +80,19 @@ def main (server, eventHandler):
     running_build.connect ("build-failed", running_build_failed_cb)
 
     try:
-        cmdline = server.runCommand(["getCmdLineAction"])
-        if not cmdline:
+        cmdline, error = server.runCommand(["getCmdLineAction"])
+        if err:
+            print("Error getting bitbake commandline: %s" % error)
+            return 1
+        elif not cmdline:
             print("Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.")
             return 1
-        elif not cmdline['action']:
-            print(cmdline['msg'])
+        ret, error = server.runCommand(cmdline)
+        if error:
+            print("Error running command '%s': %s" % (cmdline, error))
             return 1
-        ret = server.runCommand(cmdline['action'])
-        if ret != True:
-            print("Couldn't get default commandline! %s" % ret)
+        elif ret != True:
+            print("Error running command '%s': returned %s" % (cmdline, ret))
             return 1
     except xmlrpclib.Fault as x:
         print("XMLRPC Fault getting commandline:\n %s" % x)

bitbake/lib/bb/ui/knotty.py

@@ -3,7 +3,7 @@
 #
 # Handling output to TTYs or files (no TTY)
 #
-# Copyright (C) 2006-2007 Richard Purdie
+# Copyright (C) 2006-2012 Richard Purdie
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -64,12 +64,61 @@ def new_progress(msg, maxval):
     else:
         return NonInteractiveProgress(msg, maxval)
 
-def main(server, eventHandler):
+def pluralise(singular, plural, qty):
+    if(qty == 1):
+        return singular % qty
+    else:
+        return plural % qty
+
+class TerminalFilter(object):
+    def __init__(self, main, helper, console, format):
+        self.main = main
+        self.helper = helper
+
+    def clearFooter(self):
+        return
+
+    def updateFooter(self):
+        if not main.shutdown or not self.helper.needUpdate:
+            return
+
+        activetasks = self.helper.running_tasks
+        runningpids = self.helper.running_pids
+
+        if len(runningpids) == 0:
+            return
+
+        self.helper.getTasks()
+
+        tasks = []
+        for t in runningpids:
+            tasks.append("%s (pid %s)" % (activetasks[t]["title"], t))
+
+        if main.shutdown:
+            print("Waiting for %s running tasks to finish:" % len(activetasks))
+        else:
+            print("Currently %s running tasks (%s of %s):" % (len(activetasks), self.helper.tasknumber_current, self.helper.tasknumber_total))
+        for tasknum, task in enumerate(tasks):
+            print("%s: %s" % (tasknum, task))
+
+    def finish(self):
+        return
+
+def main(server, eventHandler, tf = TerminalFilter):
 
     # Get values of variables which control our output
-    includelogs = server.runCommand(["getVariable", "BBINCLUDELOGS"])
-    loglines = server.runCommand(["getVariable", "BBINCLUDELOGS_LINES"])
-    consolelogfile = server.runCommand(["getVariable", "BB_CONSOLELOG"])
+    includelogs, error = server.runCommand(["getVariable", "BBINCLUDELOGS"])
+    if error:
+        logger.error("Unable to get the value of BBINCLUDELOGS variable: %s" % error)
+        return 1
+    loglines, error = server.runCommand(["getVariable", "BBINCLUDELOGS_LINES"])
+    if error:
+        logger.error("Unable to get the value of BBINCLUDELOGS_LINES variable: %s" % error)
+        return 1
+    consolelogfile, error = server.runCommand(["getVariable", "BB_CONSOLELOG"])
+    if error:
+        logger.error("Unable to get the value of BB_CONSOLELOG variable: %s" % error)
+        return 1
 
     helper = uihelper.BBUIHelper()
 
@@ -85,49 +134,53 @@ def main(server, eventHandler):
         logger.addHandler(consolelog)
 
     try:
-        cmdline = server.runCommand(["getCmdLineAction"])
-        if not cmdline:
+        cmdline, error = server.runCommand(["getCmdLineAction"])
+        if error:
+            logger.error("Unable to get bitbake commandline arguments: %s" % error)
+            return 1
+        elif not cmdline:
             print("Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.")
             return 1
-        elif not cmdline['action']:
-            print(cmdline['msg'])
+        ret, error = server.runCommand(cmdline)
+        if error:
+            logger.error("Command '%s' failed: %s" % (cmdline, error))
             return 1
-        ret = server.runCommand(cmdline['action'])
-        if ret != True:
-            print("Couldn't get default commandline! %s" % ret)
+        elif ret != True:
+            logger.error("Command '%s' failed: returned %s" % (cmdline, ret))
             return 1
     except xmlrpclib.Fault as x:
-        print("XMLRPC Fault getting commandline:\n %s" % x)
+        logger.error("XMLRPC Fault getting commandline:\n %s" % x)
         return 1
 
-
     parseprogress = None
     cacheprogress = None
-    shutdown = 0
+    main.shutdown = 0
+    interrupted = False
     return_value = 0
     errors = 0
     warnings = 0
+    taskfailures = []
+
+    termfilter = tf(main, helper, console, format)
+
     while True:
         try:
+            termfilter.updateFooter()
             event = eventHandler.waitEvent(0.25)
             if event is None:
+                if main.shutdown > 1:
+                    break
                 continue
             helper.eventHandler(event)
             if isinstance(event, bb.runqueue.runQueueExitWait):
-                if not shutdown:
-                    shutdown = 1
-            if shutdown and helper.needUpdate:
-                activetasks, failedtasks = helper.getTasks()
-                if activetasks:
-                    print("Waiting for %s active tasks to finish:" % len(activetasks))
-                    for tasknum, task in enumerate(activetasks):
-                        print("%s: %s (pid %s)" % (tasknum, activetasks[task]["title"], task))
+                if not main.shutdown:
+                    main.shutdown = 1
 
             if isinstance(event, logging.LogRecord):
                 if event.levelno >= format.ERROR:
                     errors = errors + 1
                     return_value = 1
-                if event.levelno >= format.WARNING:
+                elif event.levelno == format.WARNING:
                     warnings = warnings + 1
                 # For "normal" logging conditions, don't show note logs from tasks
                 # but do show them if the user has changed the default log level to 
@@ -141,8 +194,9 @@ def main(server, eventHandler):
                 return_value = 1
                 logfile = event.logfile
                 if logfile and os.path.exists(logfile):
+                    termfilter.clearFooter()
                     print("ERROR: Logfile of failure stored in: %s" % logfile)
-                    if 1 or includelogs:
+                    if includelogs and not event.errprinted:
                         print("Log data follows:")
                         f = open(logfile, "r")
                         lines = []
@@ -192,18 +246,19 @@ def main(server, eventHandler):
                 print("Loaded %d entries from dependency cache." % event.num_entries)
                 continue
 
-            if isinstance(event, bb.command.CommandCompleted):
-                break
             if isinstance(event, bb.command.CommandFailed):
                 return_value = event.exitcode
+                errors = errors + 1
                 logger.error("Command execution failed: %s", event.error)
-                break
+                main.shutdown = 2
+                continue
             if isinstance(event, bb.command.CommandExit):
                 if not return_value:
                     return_value = event.exitcode
                 continue
-            if isinstance(event, bb.cooker.CookerExit):
-                break
+            if isinstance(event, (bb.command.CommandCompleted, bb.cooker.CookerExit)):
+                main.shutdown = 2
+                continue
             if isinstance(event, bb.event.MultipleProviders):
                 logger.info("multiple providers are available for %s%s (%s)", event._is_runtime and "runtime " or "",
                             event._item,
@@ -227,6 +282,10 @@ def main(server, eventHandler):
                         logger.error("%s", reason)
                 continue
 
+            if isinstance(event, bb.runqueue.sceneQueueTaskStarted):
+                logger.info("Running setscene task %d of %d (%s)" % (event.stats.completed + event.stats.active + event.stats.failed + 1, event.stats.total, event.taskstring))
+                continue
+
             if isinstance(event, bb.runqueue.runQueueTaskStarted):
                 if event.noexec:
                     tasktype = 'noexec task'
@@ -240,10 +299,16 @@ def main(server, eventHandler):
                 continue
 
             if isinstance(event, bb.runqueue.runQueueTaskFailed):
+                taskfailures.append(event.taskstring)
                 logger.error("Task %s (%s) failed with exit code '%s'",
                              event.taskid, event.taskstring, event.exitcode)
                 continue
 
+            if isinstance(event, bb.runqueue.sceneQueueTaskFailed):
+                logger.warn("Setscene task %s (%s) failed with exit code '%s' - real task will be run instead",
+                             event.taskid, event.taskstring, event.exitcode)
+                continue
+
             # ignore
             if isinstance(event, (bb.event.BuildBase,
                                   bb.event.StampUpdate,
@@ -251,29 +316,56 @@ def main(server, eventHandler):
                                   bb.event.RecipeParsed,
                                   bb.event.RecipePreFinalise,
                                   bb.runqueue.runQueueEvent,
-                                  bb.runqueue.runQueueExitWait)):
+                                  bb.runqueue.runQueueExitWait,
+                                  bb.event.OperationStarted,
+                                  bb.event.OperationCompleted,
+                                  bb.event.OperationProgress)):
                 continue
 
             logger.error("Unknown event: %s", event)
 
         except EnvironmentError as ioerror:
+            termfilter.clearFooter()
             # ignore interrupted io
             if ioerror.args[0] == 4:
                 pass
         except KeyboardInterrupt:
-            if shutdown == 2:
-                print("\nThird Keyboard Interrupt, exit.\n")
-                break
-            if shutdown == 1:
+            import time
+            termfilter.clearFooter()
+            if main.shutdown == 1:
                 print("\nSecond Keyboard Interrupt, stopping...\n")
-                server.runCommand(["stateStop"])
-            if shutdown == 0:
+                _, error = server.runCommand(["stateStop"])
+                if error:
+                    logger.error("Unable to cleanly stop: %s" % error)
+            if main.shutdown == 0:
                 print("\nKeyboard Interrupt, closing down...\n")
-                server.runCommand(["stateShutdown"])
-            shutdown = shutdown + 1
+                interrupted = True
+                _, error = server.runCommand(["stateShutdown"])
+                if error:
+                    logger.error("Unable to cleanly shutdown: %s" % error)
+            main.shutdown = main.shutdown + 1
             pass
+
+    summary = ""
+    if taskfailures:
+        summary += pluralise("\nSummary: %s task failed:",
+                             "\nSummary: %s tasks failed:", len(taskfailures))
+        for failure in taskfailures:
+            summary += "\n  %s" % failure
     if warnings:
-        print("Summary: There were %s WARNING messages shown.\n" % warnings)
+        summary += pluralise("\nSummary: There was %s WARNING message shown.",
+                             "\nSummary: There were %s WARNING messages shown.", warnings)
     if return_value:
-        print("Summary: There were %s ERROR messages shown, returning a non-zero exit code.\n" % errors)
+        summary += pluralise("\nSummary: There was %s ERROR message shown, returning a non-zero exit code.",
+                             "\nSummary: There were %s ERROR messages shown, returning a non-zero exit code.", errors)
+    if summary:
+        print(summary)
+
+    if interrupted:
+        print("Execution was interrupted, returning a non-zero exit code.")
+        if return_value == 0:
+            return_value = 1
+
+    termfilter.finish()
+
     return return_value

bitbake/lib/bb/ui/knotty2.py

@@ -0,0 +1,109 @@
+#
+# BitBake (No)TTY UI Implementation (v2)
+#
+# Handling output to TTYs or files (no TTY)
+#
+# Copyright (C) 2012 Richard Purdie
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from bb.ui import knotty
+import logging
+import sys
+logger = logging.getLogger("BitBake")
+
+class InteractConsoleLogFilter(logging.Filter):
+    def __init__(self, tf, format):
+        self.tf = tf
+        self.format = format
+
+    def filter(self, record):
+        if record.levelno == self.format.NOTE and (record.msg.startswith("Running") or record.msg.startswith("package ")):
+            return False
+        self.tf.clearFooter()
+        return True
+
+class TerminalFilter2(object):
+    def __init__(self, main, helper, console, format):
+        self.main = main
+        self.helper = helper
+        self.cuu = None
+        self.stdinbackup = None
+        self.interactive = sys.stdout.isatty()
+        self.footer_present = False
+        self.lastpids = []
+
+        if not self.interactive:
+            return
+
+        import curses
+        import termios
+        import copy
+        self.curses = curses
+        self.termios = termios
+        try:
+            fd = sys.stdin.fileno()
+            self.stdinbackup = termios.tcgetattr(fd)
+            new = copy.deepcopy(self.stdinbackup)
+            new[3] = new[3] & ~termios.ECHO
+            termios.tcsetattr(fd, termios.TCSADRAIN, new)
+            curses.setupterm()
+            self.ed = curses.tigetstr("ed")
+            if self.ed:
+                self.cuu = curses.tigetstr("cuu")
+        except:
+            self.cuu = None
+        console.addFilter(InteractConsoleLogFilter(self, format))
+
+    def clearFooter(self):
+        if self.footer_present:
+            lines = self.footer_present
+            sys.stdout.write(self.curses.tparm(self.cuu, lines))
+            sys.stdout.write(self.curses.tparm(self.ed))
+        self.footer_present = False
+
+    def updateFooter(self):
+        if not self.cuu:
+            return
+        activetasks = self.helper.running_tasks
+        failedtasks = self.helper.failed_tasks
+        runningpids = self.helper.running_pids
+        if self.footer_present and (self.lastpids == runningpids):
+            return
+        if self.footer_present:
+            self.clearFooter()
+        if not activetasks:
+            return
+        lines = 1
+        tasks = []
+        for t in runningpids:
+            tasks.append("%s (pid %s)" % (activetasks[t]["title"], t))
+
+        if self.main.shutdown:
+            print("Waiting for %s running tasks to finish:" % len(activetasks))
+        else:
+            print("Currently %s running tasks (%s of %s):" % (len(activetasks), self.helper.tasknumber_current, self.helper.tasknumber_total))
+        for tasknum, task in enumerate(tasks):
+            print("%s: %s" % (tasknum, task))
+            lines = lines + 1
+        self.footer_present = lines
+        self.lastpids = runningpids[:]
+
+    def finish(self):
+        if self.stdinbackup:
+            fd = sys.stdin.fileno()
+            self.termios.tcsetattr(fd, self.termios.TCSADRAIN, self.stdinbackup)
+
+def main(server, eventHandler):
+    bb.ui.knotty.main(server, eventHandler, TerminalFilter2)

bitbake/lib/bb/ui/ncurses.py

@@ -230,15 +230,18 @@ class NCursesUI:
         shutdown = 0
 
         try:
-            cmdline = server.runCommand(["getCmdLineAction"])
+            cmdline, error = server.runCommand(["getCmdLineAction"])
             if not cmdline:
                 print("Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.")
                 return
-            elif not cmdline['action']:
-                print(cmdline['msg'])
+            elif error:
+                print("Error getting bitbake commandline: %s" % error)
                 return
-            ret = server.runCommand(cmdline['action'])
-            if ret != True:
+            ret, error = server.runCommand(cmdline)
+            if error:
+                print("Error running command '%s': %s" % (cmdline, error))
+                return
+            elif ret != True:
                 print("Couldn't get default commandlind! %s" % ret)
                 return
         except xmlrpclib.Fault as x:
@@ -337,10 +340,14 @@ class NCursesUI:
                     exitflag = True
                 if shutdown == 1:
                     mw.appendText("Second Keyboard Interrupt, stopping...\n")
-                    server.runCommand(["stateStop"])
+                    _, error = server.runCommand(["stateStop"])
+                    if error:
+                        print("Unable to cleanly stop: %s" % error)
                 if shutdown == 0:
                     mw.appendText("Keyboard Interrupt, closing down...\n")
-                    server.runCommand(["stateShutdown"])
+                    _, error = server.runCommand(["stateShutdown"])
+                    if error:
+                        print("Unable to cleanly shutdown: %s" % error)
                 shutdown = shutdown + 1
                 pass
 

bitbake/lib/bb/ui/uievent.py

@@ -28,13 +28,14 @@ import socket, threading, pickle
 from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
 
 class BBUIEventQueue:
-    def __init__(self, BBServer):
+    def __init__(self, BBServer, clientinfo=("localhost, 0")):
 
         self.eventQueue = []
         self.eventQueueLock = threading.Lock()
         self.eventQueueNotify = threading.Event()
 
         self.BBServer = BBServer
+        self.clientinfo = clientinfo
 
         self.t = threading.Thread()
         self.t.setDaemon(True)
@@ -72,7 +73,7 @@ class BBUIEventQueue:
 
     def startCallbackHandler(self):
 
-        server = UIXMLRPCServer()
+        server = UIXMLRPCServer(self.clientinfo)
         self.host, self.port = server.socket.getsockname()
 
         server.register_function( self.system_quit, "event.quit" )
@@ -98,7 +99,7 @@ class BBUIEventQueue:
 
 class UIXMLRPCServer (SimpleXMLRPCServer):
 
-    def __init__( self, interface = ("localhost", 0) ):
+    def __init__( self, interface ):
         self.quit = False
         SimpleXMLRPCServer.__init__( self,
                                     interface,

bitbake/lib/bb/ui/uihelper.py

@@ -23,20 +23,77 @@ class BBUIHelper:
     def __init__(self):
         self.needUpdate = False
         self.running_tasks = {}
+        # Running PIDs preserves the order tasks were executed in
+        self.running_pids = []
         self.failed_tasks = []
+        self.tasknumber_current = 0
+        self.tasknumber_total = 0
 
     def eventHandler(self, event):
         if isinstance(event, bb.build.TaskStarted):
             self.running_tasks[event.pid] = { 'title' : "%s %s" % (event._package, event._task) }
+            self.running_pids.append(event.pid)
             self.needUpdate = True
         if isinstance(event, bb.build.TaskSucceeded):
             del self.running_tasks[event.pid]
+            self.running_pids.remove(event.pid)
+            self.needUpdate = True
+        if isinstance(event, bb.build.TaskFailedSilent):
+            del self.running_tasks[event.pid]
+            self.running_pids.remove(event.pid)
+            # Don't add to the failed tasks list since this is e.g. a setscene task failure
             self.needUpdate = True
         if isinstance(event, bb.build.TaskFailed):
             del self.running_tasks[event.pid]
+            self.running_pids.remove(event.pid)
             self.failed_tasks.append( { 'title' : "%s %s" % (event._package, event._task)})
             self.needUpdate = True
+        if isinstance(event, bb.runqueue.runQueueTaskStarted):
+            self.tasknumber_current = event.stats.completed + event.stats.active + event.stats.failed + 1
+            self.tasknumber_total = event.stats.total
 
     def getTasks(self):
         self.needUpdate = False
         return (self.running_tasks, self.failed_tasks)
+
+    def findServerDetails(self):
+        import sys
+        import optparse
+        from bb.server.xmlrpc import BitbakeServerInfo, BitBakeServerConnection
+        host = ""
+        port = 0
+        bind = ""
+        parser = optparse.OptionParser(
+            usage = """%prog -H host -P port -B bindaddr""")
+
+        parser.add_option("-H", "--host", help = "Bitbake server's IP address",
+                   action = "store", dest = "host", default = None)
+
+        parser.add_option("-P", "--port", help = "Bitbake server's Port number",
+                   action = "store", dest = "port", default = None)
+
+        parser.add_option("-B", "--bind", help = "Hob2 local bind address",
+                   action = "store", dest = "bind", default = None)
+
+        options, args = parser.parse_args(sys.argv)
+        for key, val in options.__dict__.items():
+            if key == 'host' and val:
+                host = val
+            elif key == 'port' and val:
+                port = int(val)
+            elif key == 'bind' and val:
+                bind = val
+
+        if not host or not port or not bind:
+            parser.print_usage()
+            sys.exit(1)
+
+        serverinfo = BitbakeServerInfo(host, port)
+        clientinfo = (bind, 0)
+        connection = BitBakeServerConnection(serverinfo, clientinfo)
+
+        server = connection.connection
+        eventHandler = connection.events
+
+        return server, eventHandler, host, bind
+

bitbake/lib/bb/utils.py

@@ -25,14 +25,12 @@ import errno
 import logging
 import bb
 import bb.msg
+import multiprocessing
 from commands import getstatusoutput
 from contextlib import contextmanager
 
 logger = logging.getLogger("BitBake.Util")
 
-# Version comparison
-separators = ".-"
-
 # Context used in better_exec, eval
 _context = {
     "os": os,
@@ -47,48 +45,57 @@ def explode_version(s):
     while (s != ''):
         if s[0] in string.digits:
             m = numeric_regexp.match(s)
-            r.append(int(m.group(1)))
+            r.append((0, int(m.group(1))))
             s = m.group(2)
             continue
         if s[0] in string.letters:
             m = alpha_regexp.match(s)
-            r.append(m.group(1))
+            r.append((1, m.group(1)))
             s = m.group(2)
             continue
-        r.append(s[0])
+        if s[0] == '~':
+            r.append((-1, s[0]))
+        else:
+            r.append((2, s[0]))
         s = s[1:]
     return r
 
+def split_version(s):
+    """Split a version string into its constituent parts (PE, PV, PR)"""
+    s = s.strip(" <>=")
+    e = 0
+    if s.count(':'):
+        e = int(s.split(":")[0])
+        s = s.split(":")[1]
+    r = ""
+    if s.count('-'):
+        r = s.rsplit("-", 1)[1]
+        s = s.rsplit("-", 1)[0]
+    v = s
+    return (e, v, r)
+
 def vercmp_part(a, b):
     va = explode_version(a)
     vb = explode_version(b)
-    sa = False
-    sb = False
     while True:
         if va == []:
-            ca = None
+            (oa, ca) = (0, None)
         else:
-            ca = va.pop(0)
+            (oa, ca) = va.pop(0)
         if vb == []:
-            cb = None
+            (ob, cb) = (0, None)
         else:
-            cb = vb.pop(0)
-        if ca == None and cb == None:
+            (ob, cb) = vb.pop(0)
+        if (oa, ca) == (0, None) and (ob, cb) == (0, None):
             return 0
-
-        if isinstance(ca, basestring):
-            sa = ca in separators
-        if isinstance(cb, basestring):
-            sb = cb in separators
-        if sa and not sb:
+        if oa < ob:
             return -1
-        if not sa and sb:
+        elif oa > ob:
             return 1
-
-        if ca > cb:
-            return 1
-        if ca < cb:
+        elif ca < cb:
             return -1
+        elif ca > cb:
+            return 1
 
 def vercmp(ta, tb):
     (ea, va, ra) = ta
@@ -862,3 +869,6 @@ def contains(variable, checkvalues, truevalue, falsevalue, d):
     if checkvalues.issubset(val): 
         return truevalue
     return falsevalue
+
+def cpu_count():
+    return multiprocessing.cpu_count()

bitbake/lib/prserv/serv.py

@@ -1,7 +1,12 @@
 import os,sys,logging
 import signal, time, atexit, threading
 from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
-import xmlrpclib,sqlite3
+import xmlrpclib
+
+try:
+    import sqlite3
+except ImportError:
+    from pysqlite2 import dbapi2 as sqlite3
 
 import bb.server.xmlrpc
 import prserv
@@ -263,7 +268,7 @@ def is_local_special(host, port):
 
 def auto_start(d):
     global singleton
-    if d.getVar('USE_PR_SERV', True) == '0':
+    if (not d.getVar('PRSERV_HOST', True)) or (not d.getVar('PRSERV_PORT', True)):
         return True
 
     if is_local_special(d.getVar('PRSERV_HOST', True), int(d.getVar('PRSERV_PORT', True))) and not singleton: