wic: Add mic w/pykickstart

This is the starting point for the implemention described in [YOCTO
3847] which came to the conclusion that it would make sense to use
kickstart syntax to implement image creation in OpenEmbedded.  I
subsequently realized that there was an existing tool that already
implemented image creation using kickstart syntax, the Tizen/Meego mic
tool.  As such, it made sense to use that as a starting point - this
commit essentially just copies the relevant Python code from the MIC
tool to the scripts/lib dir, where it can be accessed by the
previously created wic tool.

Most of this will be removed or renamed by later commits, since we're
initially focusing on partitioning only.  Care should be taken so that
we can easily add back any additional functionality should we decide
later to expand the tool, though (we may also want to contribute our
local changes to the mic tool to the Tizen project if it makes sense,
and therefore should avoid gratuitous changes to the original code if
possible).

Added the /mic subdir from Tizen mic repo as a starting point:

 git clone git://review.tizen.org/tools/mic.git

 For reference, the top commit:

 commit 20164175ddc234a17b8a12c33d04b012347b1530
 Author: Gui Chen <gui.chen@intel.com>
 Date:   Sun Jun 30 22:32:16 2013 -0400

    bump up to 0.19.2

Also added the /plugins subdir, moved to under the /mic subdir (to
match the default plugin_dir location in mic.conf.in, which was
renamed to yocto-image.conf (moved and renamed by later patches) and
put into /scripts.

(From OE-Core rev: 31f0360f1fd4ebc9dfcaed42d1c50d2448b4632e)

Signed-off-by: Tom Zanussi <tom.zanussi@linux.intel.com>
Signed-off-by: Saul Wold <sgw@linux.intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

scripts/lib/mic/plugins/backend/yumpkgmgr.py

@@ -0,0 +1,490 @@
+#!/usr/bin/python -tt
+#
+# Copyright (c) 2007 Red Hat  Inc.
+# Copyright (c) 2010, 2011 Intel, Inc.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; version 2 of the License
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc., 59
+# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+import os, sys
+import re
+import tempfile
+import glob
+from string import Template
+
+import rpmUtils
+import yum
+
+from mic import msger
+from mic.kickstart import ksparser
+from mic.utils import misc, rpmmisc
+from mic.utils.grabber import TextProgress
+from mic.utils.proxy import get_proxy_for
+from mic.utils.errors import CreatorError
+from mic.imager.baseimager import BaseImageCreator
+
+YUMCONF_TEMP = """[main]
+installroot=$installroot
+cachedir=/var/cache/yum
+persistdir=/var/lib/yum
+plugins=0
+reposdir=
+failovermethod=priority
+http_caching=packages
+sslverify=1
+"""
+
+class MyYumRepository(yum.yumRepo.YumRepository):
+    def __del__(self):
+        pass
+
+    def dirSetup(self):
+        super(MyYumRepository, self).dirSetup()
+        # relocate package dir
+        pkgdir = os.path.join(self.basecachedir, 'packages', self.id)
+        self.setAttribute('_dir_setup_pkgdir', pkgdir)
+        self._dirSetupMkdir_p(self.pkgdir)
+
+    def _getFile(self, url=None,
+                       relative=None,
+                       local=None,
+                       start=None,
+                       end=None,
+                       copy_local=None,
+                       checkfunc=None,
+                       text=None,
+                       reget='simple',
+                       cache=True,
+                       size=None):
+
+        m2c_connection = None
+        if not self.sslverify:
+            try:
+                import M2Crypto
+                m2c_connection = M2Crypto.SSL.Connection.clientPostConnectionCheck
+                M2Crypto.SSL.Connection.clientPostConnectionCheck = None
+            except ImportError, err:
+                raise CreatorError("%s, please try to install python-m2crypto" % str(err))
+
+        proxy = None
+        if url:
+            proxy = get_proxy_for(url)
+        else:
+            proxy = get_proxy_for(self.urls[0])
+
+        if proxy:
+            self.proxy = str(proxy)
+
+        size = int(size) if size else None
+        rvalue = super(MyYumRepository, self)._getFile(url,
+                                                       relative,
+                                                       local,
+                                                       start,
+                                                       end,
+                                                       copy_local,
+                                                       checkfunc,
+                                                       text,
+                                                       reget,
+                                                       cache,
+                                                       size)
+
+        if m2c_connection and \
+           not M2Crypto.SSL.Connection.clientPostConnectionCheck:
+            M2Crypto.SSL.Connection.clientPostConnectionCheck = m2c_connection
+
+        return rvalue
+
+from mic.pluginbase import BackendPlugin
+class Yum(BackendPlugin, yum.YumBase):
+    name = 'yum'
+
+    def __init__(self, target_arch, instroot, cachedir):
+        yum.YumBase.__init__(self)
+
+        self.cachedir = cachedir
+        self.instroot  = instroot
+        self.target_arch = target_arch
+
+        if self.target_arch:
+            if not rpmUtils.arch.arches.has_key(self.target_arch):
+                rpmUtils.arch.arches["armv7hl"] = "noarch"
+                rpmUtils.arch.arches["armv7tnhl"] = "armv7nhl"
+                rpmUtils.arch.arches["armv7tnhl"] = "armv7thl"
+                rpmUtils.arch.arches["armv7thl"] = "armv7hl"
+                rpmUtils.arch.arches["armv7nhl"] = "armv7hl"
+            self.arch.setup_arch(self.target_arch)
+
+        self.__pkgs_license = {}
+        self.__pkgs_content = {}
+        self.__pkgs_vcsinfo = {}
+
+        self.install_debuginfo = False
+
+    def doFileLogSetup(self, uid, logfile):
+        # don't do the file log for the livecd as it can lead to open fds
+        # being left and an inability to clean up after ourself
+        pass
+
+    def close(self):
+        try:
+            os.unlink(self.confpath)
+            os.unlink(self.conf.installroot + "/yum.conf")
+        except:
+            pass
+
+        if self.ts:
+            self.ts.close()
+        self._delRepos()
+        self._delSacks()
+        yum.YumBase.close(self)
+        self.closeRpmDB()
+
+        if not os.path.exists("/etc/fedora-release") and \
+           not os.path.exists("/etc/meego-release"):
+            for i in range(3, os.sysconf("SC_OPEN_MAX")):
+                try:
+                    os.close(i)
+                except:
+                    pass
+
+    def __del__(self):
+        pass
+
+    def _writeConf(self, confpath, installroot):
+        conf = Template(YUMCONF_TEMP).safe_substitute(installroot=installroot)
+
+        f = file(confpath, "w+")
+        f.write(conf)
+        f.close()
+
+        os.chmod(confpath, 0644)
+
+    def _cleanupRpmdbLocks(self, installroot):
+        # cleans up temporary files left by bdb so that differing
+        # versions of rpm don't cause problems
+        for f in glob.glob(installroot + "/var/lib/rpm/__db*"):
+            os.unlink(f)
+
+    def setup(self):
+        # create yum.conf
+        (fn, self.confpath) = tempfile.mkstemp(dir=self.cachedir,
+                                               prefix='yum.conf-')
+        os.close(fn)
+        self._writeConf(self.confpath, self.instroot)
+        self._cleanupRpmdbLocks(self.instroot)
+        # do setup
+        self.doConfigSetup(fn = self.confpath, root = self.instroot)
+        self.conf.cache = 0
+        self.doTsSetup()
+        self.doRpmDBSetup()
+        self.doRepoSetup()
+        self.doSackSetup()
+
+    def preInstall(self, pkg):
+        # FIXME: handle pre-install package
+        return None
+
+    def selectPackage(self, pkg):
+        """Select a given package.
+        Can be specified with name.arch or name*
+        """
+
+        try:
+            self.install(pattern = pkg)
+            return None
+        except yum.Errors.InstallError:
+            return "No package(s) available to install"
+        except yum.Errors.RepoError, e:
+            raise CreatorError("Unable to download from repo : %s" % (e,))
+        except yum.Errors.YumBaseError, e:
+            raise CreatorError("Unable to install: %s" % (e,))
+
+    def deselectPackage(self, pkg):
+        """Deselect package.  Can be specified as name.arch or name*
+        """
+
+        sp = pkg.rsplit(".", 2)
+        txmbrs = []
+        if len(sp) == 2:
+            txmbrs = self.tsInfo.matchNaevr(name=sp[0], arch=sp[1])
+
+        if len(txmbrs) == 0:
+            exact, match, unmatch = yum.packages.parsePackages(
+                                            self.pkgSack.returnPackages(),
+                                            [pkg],
+                                            casematch=1)
+            for p in exact + match:
+                txmbrs.append(p)
+
+        if len(txmbrs) > 0:
+            for x in txmbrs:
+                self.tsInfo.remove(x.pkgtup)
+                # we also need to remove from the conditionals
+                # dict so that things don't get pulled back in as a result
+                # of them.  yes, this is ugly.  conditionals should die.
+                for req, pkgs in self.tsInfo.conditionals.iteritems():
+                    if x in pkgs:
+                        pkgs.remove(x)
+                        self.tsInfo.conditionals[req] = pkgs
+        else:
+            msger.warning("No such package %s to remove" %(pkg,))
+
+    def selectGroup(self, grp, include = ksparser.GROUP_DEFAULT):
+        try:
+            yum.YumBase.selectGroup(self, grp)
+            if include == ksparser.GROUP_REQUIRED:
+                for p in grp.default_packages.keys():
+                    self.deselectPackage(p)
+
+            elif include == ksparser.GROUP_ALL:
+                for p in grp.optional_packages.keys():
+                    self.selectPackage(p)
+
+            return None
+        except (yum.Errors.InstallError, yum.Errors.GroupsError), e:
+            return e
+        except yum.Errors.RepoError, e:
+            raise CreatorError("Unable to download from repo : %s" % (e,))
+        except yum.Errors.YumBaseError, e:
+            raise CreatorError("Unable to install: %s" % (e,))
+
+    def addRepository(self, name, url = None, mirrorlist = None, proxy = None,
+                      proxy_username = None, proxy_password = None,
+                      inc = None, exc = None, ssl_verify=True, nocache=False,
+                      cost = None, priority=None):
+        # TODO: Handle priority attribute for repos
+        def _varSubstitute(option):
+            # takes a variable and substitutes like yum configs do
+            option = option.replace("$basearch", rpmUtils.arch.getBaseArch())
+            option = option.replace("$arch", rpmUtils.arch.getCanonArch())
+            return option
+
+        repo = MyYumRepository(name)
+
+        # Set proxy
+        repo.proxy = proxy
+        repo.proxy_username = proxy_username
+        repo.proxy_password = proxy_password
+
+        if url:
+            repo.baseurl.append(_varSubstitute(url))
+
+        # check LICENSE files
+        if not rpmmisc.checkRepositoryEULA(name, repo):
+            msger.warning('skip repo:%s for failed EULA confirmation' % name)
+            return None
+
+        if mirrorlist:
+            repo.mirrorlist = _varSubstitute(mirrorlist)
+
+        conf = yum.config.RepoConf()
+        for k, v in conf.iteritems():
+            if v or not hasattr(repo, k):
+                repo.setAttribute(k, v)
+
+        repo.sslverify = ssl_verify
+        repo.cache = not nocache
+
+        repo.basecachedir = self.cachedir
+        repo.base_persistdir = self.conf.persistdir
+        repo.failovermethod = "priority"
+        repo.metadata_expire = 0
+        # Enable gpg check for verifying corrupt packages
+        repo.gpgcheck = 1
+        repo.enable()
+        repo.setup(0)
+        self.repos.add(repo)
+        if cost:
+            repo.cost = cost
+
+        msger.verbose('repo: %s was added' % name)
+        return repo
+
+    def installLocal(self, pkg, po=None, updateonly=False):
+        ts = rpmUtils.transaction.initReadOnlyTransaction()
+        try:
+            hdr = rpmUtils.miscutils.hdrFromPackage(ts, pkg)
+        except rpmUtils.RpmUtilsError, e:
+            raise yum.Errors.MiscError, \
+                  'Could not open local rpm file: %s: %s' % (pkg, e)
+
+        self.deselectPackage(hdr['name'])
+        yum.YumBase.installLocal(self, pkg, po, updateonly)
+
+    def installHasFile(self, file):
+        provides_pkg = self.whatProvides(file, None, None)
+        dlpkgs = map(
+                    lambda x: x.po,
+                    filter(
+                        lambda txmbr: txmbr.ts_state in ("i", "u"),
+                        self.tsInfo.getMembers()))
+
+        for p in dlpkgs:
+            for q in provides_pkg:
+                if (p == q):
+                    return True
+
+        return False
+
+    def runInstall(self, checksize = 0):
+        os.environ["HOME"] = "/"
+        os.environ["LD_PRELOAD"] = ""
+        try:
+            (res, resmsg) = self.buildTransaction()
+        except yum.Errors.RepoError, e:
+            raise CreatorError("Unable to download from repo : %s" %(e,))
+
+        if res != 2:
+            raise CreatorError("Failed to build transaction : %s" \
+                               % str.join("\n", resmsg))
+
+        dlpkgs = map(
+                    lambda x: x.po,
+                    filter(
+                        lambda txmbr: txmbr.ts_state in ("i", "u"),
+                        self.tsInfo.getMembers()))
+
+        # record all pkg and the content
+        for pkg in dlpkgs:
+            pkg_long_name = misc.RPM_FMT % {
+                                'name': pkg.name,
+                                'arch': pkg.arch,
+                                'version': pkg.version,
+                                'release': pkg.release
+                            }
+            self.__pkgs_content[pkg_long_name] = pkg.files
+            license = pkg.license
+            if license in self.__pkgs_license.keys():
+                self.__pkgs_license[license].append(pkg_long_name)
+            else:
+                self.__pkgs_license[license] = [pkg_long_name]
+
+        total_count = len(dlpkgs)
+        cached_count = 0
+        download_total_size = sum(map(lambda x: int(x.packagesize), dlpkgs))
+
+        msger.info("\nChecking packages cached ...")
+        for po in dlpkgs:
+            local = po.localPkg()
+            repo = filter(lambda r: r.id == po.repoid, self.repos.listEnabled())[0]
+            if not repo.cache and os.path.exists(local):
+                os.unlink(local)
+            if not os.path.exists(local):
+                continue
+            if not self.verifyPkg(local, po, False):
+                msger.warning("Package %s is damaged: %s" \
+                              % (os.path.basename(local), local))
+            else:
+                download_total_size -= int(po.packagesize)
+                cached_count +=1
+
+        cache_avail_size = misc.get_filesystem_avail(self.cachedir)
+        if cache_avail_size < download_total_size:
+            raise CreatorError("No enough space used for downloading.")
+
+        # record the total size of installed pkgs
+        pkgs_total_size = 0L
+        for x in dlpkgs:
+            if hasattr(x, 'installedsize'):
+                pkgs_total_size += int(x.installedsize)
+            else:
+                pkgs_total_size += int(x.size)
+
+        # check needed size before actually download and install
+        if checksize and pkgs_total_size > checksize:
+            raise CreatorError("No enough space used for installing, "
+                               "please resize partition size in ks file")
+
+        msger.info("Packages: %d Total, %d Cached, %d Missed" \
+                   % (total_count, cached_count, total_count - cached_count))
+
+        try:
+            repos = self.repos.listEnabled()
+            for repo in repos:
+                repo.setCallback(TextProgress(total_count - cached_count))
+
+            self.downloadPkgs(dlpkgs)
+            # FIXME: sigcheck?
+
+            self.initActionTs()
+            self.populateTs(keepold=0)
+
+            deps = self.ts.check()
+            if len(deps) != 0:
+                # This isn't fatal, Ubuntu has this issue but it is ok.
+                msger.debug(deps)
+                msger.warning("Dependency check failed!")
+
+            rc = self.ts.order()
+            if rc != 0:
+                raise CreatorError("ordering packages for installation failed")
+
+            # FIXME: callback should be refactored a little in yum
+            cb = rpmmisc.RPMInstallCallback(self.ts)
+            cb.tsInfo = self.tsInfo
+            cb.filelog = False
+
+            msger.warning('\nCaution, do NOT interrupt the installation, '
+                          'else mic cannot finish the cleanup.')
+
+            installlogfile = "%s/__catched_stderr.buf" % (self.instroot)
+            msger.enable_logstderr(installlogfile)
+            self.runTransaction(cb)
+            self._cleanupRpmdbLocks(self.conf.installroot)
+
+        except rpmUtils.RpmUtilsError, e:
+            raise CreatorError("mic does NOT support delta rpm: %s" % e)
+        except yum.Errors.RepoError, e:
+            raise CreatorError("Unable to download from repo : %s" % e)
+        except yum.Errors.YumBaseError, e:
+            raise CreatorError("Unable to install: %s" % e)
+        finally:
+            msger.disable_logstderr()
+
+    def getVcsInfo(self):
+        return self.__pkgs_vcsinfo
+
+    def getAllContent(self):
+        return self.__pkgs_content
+
+    def getPkgsLicense(self):
+        return self.__pkgs_license
+
+    def getFilelist(self, pkgname):
+        if not pkgname:
+            return None
+
+        pkg = filter(lambda txmbr: txmbr.po.name == pkgname, self.tsInfo.getMembers())
+        if not pkg:
+            return None
+        return pkg[0].po.filelist
+
+    def package_url(self, pkgname):
+        pkgs = self.pkgSack.searchNevra(name=pkgname)
+        if pkgs:
+            proxy = None
+            proxies = None
+            url = pkgs[0].remote_url
+            repoid = pkgs[0].repoid
+            repos = filter(lambda r: r.id == repoid, self.repos.listEnabled())
+
+            if repos:
+                proxy = repos[0].proxy
+            if not proxy:
+                proxy = get_proxy_for(url)
+            if proxy:
+                proxies = {str(url.split(':')[0]): str(proxy)}
+
+            return (url, proxies)
+
+        return (None, None)

scripts/lib/mic/plugins/backend/zypppkgmgr.py

@@ -0,0 +1,973 @@
+#!/usr/bin/python -tt
+#
+# Copyright (c) 2010, 2011 Intel, Inc.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; version 2 of the License
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc., 59
+# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+import os
+import shutil
+import urlparse
+import rpm
+
+import zypp
+if not hasattr(zypp, 'PoolQuery') or \
+   not hasattr(zypp.RepoManager, 'loadSolvFile'):
+    raise ImportError("python-zypp in host system cannot support PoolQuery or "
+                      "loadSolvFile interface, please update it to enhanced "
+                      "version which can be found in download.tizen.org/tools")
+
+from mic import msger
+from mic.kickstart import ksparser
+from mic.utils import misc, rpmmisc, runner, fs_related
+from mic.utils.grabber import myurlgrab, TextProgress
+from mic.utils.proxy import get_proxy_for
+from mic.utils.errors import CreatorError, RepoError, RpmError
+from mic.imager.baseimager import BaseImageCreator
+
+class RepositoryStub:
+    def __init__(self):
+        self.name = None
+        self.baseurl = []
+        self.mirrorlist = None
+        self.proxy = None
+        self.proxy_username = None
+        self.proxy_password = None
+        self.nocache = False
+
+        self.enabled = True
+        self.autorefresh = True
+        self.keeppackages = True
+        self.priority = None
+
+from mic.pluginbase import BackendPlugin
+class Zypp(BackendPlugin):
+    name = 'zypp'
+
+    def __init__(self, target_arch, instroot, cachedir):
+        self.cachedir = cachedir
+        self.instroot  = instroot
+        self.target_arch = target_arch
+
+        self.__pkgs_license = {}
+        self.__pkgs_content = {}
+        self.__pkgs_vcsinfo = {}
+        self.repos = []
+        self.to_deselect = []
+        self.localpkgs = {}
+        self.repo_manager = None
+        self.repo_manager_options = None
+        self.Z = None
+        self.ts = None
+        self.ts_pre = None
+        self.incpkgs = {}
+        self.excpkgs = {}
+        self.pre_pkgs = []
+        self.probFilterFlags = [ rpm.RPMPROB_FILTER_OLDPACKAGE,
+                                 rpm.RPMPROB_FILTER_REPLACEPKG ]
+
+        self.has_prov_query = True
+        self.install_debuginfo = False
+
+    def doFileLogSetup(self, uid, logfile):
+        # don't do the file log for the livecd as it can lead to open fds
+        # being left and an inability to clean up after ourself
+        pass
+
+    def closeRpmDB(self):
+        pass
+
+    def close(self):
+        if self.ts:
+            self.ts.closeDB()
+            self.ts = None
+
+        if self.ts_pre:
+            self.ts_pre.closeDB()
+            self.ts = None
+
+        self.closeRpmDB()
+
+        if not os.path.exists("/etc/fedora-release") and \
+           not os.path.exists("/etc/meego-release"):
+            for i in range(3, os.sysconf("SC_OPEN_MAX")):
+                try:
+                    os.close(i)
+                except:
+                    pass
+
+    def __del__(self):
+        self.close()
+
+    def _cleanupRpmdbLocks(self, installroot):
+        # cleans up temporary files left by bdb so that differing
+        # versions of rpm don't cause problems
+        import glob
+        for f in glob.glob(installroot + "/var/lib/rpm/__db*"):
+            os.unlink(f)
+
+    def _cleanupZyppJunk(self, installroot):
+        try:
+            shutil.rmtree(os.path.join(installroot, '.zypp'))
+        except:
+            pass
+
+    def setup(self):
+        self._cleanupRpmdbLocks(self.instroot)
+
+    def whatObsolete(self, pkg):
+        query = zypp.PoolQuery()
+        query.addKind(zypp.ResKind.package)
+        query.addAttribute(zypp.SolvAttr.obsoletes, pkg)
+        query.setMatchExact()
+        for pi in query.queryResults(self.Z.pool()):
+            return pi
+        return None
+
+    def _zyppQueryPackage(self, pkg):
+        query = zypp.PoolQuery()
+        query.addKind(zypp.ResKind.package)
+        query.addAttribute(zypp.SolvAttr.name,pkg)
+        query.setMatchExact()
+        for pi in query.queryResults(self.Z.pool()):
+            return pi
+        return None
+
+    def _splitPkgString(self, pkg):
+        sp = pkg.rsplit(".",1)
+        name = sp[0]
+        arch = None
+        if len(sp) == 2:
+            arch = sp[1]
+            sysarch = zypp.Arch(self.target_arch)
+            if not zypp.Arch(arch).compatible_with (sysarch):
+                arch = None
+                name = ".".join(sp)
+        return name, arch
+
+    def selectPackage(self, pkg):
+        """Select a given package or package pattern, can be specified
+        with name.arch or name* or *name
+        """
+
+        if not self.Z:
+            self.__initialize_zypp()
+
+        def markPoolItem(obs, pi):
+            if obs == None:
+                pi.status().setToBeInstalled (zypp.ResStatus.USER)
+            else:
+                obs.status().setToBeInstalled (zypp.ResStatus.USER)
+
+        def cmpEVR(p1, p2):
+            # compare criterion: arch compatibility first, then repo
+            # priority, and version last
+            a1 = p1.arch()
+            a2 = p2.arch()
+            if str(a1) != str(a2):
+                if a1.compatible_with(a2):
+                    return -1
+                else:
+                    return 1
+            # Priority of a repository is an integer value between 0 (the
+            # highest priority) and 99 (the lowest priority)
+            pr1 = int(p1.repoInfo().priority())
+            pr2 = int(p2.repoInfo().priority())
+            if pr1 > pr2:
+                return -1
+            elif pr1 < pr2:
+                return 1
+
+            ed1 = p1.edition()
+            ed2 = p2.edition()
+            (e1, v1, r1) = map(str, [ed1.epoch(), ed1.version(), ed1.release()])
+            (e2, v2, r2) = map(str, [ed2.epoch(), ed2.version(), ed2.release()])
+            return rpm.labelCompare((e1, v1, r1), (e2, v2, r2))
+
+        found = False
+        startx = pkg.startswith("*")
+        endx = pkg.endswith("*")
+        ispattern = startx or endx
+        name, arch = self._splitPkgString(pkg)
+
+        q = zypp.PoolQuery()
+        q.addKind(zypp.ResKind.package)
+
+        if ispattern:
+            if startx and not endx:
+                pattern = '%s$' % (pkg[1:])
+            if endx and not startx:
+                pattern = '^%s' % (pkg[0:-1])
+            if endx and startx:
+                pattern = '%s' % (pkg[1:-1])
+            q.setMatchRegex()
+            q.addAttribute(zypp.SolvAttr.name,pattern)
+
+        elif arch:
+            q.setMatchExact()
+            q.addAttribute(zypp.SolvAttr.name,name)
+
+        else:
+            q.setMatchExact()
+            q.addAttribute(zypp.SolvAttr.name,pkg)
+
+        for pitem in sorted(
+                        q.queryResults(self.Z.pool()),
+                        cmp=lambda x,y: cmpEVR(zypp.asKindPackage(x), zypp.asKindPackage(y)),
+                        reverse=True):
+            item = zypp.asKindPackage(pitem)
+            if item.name() in self.excpkgs.keys() and \
+               self.excpkgs[item.name()] == item.repoInfo().name():
+                continue
+            if item.name() in self.incpkgs.keys() and \
+               self.incpkgs[item.name()] != item.repoInfo().name():
+                continue
+
+            found = True
+            obspkg = self.whatObsolete(item.name())
+            if arch:
+                if arch == str(item.arch()):
+                    item.status().setToBeInstalled (zypp.ResStatus.USER)
+            else:
+                markPoolItem(obspkg, pitem)
+            if not ispattern:
+                break
+
+        # Can't match using package name, then search from packge
+        # provides infomation
+        if found == False and not ispattern:
+            q.addAttribute(zypp.SolvAttr.provides, pkg)
+            q.addAttribute(zypp.SolvAttr.name,'')
+
+            for pitem in sorted(
+                            q.queryResults(self.Z.pool()),
+                            cmp=lambda x,y: cmpEVR(zypp.asKindPackage(x), zypp.asKindPackage(y)),
+                            reverse=True):
+                item = zypp.asKindPackage(pitem)
+                if item.name() in self.excpkgs.keys() and \
+                   self.excpkgs[item.name()] == item.repoInfo().name():
+                    continue
+                if item.name() in self.incpkgs.keys() and \
+                   self.incpkgs[item.name()] != item.repoInfo().name():
+                    continue
+
+                found = True
+                obspkg = self.whatObsolete(item.name())
+                markPoolItem(obspkg, pitem)
+                break
+
+        if found:
+            return None
+        else:
+            raise CreatorError("Unable to find package: %s" % (pkg,))
+
+    def inDeselectPackages(self, pitem):
+        """check if specified pacakges are in the list of inDeselectPackages
+        """
+        item = zypp.asKindPackage(pitem)
+        name = item.name()
+        for pkg in self.to_deselect:
+            startx = pkg.startswith("*")
+            endx = pkg.endswith("*")
+            ispattern = startx or endx
+            pkgname, pkgarch = self._splitPkgString(pkg)
+            if not ispattern:
+                if pkgarch:
+                    if name == pkgname and str(item.arch()) == pkgarch:
+                        return True;
+                else:
+                    if name == pkgname:
+                        return True;
+            else:
+                if startx and name.endswith(pkg[1:]):
+                    return True;
+                if endx and name.startswith(pkg[:-1]):
+                    return True;
+
+        return False;
+
+    def deselectPackage(self, pkg):
+        """collect packages should not be installed"""
+        self.to_deselect.append(pkg)
+
+    def selectGroup(self, grp, include = ksparser.GROUP_DEFAULT):
+        if not self.Z:
+            self.__initialize_zypp()
+        found = False
+        q=zypp.PoolQuery()
+        q.addKind(zypp.ResKind.pattern)
+        for pitem in q.queryResults(self.Z.pool()):
+            item = zypp.asKindPattern(pitem)
+            summary = "%s" % item.summary()
+            name = "%s" % item.name()
+            if name == grp or summary == grp:
+                found = True
+                pitem.status().setToBeInstalled (zypp.ResStatus.USER)
+                break
+
+        if found:
+            if include == ksparser.GROUP_REQUIRED:
+                map(
+                    lambda p: self.deselectPackage(p),
+                    grp.default_packages.keys())
+
+            return None
+        else:
+            raise CreatorError("Unable to find pattern: %s" % (grp,))
+
+    def addRepository(self, name,
+                            url = None,
+                            mirrorlist = None,
+                            proxy = None,
+                            proxy_username = None,
+                            proxy_password = None,
+                            inc = None,
+                            exc = None,
+                            ssl_verify = True,
+                            nocache = False,
+                            cost=None,
+                            priority=None):
+        # TODO: Handle cost attribute for repos
+
+        if not self.repo_manager:
+            self.__initialize_repo_manager()
+
+        if not proxy and url:
+            proxy = get_proxy_for(url)
+
+        repo = RepositoryStub()
+        repo.name = name
+        repo.id = name
+        repo.proxy = proxy
+        repo.proxy_username = proxy_username
+        repo.proxy_password = proxy_password
+        repo.ssl_verify = ssl_verify
+        repo.nocache = nocache
+        repo.baseurl.append(url)
+        if inc:
+            for pkg in inc:
+                self.incpkgs[pkg] = name
+        if exc:
+            for pkg in exc:
+                self.excpkgs[pkg] = name
+
+        # check LICENSE files
+        if not rpmmisc.checkRepositoryEULA(name, repo):
+            msger.warning('skip repo:%s for failed EULA confirmation' % name)
+            return None
+
+        if mirrorlist:
+            repo.mirrorlist = mirrorlist
+
+        # Enable gpg check for verifying corrupt packages
+        repo.gpgcheck = 1
+        if priority is not None:
+            # priority 0 has issue in RepoInfo.setPriority
+            repo.priority = priority + 1
+
+        try:
+            repo_info = zypp.RepoInfo()
+            repo_info.setAlias(repo.name)
+            repo_info.setName(repo.name)
+            repo_info.setEnabled(repo.enabled)
+            repo_info.setAutorefresh(repo.autorefresh)
+            repo_info.setKeepPackages(repo.keeppackages)
+            baseurl = zypp.Url(repo.baseurl[0])
+            if not ssl_verify:
+                baseurl.setQueryParam("ssl_verify", "no")
+            if proxy:
+                scheme, host, path, parm, query, frag = urlparse.urlparse(proxy)
+
+                proxyinfo = host.split(":")
+                host = proxyinfo[0]
+
+                port = "80"
+                if len(proxyinfo) > 1:
+                    port = proxyinfo[1]
+
+                if proxy.startswith("socks") and len(proxy.rsplit(':', 1)) == 2:
+                    host = proxy.rsplit(':', 1)[0]
+                    port = proxy.rsplit(':', 1)[1]
+
+                baseurl.setQueryParam ("proxy", host)
+                baseurl.setQueryParam ("proxyport", port)
+
+            repo.baseurl[0] = baseurl.asCompleteString()
+            self.repos.append(repo)
+
+            repo_info.addBaseUrl(baseurl)
+
+            if repo.priority is not None:
+                repo_info.setPriority(repo.priority)
+
+            # this hack is used to change zypp credential file location
+            # the default one is $HOME/.zypp, which cause conflicts when
+            # installing some basic packages, and the location doesn't
+            # have any interface actually, so use a tricky way anyway
+            homedir = None
+            if 'HOME' in os.environ:
+                homedir = os.environ['HOME']
+                os.environ['HOME'] = '/'
+            else:
+                os.environ['HOME'] = '/'
+
+            self.repo_manager.addRepository(repo_info)
+
+            # save back the $HOME env
+            if homedir:
+                os.environ['HOME'] = homedir
+            else:
+                del os.environ['HOME']
+
+            self.__build_repo_cache(name)
+
+        except RuntimeError, e:
+            raise CreatorError(str(e))
+
+        msger.verbose('repo: %s was added' % name)
+        return repo
+
+    def installHasFile(self, file):
+        return False
+
+    def preInstall(self, pkg):
+        self.pre_pkgs.append(pkg)
+
+    def runInstall(self, checksize = 0):
+        os.environ["HOME"] = "/"
+        os.environ["LD_PRELOAD"] = ""
+        self.buildTransaction()
+
+        todo = zypp.GetResolvablesToInsDel(self.Z.pool())
+        installed_pkgs = todo._toInstall
+        dlpkgs = []
+        for pitem in installed_pkgs:
+            if not zypp.isKindPattern(pitem) and \
+              not self.inDeselectPackages(pitem):
+                item = zypp.asKindPackage(pitem)
+                dlpkgs.append(item)
+
+                if not self.install_debuginfo or str(item.arch()) == "noarch":
+                    continue
+
+                dipkg = self._zyppQueryPackage("%s-debuginfo" % item.name())
+                if dipkg:
+                    ditem = zypp.asKindPackage(dipkg)
+                    dlpkgs.append(ditem)
+                else:
+                    msger.warning("No debuginfo rpm found for: %s" \
+                                  % item.name())
+
+        # record all pkg and the content
+        localpkgs = self.localpkgs.keys()
+        for pkg in dlpkgs:
+            license = ''
+            if pkg.name() in localpkgs:
+                hdr = rpmmisc.readRpmHeader(self.ts, self.localpkgs[pkg.name()])
+                pkg_long_name = misc.RPM_FMT % {
+                                    'name': hdr['name'],
+                                    'arch': hdr['arch'],
+                                    'version': hdr['version'],
+                                    'release': hdr['release']
+                                }
+                license = hdr['license']
+
+            else:
+                pkg_long_name = misc.RPM_FMT % {
+                                    'name': pkg.name(),
+                                    'arch': pkg.arch(),
+                                    'version': pkg.edition().version(),
+                                    'release': pkg.edition().release()
+                                }
+
+                license = pkg.license()
+
+            if license in self.__pkgs_license.keys():
+                self.__pkgs_license[license].append(pkg_long_name)
+            else:
+                self.__pkgs_license[license] = [pkg_long_name]
+
+        total_count = len(dlpkgs)
+        cached_count = 0
+        download_total_size = sum(map(lambda x: int(x.downloadSize()), dlpkgs))
+        localpkgs = self.localpkgs.keys()
+
+        msger.info("Checking packages cached ...")
+        for po in dlpkgs:
+            # Check if it is cached locally
+            if po.name() in localpkgs:
+                cached_count += 1
+            else:
+                local = self.getLocalPkgPath(po)
+                name = str(po.repoInfo().name())
+                try:
+                    repo = filter(lambda r: r.name == name, self.repos)[0]
+                except IndexError:
+                    repo = None
+                nocache = repo.nocache if repo else False
+
+                if os.path.exists(local):
+                    if nocache or self.checkPkg(local) !=0:
+                        os.unlink(local)
+                    else:
+                        download_total_size -= int(po.downloadSize())
+                        cached_count += 1
+        cache_avail_size = misc.get_filesystem_avail(self.cachedir)
+        if cache_avail_size < download_total_size:
+            raise CreatorError("No enough space used for downloading.")
+
+        # record the total size of installed pkgs
+        install_total_size = sum(map(lambda x: int(x.installSize()), dlpkgs))
+        # check needed size before actually download and install
+
+        # FIXME: for multiple partitions for loop type, check fails
+        #        skip the check temporarily
+        #if checksize and install_total_size > checksize:
+        #    raise CreatorError("No enough space used for installing, "
+        #                       "please resize partition size in ks file")
+
+        download_count =  total_count - cached_count
+        msger.info("Packages: %d Total, %d Cached, %d Missed" \
+                   % (total_count, cached_count, download_count))
+
+        try:
+            if download_count > 0:
+                msger.info("Downloading packages ...")
+            self.downloadPkgs(dlpkgs, download_count)
+
+            self.installPkgs(dlpkgs)
+
+        except (RepoError, RpmError):
+            raise
+        except Exception, e:
+            raise CreatorError("Package installation failed: %s" % (e,))
+
+    def getVcsInfo(self):
+        if self.__pkgs_vcsinfo:
+            return
+
+        if not self.ts:
+            self.__initialize_transaction()
+
+        mi = self.ts.dbMatch()
+        for hdr in mi:
+            lname = misc.RPM_FMT % {
+                        'name': hdr['name'],
+                        'arch': hdr['arch'],
+                        'version': hdr['version'],
+                        'release': hdr['release']
+                    }
+            self.__pkgs_vcsinfo[lname] = hdr['VCS']
+
+        return self.__pkgs_vcsinfo
+
+    def getAllContent(self):
+        if self.__pkgs_content:
+            return self.__pkgs_content
+
+        if not self.ts:
+            self.__initialize_transaction()
+
+        mi = self.ts.dbMatch()
+        for hdr in mi:
+            lname = misc.RPM_FMT % {
+                        'name': hdr['name'],
+                        'arch': hdr['arch'],
+                        'version': hdr['version'],
+                        'release': hdr['release']
+                    }
+            self.__pkgs_content[lname] = hdr['FILENAMES']
+
+        return self.__pkgs_content
+
+    def getPkgsLicense(self):
+        return self.__pkgs_license
+
+    def getFilelist(self, pkgname):
+        if not pkgname:
+            return None
+
+        if not self.ts:
+            self.__initialize_transaction()
+
+        mi = self.ts.dbMatch('name', pkgname)
+        for header in mi:
+            return header['FILENAMES']
+
+    def __initialize_repo_manager(self):
+        if self.repo_manager:
+            return
+
+        # Clean up repo metadata
+        shutil.rmtree(self.cachedir + "/etc", ignore_errors = True)
+        shutil.rmtree(self.cachedir + "/solv", ignore_errors = True)
+        shutil.rmtree(self.cachedir + "/raw", ignore_errors = True)
+
+        zypp.KeyRing.setDefaultAccept( zypp.KeyRing.ACCEPT_UNSIGNED_FILE
+                                     | zypp.KeyRing.ACCEPT_VERIFICATION_FAILED
+                                     | zypp.KeyRing.ACCEPT_UNKNOWNKEY
+                                     | zypp.KeyRing.TRUST_KEY_TEMPORARILY
+                                     )
+
+        self.repo_manager_options = \
+                zypp.RepoManagerOptions(zypp.Pathname(self.instroot))
+
+        self.repo_manager_options.knownReposPath = \
+                zypp.Pathname(self.cachedir + "/etc/zypp/repos.d")
+
+        self.repo_manager_options.repoCachePath = \
+                zypp.Pathname(self.cachedir)
+
+        self.repo_manager_options.repoRawCachePath = \
+                zypp.Pathname(self.cachedir + "/raw")
+
+        self.repo_manager_options.repoSolvCachePath = \
+                zypp.Pathname(self.cachedir + "/solv")
+
+        self.repo_manager_options.repoPackagesCachePath = \
+                zypp.Pathname(self.cachedir + "/packages")
+
+        self.repo_manager = zypp.RepoManager(self.repo_manager_options)
+
+    def __build_repo_cache(self, name):
+        repo = self.repo_manager.getRepositoryInfo(name)
+        if self.repo_manager.isCached(repo) or not repo.enabled():
+            return
+
+        msger.info('Refreshing repository: %s ...' % name)
+        self.repo_manager.buildCache(repo, zypp.RepoManager.BuildIfNeeded)
+
+    def __initialize_zypp(self):
+        if self.Z:
+            return
+
+        zconfig = zypp.ZConfig_instance()
+
+        # Set system architecture
+        if self.target_arch:
+            zconfig.setSystemArchitecture(zypp.Arch(self.target_arch))
+
+        msger.info("zypp architecture is <%s>" % zconfig.systemArchitecture())
+
+        # repoPackagesCachePath is corrected by this
+        self.repo_manager = zypp.RepoManager(self.repo_manager_options)
+        repos = self.repo_manager.knownRepositories()
+        for repo in repos:
+            if not repo.enabled():
+                continue
+            self.repo_manager.loadFromCache(repo)
+
+        self.Z = zypp.ZYppFactory_instance().getZYpp()
+        self.Z.initializeTarget(zypp.Pathname(self.instroot))
+        self.Z.target().load()
+
+    def buildTransaction(self):
+        if not self.Z.resolver().resolvePool():
+            probs = self.Z.resolver().problems()
+
+            for problem in probs:
+                msger.warning("repo problem: %s, %s" \
+                              % (problem.description().decode("utf-8"),
+                                 problem.details().decode("utf-8")))
+
+            raise RepoError("found %d resolver problem, abort!" \
+                            % len(probs))
+
+    def getLocalPkgPath(self, po):
+        repoinfo = po.repoInfo()
+        cacheroot = repoinfo.packagesPath()
+        location= po.location()
+        rpmpath = str(location.filename())
+        pkgpath = "%s/%s" % (cacheroot, os.path.basename(rpmpath))
+        return pkgpath
+
+    def installLocal(self, pkg, po=None, updateonly=False):
+        if not self.ts:
+            self.__initialize_transaction()
+
+        solvfile = "%s/.solv" % (self.cachedir)
+
+        rc, out = runner.runtool([fs_related.find_binary_path("rpms2solv"),
+                                  pkg])
+        if rc == 0:
+            f = open(solvfile, "w+")
+            f.write(out)
+            f.close()
+
+            warnmsg = self.repo_manager.loadSolvFile(solvfile,
+                                                     os.path.basename(pkg))
+            if warnmsg:
+                msger.warning(warnmsg)
+
+            os.unlink(solvfile)
+        else:
+            msger.warning('Can not get %s solv data.' % pkg)
+
+        hdr = rpmmisc.readRpmHeader(self.ts, pkg)
+        arch = zypp.Arch(hdr['arch'])
+        sysarch = zypp.Arch(self.target_arch)
+
+        if arch.compatible_with (sysarch):
+            pkgname = hdr['name']
+            self.localpkgs[pkgname] = pkg
+            self.selectPackage(pkgname)
+            msger.info("Marking %s to be installed" % (pkg))
+
+        else:
+            msger.warning("Cannot add package %s to transaction. "
+                          "Not a compatible architecture: %s" \
+                          % (pkg, hdr['arch']))
+
+    def downloadPkgs(self, package_objects, count):
+        localpkgs = self.localpkgs.keys()
+        progress_obj = TextProgress(count)
+
+        for po in package_objects:
+            if po.name() in localpkgs:
+                continue
+
+            filename = self.getLocalPkgPath(po)
+            if os.path.exists(filename):
+                if self.checkPkg(filename) == 0:
+                    continue
+
+            dirn = os.path.dirname(filename)
+            if not os.path.exists(dirn):
+                os.makedirs(dirn)
+
+            url = self.get_url(po)
+            proxies = self.get_proxies(po)
+
+            try:
+                filename = myurlgrab(url, filename, proxies, progress_obj)
+            except CreatorError:
+                self.close()
+                raise
+
+    def preinstallPkgs(self):
+        if not self.ts_pre:
+            self.__initialize_transaction()
+
+        self.ts_pre.order()
+        cb = rpmmisc.RPMInstallCallback(self.ts_pre)
+        cb.headmsg = "Preinstall"
+        installlogfile = "%s/__catched_stderr.buf" % (self.instroot)
+
+        # start to catch stderr output from librpm
+        msger.enable_logstderr(installlogfile)
+
+        errors = self.ts_pre.run(cb.callback, '')
+        # stop catch
+        msger.disable_logstderr()
+        self.ts_pre.closeDB()
+        self.ts_pre = None
+
+        if errors is not None:
+            if len(errors) == 0:
+                msger.warning('scriptlet or other non-fatal errors occurred '
+                              'during transaction.')
+
+            else:
+                for e in errors:
+                    msger.warning(e[0])
+                raise RepoError('Could not run transaction.')
+
+    def installPkgs(self, package_objects):
+        if not self.ts:
+            self.__initialize_transaction()
+
+        # clean rpm lock
+        self._cleanupRpmdbLocks(self.instroot)
+        self._cleanupZyppJunk(self.instroot)
+        # Set filters
+        probfilter = 0
+        for flag in self.probFilterFlags:
+            probfilter |= flag
+        self.ts.setProbFilter(probfilter)
+        self.ts_pre.setProbFilter(probfilter)
+
+        localpkgs = self.localpkgs.keys()
+
+        for po in package_objects:
+            pkgname = po.name()
+            if pkgname in localpkgs:
+                rpmpath = self.localpkgs[pkgname]
+            else:
+                rpmpath = self.getLocalPkgPath(po)
+
+            if not os.path.exists(rpmpath):
+                # Maybe it is a local repo
+                rpmuri = self.get_url(po)
+                if rpmuri.startswith("file:/"):
+                    rpmpath = rpmuri[5:]
+
+            if not os.path.exists(rpmpath):
+                raise RpmError("Error: %s doesn't exist" % rpmpath)
+
+            h = rpmmisc.readRpmHeader(self.ts, rpmpath)
+
+            if pkgname in self.pre_pkgs:
+                msger.verbose("pre-install package added: %s" % pkgname)
+                self.ts_pre.addInstall(h, rpmpath, 'u')
+
+            self.ts.addInstall(h, rpmpath, 'u')
+
+        unresolved_dependencies = self.ts.check()
+        if not unresolved_dependencies:
+            if self.pre_pkgs:
+                self.preinstallPkgs()
+
+            self.ts.order()
+            cb = rpmmisc.RPMInstallCallback(self.ts)
+            installlogfile = "%s/__catched_stderr.buf" % (self.instroot)
+
+            # start to catch stderr output from librpm
+            msger.enable_logstderr(installlogfile)
+
+            errors = self.ts.run(cb.callback, '')
+            # stop catch
+            msger.disable_logstderr()
+            self.ts.closeDB()
+            self.ts = None
+
+            if errors is not None:
+                if len(errors) == 0:
+                    msger.warning('scriptlet or other non-fatal errors occurred '
+                                  'during transaction.')
+
+                else:
+                    for e in errors:
+                        msger.warning(e[0])
+                    raise RepoError('Could not run transaction.')
+
+        else:
+            for pkg, need, needflags, sense, key in unresolved_dependencies:
+                package = '-'.join(pkg)
+
+                if needflags == rpm.RPMSENSE_LESS:
+                    deppkg = ' < '.join(need)
+                elif needflags == rpm.RPMSENSE_EQUAL:
+                    deppkg = ' = '.join(need)
+                elif needflags == rpm.RPMSENSE_GREATER:
+                    deppkg = ' > '.join(need)
+                else:
+                    deppkg = '-'.join(need)
+
+                if sense == rpm.RPMDEP_SENSE_REQUIRES:
+                    msger.warning("[%s] Requires [%s], which is not provided" \
+                                  % (package, deppkg))
+
+                elif sense == rpm.RPMDEP_SENSE_CONFLICTS:
+                    msger.warning("[%s] Conflicts with [%s]" %(package,deppkg))
+
+            raise RepoError("Unresolved dependencies, transaction failed.")
+
+    def __initialize_transaction(self):
+        if not self.ts:
+            self.ts = rpm.TransactionSet(self.instroot)
+            # Set to not verify DSA signatures.
+            self.ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES|rpm._RPMVSF_NODIGESTS)
+
+        if not self.ts_pre:
+            self.ts_pre = rpm.TransactionSet(self.instroot)
+            # Just unpack the files, don't run scripts
+            self.ts_pre.setFlags(rpm.RPMTRANS_FLAG_ALLFILES | rpm.RPMTRANS_FLAG_NOSCRIPTS)
+            # Set to not verify DSA signatures.
+            self.ts_pre.setVSFlags(rpm._RPMVSF_NOSIGNATURES|rpm._RPMVSF_NODIGESTS)
+
+    def checkPkg(self, pkg):
+        ret = 1
+        if not os.path.exists(pkg):
+            return ret
+        ret = rpmmisc.checkRpmIntegrity('rpm', pkg)
+        if ret != 0:
+            msger.warning("package %s is damaged: %s" \
+                          % (os.path.basename(pkg), pkg))
+
+        return ret
+
+    def _add_prob_flags(self, *flags):
+        for flag in flags:
+            if flag not in self.probFilterFlags:
+                self.probFilterFlags.append(flag)
+
+    def get_proxies(self, pobj):
+        if not pobj:
+            return None
+
+        proxy = None
+        proxies = None
+        repoinfo = pobj.repoInfo()
+        reponame = "%s" % repoinfo.name()
+        repos = filter(lambda r: r.name == reponame, self.repos)
+        repourl = str(repoinfo.baseUrls()[0])
+
+        if repos:
+            proxy = repos[0].proxy
+        if not proxy:
+            proxy = get_proxy_for(repourl)
+        if proxy:
+            proxies = {str(repourl.split(':')[0]): str(proxy)}
+
+        return proxies
+
+    def get_url(self, pobj):
+        if not pobj:
+            return None
+
+        name = str(pobj.repoInfo().name())
+        try:
+            repo = filter(lambda r: r.name == name, self.repos)[0]
+        except IndexError:
+            return None
+
+        baseurl = repo.baseurl[0]
+
+        index = baseurl.find("?")
+        if index > -1:
+            baseurl = baseurl[:index]
+
+        location = pobj.location()
+        location = str(location.filename())
+        if location.startswith("./"):
+            location = location[2:]
+
+        return os.path.join(baseurl, location)
+
+    def package_url(self, pkgname):
+
+        def cmpEVR(p1, p2):
+            ed1 = p1.edition()
+            ed2 = p2.edition()
+            (e1, v1, r1) = map(str, [ed1.epoch(), ed1.version(), ed1.release()])
+            (e2, v2, r2) = map(str, [ed2.epoch(), ed2.version(), ed2.release()])
+            return rpm.labelCompare((e1, v1, r1), (e2, v2, r2))
+
+        if not self.Z:
+            self.__initialize_zypp()
+
+        q = zypp.PoolQuery()
+        q.addKind(zypp.ResKind.package)
+        q.setMatchExact()
+        q.addAttribute(zypp.SolvAttr.name, pkgname)
+        items = sorted(q.queryResults(self.Z.pool()),
+                       cmp=lambda x,y: cmpEVR(zypp.asKindPackage(x), zypp.asKindPackage(y)),
+                       reverse=True)
+
+        if items:
+            item = zypp.asKindPackage(items[0])
+            url = self.get_url(item)
+            proxies = self.get_proxies(item)
+            return (url, proxies)
+
+        return (None, None)

scripts/lib/mic/plugins/hook/empty_hook.py

@@ -0,0 +1,3 @@
+#!/usr/bin/python
+
+# TODO: plugin base for hooks

scripts/lib/mic/plugins/imager/fs_plugin.py

@@ -0,0 +1,143 @@
+#!/usr/bin/python -tt
+#
+# Copyright (c) 2011 Intel, Inc.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; version 2 of the License
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc., 59
+# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+import os
+import sys
+
+from mic import chroot, msger, rt_util
+from mic.utils import cmdln, misc, errors, fs_related
+from mic.imager import fs
+from mic.conf import configmgr
+from mic.plugin import pluginmgr
+
+from mic.pluginbase import ImagerPlugin
+class FsPlugin(ImagerPlugin):
+    name = 'fs'
+
+    @classmethod
+    @cmdln.option("--include-src",
+                  dest="include_src",
+                  action="store_true",
+                  default=False,
+                  help="Generate a image with source rpms included")
+    def do_create(self, subcmd, opts, *args):
+        """${cmd_name}: create fs image
+
+        Usage:
+            ${name} ${cmd_name} <ksfile> [OPTS]
+
+        ${cmd_option_list}
+        """
+
+        if len(args) != 1:
+            raise errors.Usage("Extra arguments given")
+
+        creatoropts = configmgr.create
+        ksconf = args[0]
+
+        if creatoropts['runtime'] == 'bootstrap':
+            configmgr._ksconf = ksconf
+            rt_util.bootstrap_mic()
+
+        recording_pkgs = []
+        if len(creatoropts['record_pkgs']) > 0:
+            recording_pkgs = creatoropts['record_pkgs']
+
+        if creatoropts['release'] is not None:
+            if 'name' not in recording_pkgs:
+                recording_pkgs.append('name')
+            if 'vcs' not in recording_pkgs:
+                recording_pkgs.append('vcs')
+
+        configmgr._ksconf = ksconf
+
+        # Called After setting the configmgr._ksconf as the creatoropts['name'] is reset there.
+        if creatoropts['release'] is not None:
+            creatoropts['outdir'] = "%s/%s/images/%s/" % (creatoropts['outdir'], creatoropts['release'], creatoropts['name'])
+
+        # try to find the pkgmgr
+        pkgmgr = None
+        backends = pluginmgr.get_plugins('backend')
+        if 'auto' == creatoropts['pkgmgr']:
+            for key in configmgr.prefer_backends:
+                if key in backends:
+                    pkgmgr = backends[key]
+                    break
+        else:
+            for key in backends.keys():
+                if key == creatoropts['pkgmgr']:
+                    pkgmgr = backends[key]
+                    break
+
+        if not pkgmgr:
+            raise errors.CreatorError("Can't find backend: %s, "
+                                      "available choices: %s" %
+                                      (creatoropts['pkgmgr'],
+                                       ','.join(backends.keys())))
+
+        creator = fs.FsImageCreator(creatoropts, pkgmgr)
+        creator._include_src = opts.include_src
+
+        if len(recording_pkgs) > 0:
+            creator._recording_pkgs = recording_pkgs
+
+        self.check_image_exists(creator.destdir,
+                                creator.pack_to,
+                                [creator.name],
+                                creatoropts['release'])
+
+        try:
+            creator.check_depend_tools()
+            creator.mount(None, creatoropts["cachedir"])
+            creator.install()
+            #Download the source packages ###private options
+            if opts.include_src:
+                installed_pkgs =  creator.get_installed_packages()
+                msger.info('--------------------------------------------------')
+                msger.info('Generating the image with source rpms included ...')
+                if not misc.SrcpkgsDownload(installed_pkgs, creatoropts["repomd"], creator._instroot, creatoropts["cachedir"]):
+                    msger.warning("Source packages can't be downloaded")
+
+            creator.configure(creatoropts["repomd"])
+            creator.copy_kernel()
+            creator.unmount()
+            creator.package(creatoropts["outdir"])
+            if creatoropts['release'] is not None:
+                creator.release_output(ksconf, creatoropts['outdir'], creatoropts['release'])
+            creator.print_outimage_info()
+        except errors.CreatorError:
+            raise
+        finally:
+            creator.cleanup()
+
+        msger.info("Finished.")
+        return 0
+
+    @classmethod
+    def do_chroot(self, target, cmd=[]):#chroot.py parse opts&args
+        try:
+            if len(cmd) != 0:
+                cmdline = ' '.join(cmd)
+            else:
+                cmdline = "/bin/bash"
+            envcmd = fs_related.find_binary_inchroot("env", target)
+            if envcmd:
+                cmdline = "%s HOME=/root %s" % (envcmd, cmdline)
+            chroot.chroot(target, None, cmdline)
+        finally:
+            chroot.cleanup_after_chroot("dir", None, None, None)
+            return 1

scripts/lib/mic/plugins/imager/livecd_plugin.py

@@ -0,0 +1,255 @@
+#!/usr/bin/python -tt
+#
+# Copyright (c) 2011 Intel, Inc.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; version 2 of the License
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc., 59
+# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+import os
+import shutil
+import tempfile
+
+from mic import chroot, msger, rt_util
+from mic.utils import misc, fs_related, errors
+from mic.conf import configmgr
+import mic.imager.livecd as livecd
+from mic.plugin import pluginmgr
+
+from mic.pluginbase import ImagerPlugin
+class LiveCDPlugin(ImagerPlugin):
+    name = 'livecd'
+
+    @classmethod
+    def do_create(self, subcmd, opts, *args):
+        """${cmd_name}: create livecd image
+
+        Usage:
+            ${name} ${cmd_name} <ksfile> [OPTS]
+
+        ${cmd_option_list}
+        """
+
+        if len(args) != 1:
+            raise errors.Usage("Extra arguments given")
+
+        creatoropts = configmgr.create
+        ksconf = args[0]
+
+        if creatoropts['runtime'] == 'bootstrap':
+            configmgr._ksconf = ksconf
+            rt_util.bootstrap_mic()
+
+        if creatoropts['arch'] and creatoropts['arch'].startswith('arm'):
+            msger.warning('livecd cannot support arm images, Quit')
+            return
+
+        recording_pkgs = []
+        if len(creatoropts['record_pkgs']) > 0:
+            recording_pkgs = creatoropts['record_pkgs']
+
+        if creatoropts['release'] is not None:
+            if 'name' not in recording_pkgs:
+                recording_pkgs.append('name')
+            if 'vcs' not in recording_pkgs:
+                recording_pkgs.append('vcs')
+
+        configmgr._ksconf = ksconf
+
+        # Called After setting the configmgr._ksconf as the creatoropts['name'] is reset there.
+        if creatoropts['release'] is not None:
+            creatoropts['outdir'] = "%s/%s/images/%s/" % (creatoropts['outdir'], creatoropts['release'], creatoropts['name'])
+
+        # try to find the pkgmgr
+        pkgmgr = None
+        backends = pluginmgr.get_plugins('backend')
+        if 'auto' == creatoropts['pkgmgr']:
+            for key in configmgr.prefer_backends:
+                if key in backends:
+                    pkgmgr = backends[key]
+                    break
+        else:
+            for key in backends.keys():
+                if key == creatoropts['pkgmgr']:
+                    pkgmgr = backends[key]
+                    break
+
+        if not pkgmgr:
+            raise errors.CreatorError("Can't find backend: %s, "
+                                      "available choices: %s" %
+                                      (creatoropts['pkgmgr'],
+                                       ','.join(backends.keys())))
+
+        creator = livecd.LiveCDImageCreator(creatoropts, pkgmgr)
+
+        if len(recording_pkgs) > 0:
+            creator._recording_pkgs = recording_pkgs
+
+        self.check_image_exists(creator.destdir,
+                                creator.pack_to,
+                                [creator.name + ".iso"],
+                                creatoropts['release'])
+
+        try:
+            creator.check_depend_tools()
+            creator.mount(None, creatoropts["cachedir"])
+            creator.install()
+            creator.configure(creatoropts["repomd"])
+            creator.copy_kernel()
+            creator.unmount()
+            creator.package(creatoropts["outdir"])
+            if creatoropts['release'] is not None:
+                creator.release_output(ksconf, creatoropts['outdir'], creatoropts['release'])
+            creator.print_outimage_info()
+
+        except errors.CreatorError:
+            raise
+        finally:
+            creator.cleanup()
+
+        msger.info("Finished.")
+        return 0
+
+    @classmethod
+    def do_chroot(cls, target, cmd=[]):
+        os_image = cls.do_unpack(target)
+        os_image_dir = os.path.dirname(os_image)
+
+        # unpack image to target dir
+        imgsize = misc.get_file_size(os_image) * 1024L * 1024L
+        imgtype = misc.get_image_type(os_image)
+        if imgtype == "btrfsimg":
+            fstype = "btrfs"
+            myDiskMount = fs_related.BtrfsDiskMount
+        elif imgtype in ("ext3fsimg", "ext4fsimg"):
+            fstype = imgtype[:4]
+            myDiskMount = fs_related.ExtDiskMount
+        else:
+            raise errors.CreatorError("Unsupported filesystem type: %s" % fstype)
+
+        extmnt = misc.mkdtemp()
+        extloop = myDiskMount(fs_related.SparseLoopbackDisk(os_image, imgsize),
+                              extmnt,
+                              fstype,
+                              4096,
+                              "%s label" % fstype)
+        try:
+            extloop.mount()
+
+        except errors.MountError:
+            extloop.cleanup()
+            shutil.rmtree(extmnt, ignore_errors = True)
+            shutil.rmtree(os_image_dir, ignore_errors = True)
+            raise
+
+        try:
+            if len(cmd) != 0:
+                cmdline = ' '.join(cmd)
+            else:
+                cmdline = "/bin/bash"
+            envcmd = fs_related.find_binary_inchroot("env", extmnt)
+            if envcmd:
+                cmdline = "%s HOME=/root %s" % (envcmd, cmdline)
+            chroot.chroot(extmnt, None, cmdline)
+        except:
+            raise errors.CreatorError("Failed to chroot to %s." %target)
+        finally:
+            chroot.cleanup_after_chroot("img", extloop, os_image_dir, extmnt)
+
+    @classmethod
+    def do_pack(cls, base_on):
+        import subprocess
+
+        def __mkinitrd(instance):
+            kernelver = instance._get_kernel_versions().values()[0][0]
+            args = [ "/usr/libexec/mkliveinitrd", "/boot/initrd-%s.img" % kernelver, "%s" % kernelver ]
+            try:
+                subprocess.call(args, preexec_fn = instance._chroot)
+            except OSError, (err, msg):
+                raise errors.CreatorError("Failed to execute /usr/libexec/mkliveinitrd: %s" % msg)
+
+        def __run_post_cleanups(instance):
+            kernelver = instance._get_kernel_versions().values()[0][0]
+            args = ["rm", "-f", "/boot/initrd-%s.img" % kernelver]
+
+            try:
+                subprocess.call(args, preexec_fn = instance._chroot)
+            except OSError, (err, msg):
+                raise errors.CreatorError("Failed to run post cleanups: %s" % msg)
+
+        convertoropts = configmgr.convert
+        convertoropts['name'] = os.path.splitext(os.path.basename(base_on))[0]
+        convertor = livecd.LiveCDImageCreator(convertoropts)
+        imgtype = misc.get_image_type(base_on)
+        if imgtype == "btrfsimg":
+            fstype = "btrfs"
+        elif imgtype in ("ext3fsimg", "ext4fsimg"):
+            fstype = imgtype[:4]
+        else:
+            raise errors.CreatorError("Unsupported filesystem type: %s" % fstype)
+        convertor._set_fstype(fstype)
+        try:
+            convertor.mount(base_on)
+            __mkinitrd(convertor)
+            convertor._create_bootconfig()
+            __run_post_cleanups(convertor)
+            convertor.launch_shell(convertoropts['shell'])
+            convertor.unmount()
+            convertor.package()
+            convertor.print_outimage_info()
+        finally:
+            shutil.rmtree(os.path.dirname(base_on), ignore_errors = True)
+
+    @classmethod
+    def do_unpack(cls, srcimg):
+        img = srcimg
+        imgmnt = misc.mkdtemp()
+        imgloop = fs_related.DiskMount(fs_related.LoopbackDisk(img, 0), imgmnt)
+        try:
+            imgloop.mount()
+        except errors.MountError:
+            imgloop.cleanup()
+            raise
+
+        # legacy LiveOS filesystem layout support, remove for F9 or F10
+        if os.path.exists(imgmnt + "/squashfs.img"):
+            squashimg = imgmnt + "/squashfs.img"
+        else:
+            squashimg = imgmnt + "/LiveOS/squashfs.img"
+
+        tmpoutdir = misc.mkdtemp()
+        # unsquashfs requires outdir mustn't exist
+        shutil.rmtree(tmpoutdir, ignore_errors = True)
+        misc.uncompress_squashfs(squashimg, tmpoutdir)
+
+        try:
+            # legacy LiveOS filesystem layout support, remove for F9 or F10
+            if os.path.exists(tmpoutdir + "/os.img"):
+                os_image = tmpoutdir + "/os.img"
+            else:
+                os_image = tmpoutdir + "/LiveOS/ext3fs.img"
+
+            if not os.path.exists(os_image):
+                raise errors.CreatorError("'%s' is not a valid live CD ISO : neither "
+                                          "LiveOS/ext3fs.img nor os.img exist" %img)
+
+            imgname = os.path.basename(srcimg)
+            imgname = os.path.splitext(imgname)[0] + ".img"
+            rtimage = os.path.join(tempfile.mkdtemp(dir = "/var/tmp", prefix = "tmp"), imgname)
+            shutil.copyfile(os_image, rtimage)
+
+        finally:
+            imgloop.cleanup()
+            shutil.rmtree(tmpoutdir, ignore_errors = True)
+            shutil.rmtree(imgmnt, ignore_errors = True)
+
+        return rtimage

scripts/lib/mic/plugins/imager/liveusb_plugin.py

@@ -0,0 +1,260 @@
+#!/usr/bin/python -tt
+#
+# Copyright (c) 2011 Intel, Inc.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; version 2 of the License
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc., 59
+# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+import os
+import shutil
+import tempfile
+
+from mic import chroot, msger, rt_util
+from mic.utils import misc, fs_related, errors
+from mic.utils.partitionedfs import PartitionedMount
+from mic.conf import configmgr
+from mic.plugin import pluginmgr
+
+import mic.imager.liveusb as liveusb
+
+from mic.pluginbase import ImagerPlugin
+class LiveUSBPlugin(ImagerPlugin):
+    name = 'liveusb'
+
+    @classmethod
+    def do_create(self, subcmd, opts, *args):
+        """${cmd_name}: create liveusb image
+
+        Usage:
+            ${name} ${cmd_name} <ksfile> [OPTS]
+
+        ${cmd_option_list}
+        """
+
+        if len(args) != 1:
+            raise errors.Usage("Extra arguments given")
+
+        creatoropts = configmgr.create
+        ksconf = args[0]
+
+        if creatoropts['runtime'] == "bootstrap":
+            configmgr._ksconf = ksconf
+            rt_util.bootstrap_mic()
+
+        if creatoropts['arch'] and creatoropts['arch'].startswith('arm'):
+            msger.warning('liveusb cannot support arm images, Quit')
+            return
+
+        recording_pkgs = []
+        if len(creatoropts['record_pkgs']) > 0:
+            recording_pkgs = creatoropts['record_pkgs']
+
+        if creatoropts['release'] is not None:
+            if 'name' not in recording_pkgs:
+                recording_pkgs.append('name')
+            if 'vcs' not in recording_pkgs:
+                recording_pkgs.append('vcs')
+
+        configmgr._ksconf = ksconf
+
+        # Called After setting the configmgr._ksconf as the creatoropts['name'] is reset there.
+        if creatoropts['release'] is not None:
+            creatoropts['outdir'] = "%s/%s/images/%s/" % (creatoropts['outdir'], creatoropts['release'], creatoropts['name'])
+
+        # try to find the pkgmgr
+        pkgmgr = None
+        backends = pluginmgr.get_plugins('backend')
+        if 'auto' == creatoropts['pkgmgr']:
+            for key in configmgr.prefer_backends:
+                if key in backends:
+                    pkgmgr = backends[key]
+                    break
+        else:
+            for key in backends.keys():
+                if key == creatoropts['pkgmgr']:
+                    pkgmgr = backends[key]
+                    break
+
+        if not pkgmgr:
+            raise errors.CreatorError("Can't find backend: %s, "
+                                      "available choices: %s" %
+                                      (creatoropts['pkgmgr'],
+                                       ','.join(backends.keys())))
+
+        creator = liveusb.LiveUSBImageCreator(creatoropts, pkgmgr)
+
+        if len(recording_pkgs) > 0:
+            creator._recording_pkgs = recording_pkgs
+
+        self.check_image_exists(creator.destdir,
+                                creator.pack_to,
+                                [creator.name + ".usbimg"],
+                                creatoropts['release'])
+        try:
+            creator.check_depend_tools()
+            creator.mount(None, creatoropts["cachedir"])
+            creator.install()
+            creator.configure(creatoropts["repomd"])
+            creator.copy_kernel()
+            creator.unmount()
+            creator.package(creatoropts["outdir"])
+            if creatoropts['release'] is not None:
+                creator.release_output(ksconf, creatoropts['outdir'], creatoropts['release'])
+            creator.print_outimage_info()
+
+        except errors.CreatorError:
+            raise
+        finally:
+            creator.cleanup()
+
+        msger.info("Finished.")
+        return 0
+
+    @classmethod
+    def do_chroot(cls, target, cmd=[]):
+        os_image = cls.do_unpack(target)
+        os_image_dir = os.path.dirname(os_image)
+
+        # unpack image to target dir
+        imgsize = misc.get_file_size(os_image) * 1024L * 1024L
+        imgtype = misc.get_image_type(os_image)
+        if imgtype == "btrfsimg":
+            fstype = "btrfs"
+            myDiskMount = fs_related.BtrfsDiskMount
+        elif imgtype in ("ext3fsimg", "ext4fsimg"):
+            fstype = imgtype[:4]
+            myDiskMount = fs_related.ExtDiskMount
+        else:
+            raise errors.CreatorError("Unsupported filesystem type: %s" % fstype)
+
+        extmnt = misc.mkdtemp()
+        extloop = myDiskMount(fs_related.SparseLoopbackDisk(os_image, imgsize),
+                              extmnt,
+                              fstype,
+                              4096,
+                              "%s label" % fstype)
+
+        try:
+            extloop.mount()
+
+        except errors.MountError:
+            extloop.cleanup()
+            shutil.rmtree(extmnt, ignore_errors = True)
+            raise
+
+        try:
+            if len(cmd) != 0:
+                cmdline = ' '.join(cmd)
+            else:
+                cmdline = "/bin/bash"
+            envcmd = fs_related.find_binary_inchroot("env", extmnt)
+            if envcmd:
+                cmdline = "%s HOME=/root %s" % (envcmd, cmdline)
+            chroot.chroot(extmnt, None, cmdline)
+        except:
+            raise errors.CreatorError("Failed to chroot to %s." %target)
+        finally:
+            chroot.cleanup_after_chroot("img", extloop, os_image_dir, extmnt)
+
+    @classmethod
+    def do_pack(cls, base_on):
+        import subprocess
+
+        def __mkinitrd(instance):
+            kernelver = instance._get_kernel_versions().values()[0][0]
+            args = [ "/usr/libexec/mkliveinitrd", "/boot/initrd-%s.img" % kernelver, "%s" % kernelver ]
+            try:
+                subprocess.call(args, preexec_fn = instance._chroot)
+
+            except OSError, (err, msg):
+                raise errors.CreatorError("Failed to execute /usr/libexec/mkliveinitrd: %s" % msg)
+
+        def __run_post_cleanups(instance):
+            kernelver = instance._get_kernel_versions().values()[0][0]
+            args = ["rm", "-f", "/boot/initrd-%s.img" % kernelver]
+
+            try:
+                subprocess.call(args, preexec_fn = instance._chroot)
+            except OSError, (err, msg):
+                raise errors.CreatorError("Failed to run post cleanups: %s" % msg)
+
+        convertoropts = configmgr.convert
+        convertoropts['name'] = os.path.splitext(os.path.basename(base_on))[0]
+        convertor = liveusb.LiveUSBImageCreator(convertoropts)
+        imgtype = misc.get_image_type(base_on)
+        if imgtype == "btrfsimg":
+            fstype = "btrfs"
+        elif imgtype in ("ext3fsimg", "ext4fsimg"):
+            fstype = imgtype[:4]
+        else:
+            raise errors.CreatorError("Unsupported filesystem type: %s" % fstyp)
+        convertor._set_fstype(fstype)
+        try:
+            convertor.mount(base_on)
+            __mkinitrd(convertor)
+            convertor._create_bootconfig()
+            __run_post_cleanups(convertor)
+            convertor.launch_shell(convertoropts['shell'])
+            convertor.unmount()
+            convertor.package()
+            convertor.print_outimage_info()
+        finally:
+            shutil.rmtree(os.path.dirname(base_on), ignore_errors = True)
+
+    @classmethod
+    def do_unpack(cls, srcimg):
+        img = srcimg
+        imgsize = misc.get_file_size(img) * 1024L * 1024L
+        imgmnt = misc.mkdtemp()
+        disk = fs_related.SparseLoopbackDisk(img, imgsize)
+        imgloop = PartitionedMount(imgmnt, skipformat = True)
+        imgloop.add_disk('/dev/sdb', disk)
+        imgloop.add_partition(imgsize/1024/1024, "/dev/sdb", "/", "vfat", boot=False)
+        try:
+            imgloop.mount()
+        except errors.MountError:
+            imgloop.cleanup()
+            raise
+
+        # legacy LiveOS filesystem layout support, remove for F9 or F10
+        if os.path.exists(imgmnt + "/squashfs.img"):
+            squashimg = imgmnt + "/squashfs.img"
+        else:
+            squashimg = imgmnt + "/LiveOS/squashfs.img"
+
+        tmpoutdir = misc.mkdtemp()
+        # unsquashfs requires outdir mustn't exist
+        shutil.rmtree(tmpoutdir, ignore_errors = True)
+        misc.uncompress_squashfs(squashimg, tmpoutdir)
+
+        try:
+            # legacy LiveOS filesystem layout support, remove for F9 or F10
+            if os.path.exists(tmpoutdir + "/os.img"):
+                os_image = tmpoutdir + "/os.img"
+            else:
+                os_image = tmpoutdir + "/LiveOS/ext3fs.img"
+
+            if not os.path.exists(os_image):
+                raise errors.CreatorError("'%s' is not a valid live CD ISO : neither "
+                                          "LiveOS/ext3fs.img nor os.img exist" %img)
+            imgname = os.path.basename(srcimg)
+            imgname = os.path.splitext(imgname)[0] + ".img"
+            rtimage = os.path.join(tempfile.mkdtemp(dir = "/var/tmp", prefix = "tmp"), imgname)
+            shutil.copyfile(os_image, rtimage)
+
+        finally:
+            imgloop.cleanup()
+            shutil.rmtree(tmpoutdir, ignore_errors = True)
+            shutil.rmtree(imgmnt, ignore_errors = True)
+
+        return rtimage

scripts/lib/mic/plugins/imager/loop_plugin.py

@@ -0,0 +1,255 @@
+#!/usr/bin/python -tt
+#
+# Copyright (c) 2011 Intel, Inc.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; version 2 of the License
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc., 59
+# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+import os
+import shutil
+import tempfile
+
+from mic import chroot, msger, rt_util
+from mic.utils import misc, fs_related, errors, cmdln
+from mic.conf import configmgr
+from mic.plugin import pluginmgr
+from mic.imager.loop import LoopImageCreator, load_mountpoints
+
+from mic.pluginbase import ImagerPlugin
+class LoopPlugin(ImagerPlugin):
+    name = 'loop'
+
+    @classmethod
+    @cmdln.option("--compress-disk-image", dest="compress_image",
+                  type='choice', choices=("gz", "bz2"), default=None,
+                  help="Same with --compress-image")
+                  # alias to compress-image for compatibility
+    @cmdln.option("--compress-image", dest="compress_image",
+                  type='choice', choices=("gz", "bz2"), default=None,
+                  help="Compress all loop images with 'gz' or 'bz2'")
+    @cmdln.option("--shrink", action='store_true', default=False,
+                  help="Whether to shrink loop images to minimal size")
+    def do_create(self, subcmd, opts, *args):
+        """${cmd_name}: create loop image
+
+        Usage:
+            ${name} ${cmd_name} <ksfile> [OPTS]
+
+        ${cmd_option_list}
+        """
+
+        if len(args) != 1:
+            raise errors.Usage("Extra arguments given")
+
+        creatoropts = configmgr.create
+        ksconf = args[0]
+
+        if creatoropts['runtime'] == "bootstrap":
+            configmgr._ksconf = ksconf
+            rt_util.bootstrap_mic()
+
+        recording_pkgs = []
+        if len(creatoropts['record_pkgs']) > 0:
+            recording_pkgs = creatoropts['record_pkgs']
+
+        if creatoropts['release'] is not None:
+            if 'name' not in recording_pkgs:
+                recording_pkgs.append('name')
+            if 'vcs' not in recording_pkgs:
+                recording_pkgs.append('vcs')
+
+        configmgr._ksconf = ksconf
+
+        # Called After setting the configmgr._ksconf
+        # as the creatoropts['name'] is reset there.
+        if creatoropts['release'] is not None:
+            creatoropts['outdir'] = "%s/%s/images/%s/" % (creatoropts['outdir'],
+                                                          creatoropts['release'],
+                                                          creatoropts['name'])
+        # try to find the pkgmgr
+        pkgmgr = None
+        backends = pluginmgr.get_plugins('backend')
+        if 'auto' == creatoropts['pkgmgr']:
+            for key in configmgr.prefer_backends:
+                if key in backends:
+                    pkgmgr = backends[key]
+                    break
+        else:
+            for key in backends.keys():
+                if key == creatoropts['pkgmgr']:
+                    pkgmgr = backends[key]
+                    break
+
+        if not pkgmgr:
+            raise errors.CreatorError("Can't find backend: %s, "
+                                      "available choices: %s" %
+                                      (creatoropts['pkgmgr'],
+                                       ','.join(backends.keys())))
+
+        creator = LoopImageCreator(creatoropts,
+                                   pkgmgr,
+                                   opts.compress_image,
+                                   opts.shrink)
+
+        if len(recording_pkgs) > 0:
+            creator._recording_pkgs = recording_pkgs
+
+        image_names = [creator.name + ".img"]
+        image_names.extend(creator.get_image_names())
+        self.check_image_exists(creator.destdir,
+                                creator.pack_to,
+                                image_names,
+                                creatoropts['release'])
+
+        try:
+            creator.check_depend_tools()
+            creator.mount(None, creatoropts["cachedir"])
+            creator.install()
+            creator.configure(creatoropts["repomd"])
+            creator.copy_kernel()
+            creator.unmount()
+            creator.package(creatoropts["outdir"])
+
+            if creatoropts['release'] is not None:
+                creator.release_output(ksconf,
+                                       creatoropts['outdir'],
+                                       creatoropts['release'])
+            creator.print_outimage_info()
+
+        except errors.CreatorError:
+            raise
+        finally:
+            creator.cleanup()
+
+        msger.info("Finished.")
+        return 0
+
+    @classmethod
+    def _do_chroot_tar(cls, target, cmd=[]):
+        mountfp_xml = os.path.splitext(target)[0] + '.xml'
+        if not os.path.exists(mountfp_xml):
+            raise errors.CreatorError("No mount point file found for this tar "
+                                      "image, please check %s" % mountfp_xml)
+
+        import tarfile
+        tar = tarfile.open(target, 'r')
+        tmpdir = misc.mkdtemp()
+        tar.extractall(path=tmpdir)
+        tar.close()
+
+        mntdir = misc.mkdtemp()
+
+        loops = []
+        for (mp, label, name, size, fstype) in load_mountpoints(mountfp_xml):
+            if fstype in ("ext2", "ext3", "ext4"):
+                myDiskMount = fs_related.ExtDiskMount
+            elif fstype == "btrfs":
+                myDiskMount = fs_related.BtrfsDiskMount
+            elif fstype in ("vfat", "msdos"):
+                myDiskMount = fs_related.VfatDiskMount
+            else:
+                msger.error("Cannot support fstype: %s" % fstype)
+
+            name = os.path.join(tmpdir, name)
+            size = size * 1024L * 1024L
+            loop = myDiskMount(fs_related.SparseLoopbackDisk(name, size),
+                               os.path.join(mntdir, mp.lstrip('/')),
+                               fstype, size, label)
+
+            try:
+                msger.verbose("Mount %s to %s" % (mp, mntdir + mp))
+                fs_related.makedirs(os.path.join(mntdir, mp.lstrip('/')))
+                loop.mount()
+
+            except:
+                loop.cleanup()
+                for lp in reversed(loops):
+                    chroot.cleanup_after_chroot("img", lp, None, mntdir)
+
+                shutil.rmtree(tmpdir, ignore_errors=True)
+                raise
+
+            loops.append(loop)
+
+        try:
+            if len(cmd) != 0:
+                cmdline = "/usr/bin/env HOME=/root " + ' '.join(cmd)
+            else:
+                cmdline = "/usr/bin/env HOME=/root /bin/bash"
+            chroot.chroot(mntdir, None, cmdline)
+        except:
+            raise errors.CreatorError("Failed to chroot to %s." % target)
+        finally:
+            for loop in reversed(loops):
+                chroot.cleanup_after_chroot("img", loop, None, mntdir)
+
+            shutil.rmtree(tmpdir, ignore_errors=True)
+
+    @classmethod
+    def do_chroot(cls, target, cmd=[]):
+        if target.endswith('.tar'):
+            import tarfile
+            if tarfile.is_tarfile(target):
+                LoopPlugin._do_chroot_tar(target, cmd)
+                return
+            else:
+                raise errors.CreatorError("damaged tarball for loop images")
+
+        img = target
+        imgsize = misc.get_file_size(img) * 1024L * 1024L
+        imgtype = misc.get_image_type(img)
+        if imgtype == "btrfsimg":
+            fstype = "btrfs"
+            myDiskMount = fs_related.BtrfsDiskMount
+        elif imgtype in ("ext3fsimg", "ext4fsimg"):
+            fstype = imgtype[:4]
+            myDiskMount = fs_related.ExtDiskMount
+        else:
+            raise errors.CreatorError("Unsupported filesystem type: %s" \
+                                      % imgtype)
+
+        extmnt = misc.mkdtemp()
+        extloop = myDiskMount(fs_related.SparseLoopbackDisk(img, imgsize),
+                                                         extmnt,
+                                                         fstype,
+                                                         4096,
+                                                         "%s label" % fstype)
+        try:
+            extloop.mount()
+
+        except errors.MountError:
+            extloop.cleanup()
+            shutil.rmtree(extmnt, ignore_errors=True)
+            raise
+
+        try:
+            if len(cmd) != 0:
+                cmdline = ' '.join(cmd)
+            else:
+                cmdline = "/bin/bash"
+            envcmd = fs_related.find_binary_inchroot("env", extmnt)
+            if envcmd:
+                cmdline = "%s HOME=/root %s" % (envcmd, cmdline)
+            chroot.chroot(extmnt, None, cmdline)
+        except:
+            raise errors.CreatorError("Failed to chroot to %s." % img)
+        finally:
+            chroot.cleanup_after_chroot("img", extloop, None, extmnt)
+
+    @classmethod
+    def do_unpack(cls, srcimg):
+        image = os.path.join(tempfile.mkdtemp(dir="/var/tmp", prefix="tmp"),
+                             "target.img")
+        msger.info("Copying file system ...")
+        shutil.copyfile(srcimg, image)
+        return image

scripts/lib/mic/plugins/imager/raw_plugin.py

@@ -0,0 +1,275 @@
+#!/usr/bin/python -tt
+#
+# Copyright (c) 2011 Intel, Inc.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; version 2 of the License
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc., 59
+# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+import os
+import shutil
+import re
+import tempfile
+
+from mic import chroot, msger, rt_util
+from mic.utils import misc, fs_related, errors, runner, cmdln
+from mic.conf import configmgr
+from mic.plugin import pluginmgr
+from mic.utils.partitionedfs import PartitionedMount
+
+import mic.imager.raw as raw
+
+from mic.pluginbase import ImagerPlugin
+class RawPlugin(ImagerPlugin):
+    name = 'raw'
+
+    @classmethod
+    @cmdln.option("--compress-disk-image", dest="compress_image", type='choice',
+                  choices=("gz", "bz2"), default=None,
+                  help="Same with --compress-image")
+    @cmdln.option("--compress-image", dest="compress_image", type='choice',
+                  choices=("gz", "bz2"), default = None,
+                  help="Compress all raw images before package")
+    @cmdln.option("--generate-bmap", action="store_true", default = None,
+                  help="also generate the block map file")
+    @cmdln.option("--fstab-entry", dest="fstab_entry", type='choice',
+                  choices=("name", "uuid"), default="uuid",
+                  help="Set fstab entry, 'name' means using device names, "
+                       "'uuid' means using filesystem uuid")
+    def do_create(self, subcmd, opts, *args):
+        """${cmd_name}: create raw image
+
+        Usage:
+            ${name} ${cmd_name} <ksfile> [OPTS]
+
+        ${cmd_option_list}
+        """
+
+        if len(args) != 1:
+            raise errors.Usage("Extra arguments given")
+
+        creatoropts = configmgr.create
+        ksconf = args[0]
+
+        if creatoropts['runtime'] == "bootstrap":
+            configmgr._ksconf = ksconf
+            rt_util.bootstrap_mic()
+
+        recording_pkgs = []
+        if len(creatoropts['record_pkgs']) > 0:
+            recording_pkgs = creatoropts['record_pkgs']
+
+        if creatoropts['release'] is not None:
+            if 'name' not in recording_pkgs:
+                recording_pkgs.append('name')
+            if 'vcs' not in recording_pkgs:
+                recording_pkgs.append('vcs')
+
+        configmgr._ksconf = ksconf
+
+        # Called After setting the configmgr._ksconf as the creatoropts['name'] is reset there.
+        if creatoropts['release'] is not None:
+            creatoropts['outdir'] = "%s/%s/images/%s/" % (creatoropts['outdir'], creatoropts['release'], creatoropts['name'])
+
+        # try to find the pkgmgr
+        pkgmgr = None
+        backends = pluginmgr.get_plugins('backend')
+        if 'auto' == creatoropts['pkgmgr']:
+            for key in configmgr.prefer_backends:
+                if key in backends:
+                    pkgmgr = backends[key]
+                    break
+        else:
+            for key in backends.keys():
+                if key == creatoropts['pkgmgr']:
+                    pkgmgr = backends[key]
+                    break
+
+        if not pkgmgr:
+            raise errors.CreatorError("Can't find backend: %s, "
+                                      "available choices: %s" %
+                                      (creatoropts['pkgmgr'],
+                                       ','.join(backends.keys())))
+
+        creator = raw.RawImageCreator(creatoropts, pkgmgr, opts.compress_image,
+                                      opts.generate_bmap, opts.fstab_entry)
+
+        if len(recording_pkgs) > 0:
+            creator._recording_pkgs = recording_pkgs
+
+        images = ["%s-%s.raw" % (creator.name, disk_name)
+                  for disk_name in creator.get_disk_names()]
+        self.check_image_exists(creator.destdir,
+                                creator.pack_to,
+                                images,
+                                creatoropts['release'])
+
+        try:
+            creator.check_depend_tools()
+            creator.mount(None, creatoropts["cachedir"])
+            creator.install()
+            creator.configure(creatoropts["repomd"])
+            creator.copy_kernel()
+            creator.unmount()
+            creator.generate_bmap()
+            creator.package(creatoropts["outdir"])
+            if creatoropts['release'] is not None:
+                creator.release_output(ksconf, creatoropts['outdir'], creatoropts['release'])
+            creator.print_outimage_info()
+
+        except errors.CreatorError:
+            raise
+        finally:
+            creator.cleanup()
+
+        msger.info("Finished.")
+        return 0
+
+    @classmethod
+    def do_chroot(cls, target, cmd=[]):
+        img = target
+        imgsize = misc.get_file_size(img) * 1024L * 1024L
+        partedcmd = fs_related.find_binary_path("parted")
+        disk = fs_related.SparseLoopbackDisk(img, imgsize)
+        imgmnt = misc.mkdtemp()
+        imgloop = PartitionedMount(imgmnt, skipformat = True)
+        imgloop.add_disk('/dev/sdb', disk)
+        img_fstype = "ext3"
+
+        msger.info("Partition Table:")
+        partnum = []
+        for line in runner.outs([partedcmd, "-s", img, "print"]).splitlines():
+            # no use strip to keep line output here
+            if "Number" in line:
+                msger.raw(line)
+            if line.strip() and line.strip()[0].isdigit():
+                partnum.append(line.strip()[0])
+                msger.raw(line)
+
+        rootpart = None
+        if len(partnum) > 1:
+            rootpart = msger.choice("please choose root partition", partnum)
+
+        # Check the partitions from raw disk.
+        # if choose root part, the mark it as mounted
+        if rootpart:
+            root_mounted = True
+        else:
+            root_mounted = False
+        partition_mounts = 0
+        for line in runner.outs([partedcmd,"-s",img,"unit","B","print"]).splitlines():
+            line = line.strip()
+
+            # Lines that start with number are the partitions,
+            # because parted can be translated we can't refer to any text lines.
+            if not line or not line[0].isdigit():
+                continue
+
+            # Some vars have extra , as list seperator.
+            line = line.replace(",","")
+
+            # Example of parted output lines that are handled:
+            # Number  Start        End          Size         Type     File system     Flags
+            #  1      512B         3400000511B  3400000000B  primary
+            #  2      3400531968B  3656384511B  255852544B   primary  linux-swap(v1)
+            #  3      3656384512B  3720347647B  63963136B    primary  fat16           boot, lba
+
+            partition_info = re.split("\s+",line)
+
+            size = partition_info[3].split("B")[0]
+
+            if len(partition_info) < 6 or partition_info[5] in ["boot"]:
+                # No filesystem can be found from partition line. Assuming
+                # btrfs, because that is the only MeeGo fs that parted does
+                # not recognize properly.
+                # TODO: Can we make better assumption?
+                fstype = "btrfs"
+            elif partition_info[5] in ["ext2","ext3","ext4","btrfs"]:
+                fstype = partition_info[5]
+            elif partition_info[5] in ["fat16","fat32"]:
+                fstype = "vfat"
+            elif "swap" in partition_info[5]:
+                fstype = "swap"
+            else:
+                raise errors.CreatorError("Could not recognize partition fs type '%s'." % partition_info[5])
+
+            if rootpart and rootpart == line[0]:
+                mountpoint = '/'
+            elif not root_mounted and fstype in ["ext2","ext3","ext4","btrfs"]:
+                # TODO: Check that this is actually the valid root partition from /etc/fstab
+                mountpoint = "/"
+                root_mounted = True
+            elif fstype == "swap":
+                mountpoint = "swap"
+            else:
+                # TODO: Assing better mount points for the rest of the partitions.
+                partition_mounts += 1
+                mountpoint = "/media/partition_%d" % partition_mounts
+
+            if "boot" in partition_info:
+                boot = True
+            else:
+                boot = False
+
+            msger.verbose("Size: %s Bytes, fstype: %s, mountpoint: %s, boot: %s" % (size, fstype, mountpoint, boot))
+            # TODO: add_partition should take bytes as size parameter.
+            imgloop.add_partition((int)(size)/1024/1024, "/dev/sdb", mountpoint, fstype = fstype, boot = boot)
+
+        try:
+            imgloop.mount()
+
+        except errors.MountError:
+            imgloop.cleanup()
+            raise
+
+        try:
+            if len(cmd) != 0:
+                cmdline = ' '.join(cmd)
+            else:
+                cmdline = "/bin/bash"
+            envcmd = fs_related.find_binary_inchroot("env", imgmnt)
+            if envcmd:
+                cmdline = "%s HOME=/root %s" % (envcmd, cmdline)
+            chroot.chroot(imgmnt, None, cmdline)
+        except:
+            raise errors.CreatorError("Failed to chroot to %s." %img)
+        finally:
+            chroot.cleanup_after_chroot("img", imgloop, None, imgmnt)
+
+    @classmethod
+    def do_unpack(cls, srcimg):
+        srcimgsize = (misc.get_file_size(srcimg)) * 1024L * 1024L
+        srcmnt = misc.mkdtemp("srcmnt")
+        disk = fs_related.SparseLoopbackDisk(srcimg, srcimgsize)
+        srcloop = PartitionedMount(srcmnt, skipformat = True)
+
+        srcloop.add_disk('/dev/sdb', disk)
+        srcloop.add_partition(srcimgsize/1024/1024, "/dev/sdb", "/", "ext3", boot=False)
+        try:
+            srcloop.mount()
+
+        except errors.MountError:
+            srcloop.cleanup()
+            raise
+
+        image = os.path.join(tempfile.mkdtemp(dir = "/var/tmp", prefix = "tmp"), "target.img")
+        args = ['dd', "if=%s" % srcloop.partitions[0]['device'], "of=%s" % image]
+
+        msger.info("`dd` image ...")
+        rc = runner.show(args)
+        srcloop.cleanup()
+        shutil.rmtree(os.path.dirname(srcmnt), ignore_errors = True)
+
+        if rc != 0:
+            raise errors.CreatorError("Failed to dd")
+        else:
+            return image