bitbake: fetch2: Add the ability to list expanded URL data

Some fetchers may download additional sources along with those
explicitly listed in SRC_URI. These "implicit URLs" will be needed by
the archiver to ensure that all sources can be archived.

We can't just return a list of URL strings since each URL may need its
own SRCREV data so we return a list of FetchData objects.

Each fetcher can override the implicit_urldata() function to provide the
additional FetchData objects. For now this is just needed in the gitsm
fetcher to walk git submodules recursively.

(Bitbake rev: 1350f241b7d991bd191ce9e44f6662e4376c6e24)

Signed-off-by: Paul Barker <pbarker@konsulko.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

bitbake/lib/bb/fetch2/__init__.py

@@ -1617,6 +1617,13 @@ class FetchMethod(object):
             return True
         return False
 
+    def implicit_urldata(self, ud, d):
+        """
+        Get a list of FetchData objects for any implicit URLs that will also
+        be downloaded when we fetch the given URL.
+        """
+        return []
+
 class Fetch(object):
     def __init__(self, urls, d, cache = True, localonly = False, connection_cache = None):
         if localonly and cache:
@@ -1842,6 +1849,24 @@ class Fetch(object):
             if ud.lockfile:
                 bb.utils.unlockfile(lf)
 
+    def expanded_urldata(self, urls=None):
+        """
+        Get an expanded list of FetchData objects covering both the given
+        URLS and any additional implicit URLs that are added automatically by
+        the appropriate FetchMethod.
+        """
+
+        if not urls:
+            urls = self.urls
+
+        urldata = []
+        for url in urls:
+            ud = self.ud[url]
+            urldata.append(ud)
+            urldata += ud.method.implicit_urldata(ud, self.d)
+
+        return urldata
+
 class FetchConnectionCache(object):
     """
         A class which represents an container for socket connections.

bitbake/lib/bb/fetch2/gitsm.py

@@ -223,3 +223,24 @@ class GitSM(Git):
             # up the configuration and checks out the files.  The main project config should remain
             # unmodified, and no download from the internet should occur.
             runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)
+
+    def implicit_urldata(self, ud, d):
+        import shutil, subprocess, tempfile
+
+        urldata = []
+        def add_submodule(ud, url, module, modpath, workdir, d):
+            url += ";bareclone=1;nobranch=1"
+            newfetch = Fetch([url], d, cache=False)
+            urldata.extend(newfetch.expanded_urldata())
+
+        # If we're using a shallow mirror tarball it needs to be unpacked
+        # temporarily so that we can examine the .gitmodules file
+        if ud.shallow and os.path.exists(ud.fullshallow) and ud.method.need_update(ud, d):
+            tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
+            subprocess.check_call("tar -xzf %s" % ud.fullshallow, cwd=tmpdir, shell=True)
+            self.process_submodules(ud, tmpdir, add_submodule, d)
+            shutil.rmtree(tmpdir)
+        else:
+            self.process_submodules(ud, ud.clonedir, add_submodule, d)
+
+        return urldata