diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py index 79a8906af7..1f63a045c1 100644 --- a/bitbake/lib/bb/fetch2/__init__.py +++ b/bitbake/lib/bb/fetch2/__init__.py @@ -425,7 +425,7 @@ def encodeurl(decoded): return url -def uri_replace(ud, uri_find, uri_replace, replacements, d): +def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None): if not ud.url or not uri_find or not uri_replace: logger.error("uri_replace: passed an undefined value, not replacing") return None @@ -464,9 +464,9 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d): if loc == 2: # Handle path manipulations basename = None - if uri_decoded[0] != uri_replace_decoded[0] and ud.mirrortarball: + if uri_decoded[0] != uri_replace_decoded[0] and mirrortarball: # If the source and destination url types differ, must be a mirrortarball mapping - basename = os.path.basename(ud.mirrortarball) + basename = os.path.basename(mirrortarball) # Kill parameters, they make no sense for mirror tarballs uri_decoded[5] = {} elif ud.localpath and ud.method.supports_checksum(ud): @@ -892,45 +892,47 @@ def build_mirroruris(origud, mirrors, ld): replacements["BASENAME"] = origud.path.split("/")[-1] replacements["MIRRORNAME"] = origud.host.replace(':','.') + origud.path.replace('/', '.').replace('*', '.') - def adduri(ud, uris, uds, mirrors): + def adduri(ud, uris, uds, mirrors, tarballs): for line in mirrors: try: (find, replace) = line except ValueError: continue - newuri = uri_replace(ud, find, replace, replacements, ld) - if not newuri or newuri in uris or newuri == origud.url: - continue - if not trusted_network(ld, newuri): - logger.debug(1, "Mirror %s not in the list of trusted networks, skipping" % (newuri)) - continue + for tarball in tarballs: + newuri = uri_replace(ud, find, replace, replacements, ld, tarball) + if not newuri or newuri in uris or newuri == origud.url: + continue - # Create a local copy of the mirrors minus the current line - # this will prevent us from recursively processing the same line - # as well as indirect recursion A -> B -> C -> A - localmirrors = list(mirrors) - localmirrors.remove(line) + if not trusted_network(ld, newuri): + logger.debug(1, "Mirror %s not in the list of trusted networks, skipping" % (newuri)) + continue + + # Create a local copy of the mirrors minus the current line + # this will prevent us from recursively processing the same line + # as well as indirect recursion A -> B -> C -> A + localmirrors = list(mirrors) + localmirrors.remove(line) - try: - newud = FetchData(newuri, ld) - newud.setup_localpath(ld) - except bb.fetch2.BBFetchException as e: - logger.debug(1, "Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url)) - logger.debug(1, str(e)) try: - # setup_localpath of file:// urls may fail, we should still see - # if mirrors of the url exist - adduri(newud, uris, uds, localmirrors) - except UnboundLocalError: - pass - continue - uris.append(newuri) - uds.append(newud) + newud = FetchData(newuri, ld) + newud.setup_localpath(ld) + except bb.fetch2.BBFetchException as e: + logger.debug(1, "Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url)) + logger.debug(1, str(e)) + try: + # setup_localpath of file:// urls may fail, we should still see + # if mirrors of the url exist + adduri(newud, uris, uds, localmirrors, tarballs) + except UnboundLocalError: + pass + continue + uris.append(newuri) + uds.append(newud) - adduri(newud, uris, uds, localmirrors) + adduri(newud, uris, uds, localmirrors, tarballs) - adduri(origud, uris, uds, mirrors) + adduri(origud, uris, uds, mirrors, origud.mirrortarballs or [None]) return uris, uds @@ -975,8 +977,8 @@ def try_mirror_url(fetch, origud, ud, ld, check = False): # We may be obtaining a mirror tarball which needs further processing by the real fetcher # If that tarball is a local file:// we need to provide a symlink to it dldir = ld.getVar("DL_DIR") - if origud.mirrortarball and os.path.basename(ud.localpath) == os.path.basename(origud.mirrortarball) \ - and os.path.basename(ud.localpath) != os.path.basename(origud.localpath): + + if origud.mirrortarballs and os.path.basename(ud.localpath) in origud.mirrortarballs and os.path.basename(ud.localpath) != os.path.basename(origud.localpath): # Create donestamp in old format to avoid triggering a re-download if ud.donestamp: bb.utils.mkdirhier(os.path.dirname(ud.donestamp)) @@ -993,7 +995,7 @@ def try_mirror_url(fetch, origud, ud, ld, check = False): pass if not verify_donestamp(origud, ld) or origud.method.need_update(origud, ld): origud.method.download(origud, ld) - if hasattr(origud.method,"build_mirror_data"): + if hasattr(origud.method, "build_mirror_data"): origud.method.build_mirror_data(origud, ld) return origud.localpath # Otherwise the result is a local file:// and we symlink to it @@ -1190,7 +1192,7 @@ class FetchData(object): self.localfile = "" self.localpath = None self.lockfile = None - self.mirrortarball = None + self.mirrortarballs = [] self.basename = None self.basepath = None (self.type, self.host, self.path, self.user, self.pswd, self.parm) = decodeurl(d.expand(url)) diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py index 2550bde838..01d4bbdc2e 100644 --- a/bitbake/lib/bb/fetch2/git.py +++ b/bitbake/lib/bb/fetch2/git.py @@ -205,8 +205,9 @@ class Git(FetchMethod): if ud.rebaseable: for name in ud.names: gitsrcname = gitsrcname + '_' + ud.revisions[name] - ud.mirrortarball = 'git2_%s.tar.gz' % gitsrcname - ud.fullmirror = os.path.join(d.getVar("DL_DIR"), ud.mirrortarball) + mirrortarball = 'git2_%s.tar.gz' % gitsrcname + ud.fullmirror = os.path.join(d.getVar("DL_DIR"), mirrortarball) + ud.mirrortarballs = [mirrortarball] gitdir = d.getVar("GITDIR") or (d.getVar("DL_DIR") + "/git2/") ud.clonedir = os.path.join(gitdir, gitsrcname) diff --git a/bitbake/lib/bb/fetch2/hg.py b/bitbake/lib/bb/fetch2/hg.py index b5f268601c..d0857e63f7 100644 --- a/bitbake/lib/bb/fetch2/hg.py +++ b/bitbake/lib/bb/fetch2/hg.py @@ -76,8 +76,9 @@ class Hg(FetchMethod): # Create paths to mercurial checkouts hgsrcname = '%s_%s_%s' % (ud.module.replace('/', '.'), \ ud.host, ud.path.replace('/', '.')) - ud.mirrortarball = 'hg_%s.tar.gz' % hgsrcname - ud.fullmirror = os.path.join(d.getVar("DL_DIR"), ud.mirrortarball) + mirrortarball = 'hg_%s.tar.gz' % hgsrcname + ud.fullmirror = os.path.join(d.getVar("DL_DIR"), mirrortarball) + ud.mirrortarballs = [mirrortarball] hgdir = d.getVar("HGDIR") or (d.getVar("DL_DIR") + "/hg/") ud.pkgdir = os.path.join(hgdir, hgsrcname) diff --git a/bitbake/lib/bb/fetch2/npm.py b/bitbake/lib/bb/fetch2/npm.py index 73a75fe989..5f799724e7 100644 --- a/bitbake/lib/bb/fetch2/npm.py +++ b/bitbake/lib/bb/fetch2/npm.py @@ -91,9 +91,10 @@ class Npm(FetchMethod): ud.prefixdir = prefixdir ud.write_tarballs = ((d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0") != "0") - ud.mirrortarball = 'npm_%s-%s.tar.xz' % (ud.pkgname, ud.version) - ud.mirrortarball = ud.mirrortarball.replace('/', '-') - ud.fullmirror = os.path.join(d.getVar("DL_DIR"), ud.mirrortarball) + mirrortarball = 'npm_%s-%s.tar.xz' % (ud.pkgname, ud.version) + mirrortarball = ud.mirrortarball.replace('/', '-') + ud.fullmirror = os.path.join(d.getVar("DL_DIR"), mirrortarball) + ud.mirrortarballs = [mirrortarball] def need_update(self, ud, d): if os.path.exists(ud.localpath):