mirrors/poky
1
0
Fork 0
mirror of https://git.yoctoproject.org/poky synced 2026-04-27 03:32:12 +02:00
Code Issues Packages Projects Releases Wiki Activity

bitbake: fetch/wget: mitigate a wget race condition when listing FTP directories

Browse Source 
When wget is fetching a listing for a directory over FTP it writes to a
temporary file called .listing in the current directory.  If there are many such
operations happening in parallel - for example during 'bitbake world -c
checkpkg' - then up to BB_NUMBER_THREADS instances of wget will be racing to
write to, read, and delete the same file.

This results in various failures such as the file disappearing before wget has
processed it or the file changing contents, which causes checkpkg to randomly
fail.

Mitigate the race condition by creating a temporary directory to run wget in
when doing directory listings.

[ YOCTO #11828 ]

(Bitbake rev: 91d4ca93df092cf86ab84faaa94cc66ff9f43057)

Signed-off-by: Ross Burton <ross.burton@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Ross Burton
2017-07-24 22:42:01 +01:00
committed by Richard Purdie
parent 33b73b2d79
commit 3c1fba5d30
1 changed files with 11 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
bitbake/lib/bb/fetch2/wget.py
View File

@@ -90,13 +90,13 @@ class Wget(FetchMethod):
self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -t 2 -T 30 --passive-ftp --no-check-certificate"
def _runwget(self, ud, d, command, quiet):
def _runwget(self, ud, d, command, quiet, workdir=None):
progresshandler = WgetProgressHandler(d)
logger.debug(2, "Fetching %s using command '%s'" % (ud.url, command))
bb.fetch2.check_network_access(d, command, ud.url)
runfetchcmd(command + ' --progress=dot -v', d, quiet, log=progresshandler)
runfetchcmd(command + ' --progress=dot -v', d, quiet, log=progresshandler, workdir=workdir)
def download(self, ud, d):
"""Fetch urls"""
@@ -422,17 +422,16 @@ class Wget(FetchMethod):
Run fetch checkstatus to get directory information
"""
f = tempfile.NamedTemporaryFile()
with tempfile.TemporaryDirectory(prefix="wget-index-") as workdir, tempfile.NamedTemporaryFile(dir=workdir, prefix="wget-listing-") as f:
agent = "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.12) Gecko/20101027 Ubuntu/9.10 (karmic) Firefox/3.6.12"
fetchcmd = self.basecmd
fetchcmd += " -O " + f.name + " --user-agent='" + agent + "' '" + uri + "'"
try:
self._runwget(ud, d, fetchcmd, True, workdir=workdir)
fetchresult = f.read()
except bb.fetch2.BBFetchException:
fetchresult = ""
agent = "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.12) Gecko/20101027 Ubuntu/9.10 (karmic) Firefox/3.6.12"
fetchcmd = self.basecmd
fetchcmd += " -O " + f.name + " --user-agent='" + agent + "' '" + uri + "'"
try:
self._runwget(ud, d, fetchcmd, True)
fetchresult = f.read()
except bb.fetch2.BBFetchException:
fetchresult = ""
f.close()
return fetchresult
def _check_latest_version(self, url, package, package_regex, current_version, ud, d):