bitbake: fetch2: Drop DL_DIR fallback for local file fetcher

A long time ago, we made DL_DIR a final fallback for the local fetcher.
Since then we added checksum support and task hashes and the world
has changed. There were warnings added some time ago if this fallback
triggers and it is now time to drop it entirely.

The original use case was for sstate however the sstate code now sets
FILESPATH correctly so DL_DIR is no longer needed.

There have been a few small bugs exposed by this change, missing mkdir
calls and some minor test issues that needed tweaks. In general this
simplifies and improves the fetcher code flow though.

This completes a cleanup that ensures local files are correctly covered
at parse time which ensures rebuilds and reparses happen at the right
times.

(Bitbake rev: 3e1444e536c71d3885ef6b9d428807163c309640)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

bitbake/lib/bb/fetch2/__init__.py

@@ -1222,25 +1222,21 @@ def get_checksum_file_list(d):
     SRC_URI as a space-separated string
     """
     fetch = Fetch([], d, cache = False, localonly = True)
-
-    dl_dir = d.getVar('DL_DIR')
     filelist = []
     for u in fetch.urls:
         ud = fetch.ud[u]
-
         if ud and isinstance(ud.method, local.Local):
+            found = False
             paths = ud.method.localpaths(ud, d)
             for f in paths:
                 pth = ud.decodedurl
-                if f.startswith(dl_dir):
-                    # The local fetcher's behaviour is to return a path under DL_DIR if it couldn't find the file anywhere else
-                    if os.path.exists(f):
-                        bb.warn("Getting checksum for %s SRC_URI entry %s: file not found except in DL_DIR" % (d.getVar('PN'), os.path.basename(f)))
-                    else:
-                        bb.fatal(("Unable to get checksum for %s SRC_URI entry %s: file could not be found"
+                if os.path.exists(f):
+                    found = True
+                filelist.append(f + ":" + str(os.path.exists(f)))
+            if not found:
+                bb.fatal(("Unable to get checksum for %s SRC_URI entry %s: file could not be found"
                             "\nThe following paths were searched:"
                             "\n%s") % (d.getVar('PN'), os.path.basename(f), '\n'.join(paths)))
-                filelist.append(f + ":" + str(os.path.exists(f)))
 
     return " ".join(filelist)
 

bitbake/lib/bb/fetch2/local.py

@@ -57,12 +57,6 @@ class Local(FetchMethod):
             logger.debug2("Searching for %s in paths:\n    %s" % (path, "\n    ".join(filespath.split(":"))))
             newpath, hist = bb.utils.which(filespath, path, history=True)
             searched.extend(hist)
-        if not os.path.exists(newpath):
-            dldirfile = os.path.join(d.getVar("DL_DIR"), path)
-            logger.debug2("Defaulting to %s for %s" % (dldirfile, path))
-            bb.utils.mkdirhier(os.path.dirname(dldirfile))
-            searched.append(dldirfile)
-            return searched
         return searched
 
     def need_update(self, ud, d):
@@ -78,8 +72,6 @@ class Local(FetchMethod):
             filespath = d.getVar('FILESPATH')
             if filespath:
                 locations = filespath.split(":")
-            locations.append(d.getVar("DL_DIR"))
-
             msg = "Unable to find file " + urldata.url + " anywhere. The paths that were searched were:\n    " + "\n    ".join(locations)
             raise FetchError(msg)
 

bitbake/lib/bb/tests/fetch.py

@@ -468,6 +468,7 @@ class MirrorUriTest(FetcherTest):
                 "http://.*/.* file:///someotherpath/downloads/"
 
     def test_urireplace(self):
+        self.d.setVar("FILESPATH", ".")
         for k, v in self.replaceuris.items():
             ud = bb.fetch.FetchData(k[0], self.d)
             ud.setup_localpath(self.d)
@@ -925,6 +926,7 @@ class FetcherNetworkTest(FetcherTest):
 
     @skipIfNoNetwork()
     def test_fetch_file_mirror_of_mirror(self):
+        self.d.setVar("FILESPATH", ".")
         self.d.setVar("MIRRORS", "http://.*/.* file:///some1where/ file:///some1where/.* file://some2where/ file://some2where/.* https://downloads.yoctoproject.org/releases/bitbake")
         fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
         os.mkdir(self.dldir + "/some2where")