oeqa/runtime/login: Various code improvements and fixes

* Allow tools to be found from the host PATH so that imagemagick from a buildtools
  tarball/sdk can work
* Reformat the code to have imports at the start of the file and have more standard
  formatting and whitespace
* Always save copies of the images, the space imapct is negligle compared to the
  debug win
* Write the images to ${T}
* Use bb.utils.mkdirhier() instead of more complex code
* Restrict the tests to images containing matchbox-desktop

(From OE-Core rev: d09989b49517830297654e4d1d150aaa8723c41a)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

meta/classes-recipe/testimage.bbclass

@@ -221,6 +221,7 @@ def testimage_main(d):
     # ones of the current environment because some tests require them.
     for var in d.getVar('TESTIMAGE_UPDATE_VARS').split():
         td[var] = d.getVar(var)
+    td['ORIGPATH'] = d.getVar("BB_ORIGENV").getVar("PATH")
 
     image_manifest = "%s.manifest" % image_name
     image_packages = OERuntimeTestContextExecutor.readPackagesManifest(image_manifest)

meta/lib/oeqa/runtime/cases/login.py

@@ -4,9 +4,13 @@
 # SPDX-License-Identifier: MIT
 #
 
+import shutil
 import subprocess
-from oeqa.runtime.case import OERuntimeTestCase
 import tempfile
+import time
+import os
+from datetime import datetime
+from oeqa.runtime.case import OERuntimeTestCase
 from oeqa.runtime.decorator.package import OEHasPackage
 
 ### Status of qemu images.
@@ -22,14 +26,27 @@ from oeqa.runtime.decorator.package import OEHasPackage
 ###
 
 class LoginTest(OERuntimeTestCase):
+    @OEHasPackage(['matchbox-desktop'])
     def test_screenshot(self):
         if self.td.get('MACHINE') in ("qemuppc64", "qemuarmv5", "qemuriscv32", "qemuloongarch64"):
             self.skipTest("{0} is not currently supported.".format(self.td.get('MACHINE')))
 
-        # Set DEBUG_CREATE_IMAGES to 1 in order to populate the image-test images directory.
-        DEBUG_CREATE_IMAGES="0"
-        # Store failed images so we can debug them.
-        failed_image_dir=self.td.get('TOPDIR') + "/failed-images/"
+        pn = self.td.get('PN')
+
+        ourenv = os.environ.copy()
+        origpath = self.td.get("ORIGPATH")
+        if origpath:
+            ourenv['PATH'] = ourenv['PATH'] + ":" + origpath
+
+        for cmd in ["identify.im7", "convert.im7", "compare.im7"]:
+            try:
+                subprocess.check_output(["which", cmd], env=ourenv)
+            except subprocess.CalledProcessError:
+                self.skipTest("%s (from imagemagick) not available" % cmd)
+
+
+        # Store images so we can debug them if needed
+        saved_screenshots_dir = self.td.get('T') + "/saved-screenshots/"
 
         ###
         # This is a really horrible way of doing this but I've not found the
@@ -50,8 +67,6 @@ class LoginTest(OERuntimeTestCase):
         # 'works'.
         ###
 
-        import time
-        
         # qemumips takes forever to render. We could probably get away with 20
         # here were it not for that.
         time.sleep(40)
@@ -62,46 +77,36 @@ class LoginTest(OERuntimeTestCase):
             # Find out size of image so we can determine where to blank out clock.
             # qemuarm and qemuppc are odd as it doesn't resize the window and returns
             # incorrect widths
-            if self.td.get('MACHINE')=="qemuarm" or self.td.get('MACHINE')=="qemuppc":
-                width="640"
+            if self.td.get('MACHINE') == "qemuarm" or self.td.get('MACHINE') == "qemuppc":
+                width = "640"
             else:
                 cmd = "identify.im7 -ping -format '%w' {0}".format(t.name)
-                width = subprocess.check_output(cmd, shell=True).decode()
+                width = subprocess.check_output(cmd, shell=True, env=ourenv).decode()
 
-            rblank=int(float(width))
-            lblank=rblank-40
+            rblank = int(float(width))
+            lblank = rblank-40
 
             # Use the meta-oe version of convert, along with it's suffix. This blanks out the clock.
             cmd = "convert.im7 {0} -fill white -draw 'rectangle {1},10 {2},22' {3}".format(t.name, str(rblank), str(lblank), t.name)
-            convert_out=subprocess.check_output(cmd, shell=True).decode()
+            convert_out=subprocess.check_output(cmd, shell=True, env=ourenv).decode()
 
-            if DEBUG_CREATE_IMAGES=="1":
-                # You probably aren't interested in this as it's just to create the images we compare against.
-                import shutil
-                shutil.copy2(t.name, "{0}/meta/files/image-tests/core-image-sato-{1}.png".format(self.td.get('COREBASE'), \
-                                                                                                 self.td.get('MACHINE')))
-                self.skipTest("Created a reference image for {0} and placed it in {1}/meta/files/image-tests/.".format(self.td.get('MACHINE'), self.td.get('COREBASE')))
+
+            bb.utils.mkdirhier(saved_screenshots_dir)
+            savedfile = "{0}/saved-{1}-{2}-{3}.png".format(saved_screenshots_dir, \
+                                                                        datetime.timestamp(datetime.now()), \
+                                                                        pn, \
+                                                                        self.td.get('MACHINE'))
+            shutil.copy2(t.name, savedfile)
+
+            refimage = self.td.get('COREBASE') + "/meta/files/screenshot-tests/" + pn + "-" + self.td.get('MACHINE') +".png"
+            if not os.path.exists(refimage):
+                self.skipTest("No reference image for comparision (%s)" % refimage)
+
+            cmd = "compare.im7 -metric MSE {0} {1} /dev/null".format(t.name, refimage)
+            compare_out = subprocess.run(cmd, shell=True, capture_output=True, text=True, env=ourenv)
+            diff=float(compare_out.stderr.replace("(", "").replace(")","").split()[1])
+            if diff > 0:
+                # Keep a copy of the failed screenshot so we can see what happened.
+                self.fail("Screenshot diff is {0}. Failed image stored in {1}".format(str(diff), savedfile))
             else:
-                # Use the meta-oe version of compare, along with it's suffix.
-                cmd = "compare.im7 -metric MSE {0} {1}/meta/files/image-tests/core-image-sato-{2}.png /dev/null".format(t.name, \
-                                                                                                                        self.td.get('COREBASE'), \
-                                                                                                                        self.td.get('MACHINE'))
-                compare_out = subprocess.run(cmd, shell=True, capture_output=True, text=True)
-                diff=float(compare_out.stderr.replace("(", "").replace(")","").split()[1])
-                if diff > 0:
-                    from datetime import datetime
-                    import shutil
-                    import os
-                    try:
-                        os.mkdir(failed_image_dir)
-                    except FileExistsError:
-                        # directory exists
-                        pass
-                    # Keep a copy of the failed screenshot so we can see what happened.
-                    failedfile="{0}/failed-{1}-core-image-sato-{2}.png".format(failed_image_dir, \
-                                                                               datetime.timestamp(datetime.now()), \
-                                                                               self.td.get('MACHINE'))
-                    shutil.copy2(t.name, failedfile)
-                    self.fail("Screenshot diff is {0}. Failed image stored in {1}".format(str(diff), failedfile))
-                else:
-                    self.assertEqual(0, diff, "Screenshot diff is {0}.".format(str(diff)))
+                self.assertEqual(0, diff, "Screenshot diff is {0}.".format(str(diff)))