bitbake: tests: Add initial scenario based test for runqueue

We need some tests for runqueue, its been something which has been hard to test
for a long time. Add some dummy metadata to allow this, mirroring the OE
structure in spirit.

(Bitbake rev: 37564d7440c5d7aa05ec537f3b79026b1c83bb68)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

bitbake/bin/bitbake-selftest

@@ -25,6 +25,7 @@ tests = ["bb.tests.codeparser",
          "bb.tests.fetch",
          "bb.tests.parse",
          "bb.tests.persist_data",
+         "bb.tests.runqueue",
          "bb.tests.utils",
          "hashserv.tests",
          "layerindexlib.tests.layerindexobj",

bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass

@@ -0,0 +1,219 @@
+SLOWTASKS ??= ""
+SSTATEVALID ??= ""
+
+def stamptask(d):
+    import time
+
+    thistask = d.expand("${PN}:${BB_CURRENTTASK}") 
+    if thistask in d.getVar("SLOWTASKS").split():
+        bb.note("Slowing task %s" % thistask)
+        time.sleep(0.5)
+
+    with open(d.expand("${TOPDIR}/task.log"), "a+") as f:
+        f.write(thistask + "\n")
+
+python do_fetch() {
+    stamptask(d)
+}
+python do_unpack() {
+    stamptask(d)
+}
+python do_patch() {
+    stamptask(d)
+}
+python do_populate_lic() {
+    stamptask(d)
+}
+python do_prepare_recipe_sysroot() {
+    stamptask(d)
+}
+python do_configure() {
+    stamptask(d)
+}
+python do_compile() {
+    stamptask(d)
+}
+python do_install() {
+    stamptask(d)
+}
+python do_populate_sysroot() {
+    stamptask(d)
+}
+python do_package() {
+    stamptask(d)
+}
+python do_package_write_ipk() {
+    stamptask(d)
+}
+python do_package_write_rpm() {
+    stamptask(d)
+}
+python do_packagedata() {
+    stamptask(d)
+}
+python do_package_qa() {
+    stamptask(d)
+}
+python do_build() {
+    stamptask(d)
+}
+do_prepare_recipe_sysroot[deptask] = "do_populate_sysroot"
+do_package[deptask] += "do_packagedata"
+do_build[recrdeptask] += "do_deploy"
+do_build[recrdeptask] += "do_package_write_ipk"
+do_build[recrdeptask] += "do_package_write_rpm"
+do_package_qa[rdeptask] = "do_packagedata"
+do_populate_lic_deploy[recrdeptask] += "do_populate_lic do_deploy"
+
+DEBIANRDEP = "do_packagedata"
+oo_package_write_ipk[rdeptask] = "${DEBIANRDEP}"
+do_package_write_rpm[rdeptask] = "${DEBIANRDEP}"
+
+addtask fetch
+addtask unpack after do_fetch
+addtask patch after do_unpack
+addtask prepare_recipe_sysroot after do_patch
+addtask configure after do_prepare_recipe_sysroot
+addtask compile after do_configure
+addtask install after do_compile
+addtask populate_sysroot after do_install
+addtask package after do_install
+addtask package_write_ipk after do_packagedata do_package
+addtask package_write_rpm after do_packagedata do_package
+addtask packagedata after do_package
+addtask package_qa after do_package
+addtask build after do_package_qa do_package_write_rpm do_package_write_ipk do_populate_sysroot
+
+python do_package_setscene() {
+    stamptask(d)
+}
+python do_package_qa_setscene() {
+    stamptask(d)
+}
+python do_package_write_ipk_setscene() {
+    stamptask(d)
+}
+python do_package_write_rpm_setscene() {
+    stamptask(d)
+}
+python do_packagedata_setscene() {
+    stamptask(d)
+}
+python do_populate_lic_setscene() {
+    stamptask(d)
+}
+python do_populate_sysroot_setscene() {
+    stamptask(d)
+}
+
+addtask package_setscene
+addtask package_qa_setscene
+addtask package_write_ipk_setscene
+addtask package_write_rpm_setscene
+addtask packagedata_setscene
+addtask populate_lic_setscene
+addtask populate_sysroot_setscene
+
+BB_SETSCENE_DEPVALID = "setscene_depvalid"
+
+def setscene_depvalid(task, taskdependees, notneeded, d, log=None):
+    # taskdependees is a dict of tasks which depend on task, each being a 3 item list of [PN, TASKNAME, FILENAME]
+    # task is included in taskdependees too
+    # Return - False - We need this dependency
+    #        - True - We can skip this dependency
+    import re
+
+    def logit(msg, log):
+        if log is not None:
+            log.append(msg)
+        else:
+            bb.debug(2, msg)
+
+    logit("Considering setscene task: %s" % (str(taskdependees[task])), log)
+
+    def isNativeCross(x):
+        return x.endswith("-native") or "-cross-" in x or "-crosssdk" in x or x.endswith("-cross")
+
+    # We only need to trigger populate_lic through direct dependencies
+    if taskdependees[task][1] == "do_populate_lic":
+        return True
+
+    # We only need to trigger packagedata through direct dependencies
+    # but need to preserve packagedata on packagedata links
+    if taskdependees[task][1] == "do_packagedata":
+        for dep in taskdependees:
+            if taskdependees[dep][1] == "do_packagedata":
+                return False
+        return True
+
+    for dep in taskdependees:
+        logit("  considering dependency: %s" % (str(taskdependees[dep])), log)
+        if task == dep:
+            continue
+        if dep in notneeded:
+            continue
+        # do_package_write_* and do_package doesn't need do_package
+        if taskdependees[task][1] == "do_package" and taskdependees[dep][1] in ['do_package', 'do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata', 'do_package_qa']:
+            continue
+        # do_package_write_* need do_populate_sysroot as they're mainly postinstall dependencies
+        if taskdependees[task][1] == "do_populate_sysroot" and taskdependees[dep][1] in ['do_package_write_ipk', 'do_package_write_rpm']:
+            return False
+        # do_package/packagedata/package_qa don't need do_populate_sysroot
+        if taskdependees[task][1] == "do_populate_sysroot" and taskdependees[dep][1] in ['do_package', 'do_packagedata', 'do_package_qa']:
+            continue
+        # Native/Cross packages don't exist and are noexec anyway
+        if isNativeCross(taskdependees[dep][0]) and taskdependees[dep][1] in ['do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata', 'do_package', 'do_package_qa']:
+            continue
+
+        # This is due to the [depends] in useradd.bbclass complicating matters
+        # The logic *is* reversed here due to the way hard setscene dependencies are injected
+        if (taskdependees[task][1] == 'do_package' or taskdependees[task][1] == 'do_populate_sysroot') and taskdependees[dep][0].endswith(('shadow-native', 'shadow-sysroot', 'base-passwd', 'pseudo-native')) and taskdependees[dep][1] == 'do_populate_sysroot':
+            continue
+
+        # Consider sysroot depending on sysroot tasks
+        if taskdependees[task][1] == 'do_populate_sysroot' and taskdependees[dep][1] == 'do_populate_sysroot':
+            # Native/Cross populate_sysroot need their dependencies
+            if isNativeCross(taskdependees[task][0]) and isNativeCross(taskdependees[dep][0]):
+                return False
+            # Target populate_sysroot depended on by cross tools need to be installed
+            if isNativeCross(taskdependees[dep][0]):
+                return False
+            # Native/cross tools depended upon by target sysroot are not needed
+            # Add an exception for shadow-native as required by useradd.bbclass
+            if isNativeCross(taskdependees[task][0]) and taskdependees[task][0] != 'shadow-native':
+                continue
+            # Target populate_sysroot need their dependencies
+            return False
+
+
+        if taskdependees[dep][1] == "do_populate_lic":
+            continue
+
+        # Safe fallthrough default
+        logit(" Default setscene dependency fall through due to dependency: %s" % (str(taskdependees[dep])), log)
+        return False
+    return True
+
+BB_HASHCHECK_FUNCTION = "sstate_checkhashes"
+
+def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False, *, sq_unihash=None):
+
+    ret = []
+    missed = []
+
+    valid = d.getVar("SSTATEVALID").split()
+
+    for task in range(len(sq_fn)):
+        n = os.path.basename(sq_fn[task]).rsplit(".", 1)[0] + ":" + sq_task[task]
+        if n in valid:
+            bb.note("SState: Found valid sstate for %s" % n)
+            ret.append(task)
+        else:
+            missed.append(task)
+            bb.note("SState: Found no valid sstate for %s" % n)
+
+    if hasattr(bb.parse.siggen, "checkhashes"):
+        bb.parse.siggen.checkhashes(missed, ret, sq_fn, sq_task, sq_hash, sq_hashfn, d)
+
+    return ret
+

bitbake/lib/bb/tests/runqueue-tests/classes/image.bbclass

@@ -0,0 +1,5 @@
+do_rootfs[recrdeptask] += "do_package_write_deb do_package_qa"
+do_rootfs[recrdeptask] += "do_package_write_ipk do_package_qa"
+do_rootfs[recrdeptask] += "do_package_write_rpm do_package_qa
+do_rootfs[recrdeptask] += "do_packagedata"
+do_rootfs[recrdeptask] += "do_populate_lic"

bitbake/lib/bb/tests/runqueue-tests/classes/native.bbclass

@@ -0,0 +1,2 @@
+RECIPERDEPTASK = "do_populate_sysroot"
+do_populate_sysroot[rdeptask] = "${RECIPERDEPTASK}"

bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf

@@ -0,0 +1,11 @@
+CACHE = "${TOPDIR}/cache"
+THISDIR = "${@os.path.dirname(d.getVar('FILE'))}"
+COREBASE := "${@os.path.normpath(os.path.dirname(d.getVar('FILE')+'/../../'))}"
+BBFILES = "${COREBASE}/recipes/*.bb"
+PROVIDES = "${PN}"
+PN = "${@bb.parse.vars_from_file(d.getVar('FILE', False),d)[0]}"
+PF = "${BB_CURRENT_MC}:${PN}"
+export PATH
+STAMP = "${TOPDIR}/stamps/${PN}"
+T = "${TOPDIR}/workdir/${PN}/temp"
+BB_NUMBER_THREADS = "4"

bitbake/lib/bb/tests/runqueue-tests/recipes/b1.bb

@@ -0,0 +1 @@
+DEPENDS = "a1"

bitbake/lib/bb/tests/runqueue-tests/recipes/d1.bb

@@ -0,0 +1,3 @@
+DEPENDS = "a1"
+
+do_package_setscene[depends] = "a1:do_populate_sysroot_setscene"

bitbake/lib/bb/tests/runqueue.py

@@ -0,0 +1,200 @@
+#
+# BitBake Tests for runqueue task processing
+#
+# Copyright (C) 2019 Richard Purdie
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import unittest
+import bb
+import os
+import tempfile
+import subprocess
+
+#
+# TODO:
+# Add tests on task ordering (X happens before Y after Z)
+#
+
+class RunQueueTests(unittest.TestCase):
+
+    alltasks = ['package', 'fetch', 'unpack', 'patch', 'prepare_recipe_sysroot', 'configure',
+                'compile', 'install', 'packagedata', 'package_qa', 'package_write_rpm', 'package_write_ipk',
+                'populate_sysroot', 'build']
+    a1_sstatevalid = "a1:do_package a1:do_package_qa a1:do_packagedata a1:do_package_write_ipk a1:do_package_write_rpm a1:do_populate_lic a1:do_populate_sysroot"
+    b1_sstatevalid = "b1:do_package b1:do_package_qa b1:do_packagedata b1:do_package_write_ipk b1:do_package_write_rpm b1:do_populate_lic b1:do_populate_sysroot"
+
+    def run_bitbakecmd(self, cmd, builddir, sstatevalid="", slowtasks="", extraenv=None):
+        env = os.environ.copy()
+        env["BBPATH"] = os.path.realpath(os.path.join(os.path.dirname(__file__), "runqueue-tests"))
+        env["BB_ENV_EXTRAWHITE"] = "SSTATEVALID SLOWTASKS"
+        env["SSTATEVALID"] = sstatevalid
+        env["SLOWTASKS"] = slowtasks
+        if extraenv:
+            for k in extraenv:
+                env[k] = extraenv[k]
+                env["BB_ENV_EXTRAWHITE"] = env["BB_ENV_EXTRAWHITE"] + " " + k
+        try:
+            output = subprocess.check_output(cmd, env=env, stderr=subprocess.STDOUT,universal_newlines=True, cwd=builddir)
+        except subprocess.CalledProcessError as e:
+            self.fail("Command %s failed with %s" % (cmd, e.output))
+        tasks = []
+        with open(builddir + "/task.log", "r") as f:
+            tasks = [line.rstrip() for line in f]
+        return tasks
+
+    def test_no_setscenevalid(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "a1"]
+            sstatevalid = ""
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:' + x for x in self.alltasks]
+            self.assertEqual(set(tasks), set(expected))
+
+    def test_single_setscenevalid(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "a1"]
+            sstatevalid = "a1:do_package"
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:package_setscene', 'a1:fetch', 'a1:unpack', 'a1:patch', 'a1:prepare_recipe_sysroot', 'a1:configure',
+                        'a1:compile', 'a1:install', 'a1:packagedata', 'a1:package_qa', 'a1:package_write_rpm', 'a1:package_write_ipk',
+                        'a1:populate_sysroot', 'a1:build']
+            self.assertEqual(set(tasks), set(expected))
+
+    def test_intermediate_setscenevalid(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "a1"]
+            sstatevalid = "a1:do_package a1:do_populate_sysroot"
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:package_setscene', 'a1:packagedata', 'a1:package_qa', 'a1:package_write_rpm', 'a1:package_write_ipk',
+                        'a1:populate_sysroot_setscene', 'a1:build']
+            self.assertEqual(set(tasks), set(expected))
+
+    def test_intermediate_notcovered(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "a1"]
+            sstatevalid = "a1:do_package_qa a1:do_packagedata a1:do_package_write_ipk a1:do_package_write_rpm a1:do_populate_lic a1:do_populate_sysroot"
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:package_write_ipk_setscene', 'a1:package_write_rpm_setscene', 'a1:packagedata_setscene',
+                        'a1:package_qa_setscene', 'a1:build', 'a1:populate_sysroot_setscene']
+            self.assertEqual(set(tasks), set(expected))
+
+    def test_all_setscenevalid(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "a1"]
+            sstatevalid = self.a1_sstatevalid
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:package_write_ipk_setscene', 'a1:package_write_rpm_setscene', 'a1:packagedata_setscene',
+                        'a1:package_qa_setscene', 'a1:build', 'a1:populate_sysroot_setscene']
+            self.assertEqual(set(tasks), set(expected))
+
+    def test_no_settasks(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "a1", "-c", "patch"]
+            sstatevalid = self.a1_sstatevalid
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:fetch', 'a1:unpack', 'a1:patch']
+            self.assertEqual(set(tasks), set(expected))
+
+    # Test targets with intermediate setscene tasks alongside a target with no intermediate setscene tasks
+    def test_mixed_direct_tasks_setscene_tasks(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "c1:do_patch", "a1"]
+            sstatevalid = self.a1_sstatevalid
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['c1:fetch', 'c1:unpack', 'c1:patch', 'a1:package_write_ipk_setscene', 'a1:package_write_rpm_setscene', 'a1:packagedata_setscene',
+                        'a1:package_qa_setscene', 'a1:build', 'a1:populate_sysroot_setscene']
+            self.assertEqual(set(tasks), set(expected))
+
+    # This test slows down the execution of do_package_setscene until after other real tasks have
+    # started running which tests for a bug where tasks were being lost from the buildable list of real
+    # tasks if they weren't in tasks_covered or tasks_notcovered
+    def test_slow_setscene(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "a1"]
+            sstatevalid = "a1:do_package"
+            slowtasks = "a1:package_setscene"
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, slowtasks)
+            expected = ['a1:package_setscene', 'a1:fetch', 'a1:unpack', 'a1:patch', 'a1:prepare_recipe_sysroot', 'a1:configure',
+                        'a1:compile', 'a1:install', 'a1:packagedata', 'a1:package_qa', 'a1:package_write_rpm', 'a1:package_write_ipk',
+                        'a1:populate_sysroot', 'a1:build']
+            self.assertEqual(set(tasks), set(expected))
+
+    def test_setscenewhitelist(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "a1"]
+            extraenv = {
+                "BB_SETSCENE_ENFORCE" : "1",
+                "BB_SETSCENE_ENFORCE_WHITELIST" : "a1:do_package_write_rpm a1:do_build"
+            }
+            sstatevalid = "a1:do_package a1:do_package_qa a1:do_packagedata a1:do_package_write_ipk a1:do_populate_lic a1:do_populate_sysroot"
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv)
+            expected = ['a1:packagedata_setscene', 'a1:package_qa_setscene', 'a1:package_write_ipk_setscene',
+                        'a1:populate_sysroot_setscene', 'a1:package_setscene']
+            self.assertEqual(set(tasks), set(expected))
+
+    # Tests for problems with dependencies between setscene tasks
+    def test_no_setscenevalid_harddeps(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "d1"]
+            sstatevalid = ""
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:package', 'a1:fetch', 'a1:unpack', 'a1:patch', 'a1:prepare_recipe_sysroot', 'a1:configure',
+                        'a1:compile', 'a1:install', 'a1:packagedata', 'a1:package_write_rpm', 'a1:package_write_ipk',
+                        'a1:populate_sysroot', 'd1:package', 'd1:fetch', 'd1:unpack', 'd1:patch', 'd1:prepare_recipe_sysroot', 'd1:configure',
+                        'd1:compile', 'd1:install', 'd1:packagedata', 'd1:package_qa', 'd1:package_write_rpm', 'd1:package_write_ipk',
+                        'd1:populate_sysroot', 'd1:build']
+            self.assertEqual(set(tasks), set(expected))
+
+    def test_no_setscenevalid_withdeps(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "b1"]
+            sstatevalid = ""
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks]
+            expected.remove('a1:build')
+            expected.remove('a1:package_qa')
+            self.assertEqual(set(tasks), set(expected))
+
+    def test_single_a1_setscenevalid_withdeps(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "b1"]
+            sstatevalid = "a1:do_package"
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:package_setscene', 'a1:fetch', 'a1:unpack', 'a1:patch', 'a1:prepare_recipe_sysroot', 'a1:configure',
+                        'a1:compile', 'a1:install', 'a1:packagedata', 'a1:package_write_rpm', 'a1:package_write_ipk',
+                        'a1:populate_sysroot'] + ['b1:' + x for x in self.alltasks]
+            self.assertEqual(set(tasks), set(expected))
+
+    def test_single_b1_setscenevalid_withdeps(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "b1"]
+            sstatevalid = "b1:do_package"
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:package', 'a1:fetch', 'a1:unpack', 'a1:patch', 'a1:prepare_recipe_sysroot', 'a1:configure',
+                        'a1:compile', 'a1:install', 'a1:packagedata', 'a1:package_write_rpm', 'a1:package_write_ipk',
+                        'a1:populate_sysroot', 'b1:package_setscene'] + ['b1:' + x for x in self.alltasks]
+            expected.remove('b1:package')
+            self.assertEqual(set(tasks), set(expected))
+
+    def test_intermediate_setscenevalid_withdeps(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "b1"]
+            sstatevalid = "a1:do_package a1:do_populate_sysroot b1:do_package"
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:package_setscene', 'a1:packagedata', 'a1:package_write_rpm', 'a1:package_write_ipk',
+                        'a1:populate_sysroot_setscene', 'b1:package_setscene'] + ['b1:' + x for x in self.alltasks]
+            expected.remove('b1:package')
+            self.assertEqual(set(tasks), set(expected))
+
+    def test_all_setscenevalid_withdeps(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "b1"]
+            sstatevalid = self.a1_sstatevalid + " " + self.b1_sstatevalid
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:package_write_ipk_setscene', 'a1:package_write_rpm_setscene', 'a1:packagedata_setscene',
+                        'b1:build', 'a1:populate_sysroot_setscene', 'b1:package_write_ipk_setscene', 'b1:package_write_rpm_setscene',
+                        'b1:packagedata_setscene', 'b1:package_qa_setscene', 'b1:populate_sysroot_setscene']
+            self.assertEqual(set(tasks), set(expected))
+