build-appliance-image: Update to scarthgap head revision

(From OE-Core rev: f888dd911529a828820799a7a1b75dfd3a44847c)

Signed-off-by: Steve Sakoman <steve@sakoman.com>

.gitignore

@@ -36,3 +36,4 @@ _toaster_clones/
 downloads/
 sstate-cache/
 toaster.sqlite
+.vscode/

bitbake/README

@@ -18,7 +18,7 @@ Bitbake requires Python version 3.8 or newer.
 Contributing
 ------------
 
-Please refer to our contributor guide here: https://docs.yoctoproject.org/dev/contributor-guide/
+Please refer to our contributor guide here: https://docs.yoctoproject.org/contributor-guide/
 for full details on how to submit changes.
 
 As a quick guide, patches should be sent to bitbake-devel@lists.openembedded.org

bitbake/SECURITY.md

@@ -0,0 +1,24 @@
+How to Report a Potential Vulnerability?
+========================================
+
+If you would like to report a public issue (for example, one with a released
+CVE number), please report it using the
+[https://bugzilla.yoctoproject.org/enter_bug.cgi?product=Security Security Bugzilla].
+If you have a patch ready, submit it following the same procedure as any other
+patch as described in README.md.
+
+If you are dealing with a not-yet released or urgent issue, please send a
+message to security AT yoctoproject DOT org, including as many details as
+possible: the layer or software module affected, the recipe and its version,
+and any example code, if available.
+
+Branches maintained with security fixes
+---------------------------------------
+
+See [https://wiki.yoctoproject.org/wiki/Stable_Release_and_LTS Stable release and LTS]
+for detailed info regarding the policies and maintenance of Stable branches.
+
+The [https://wiki.yoctoproject.org/wiki/Releases Release page] contains a list of all
+releases of the Yocto Project. Versions in grey are no longer actively maintained with
+security patches, but well-tested patches may still be accepted for them for
+significant issues.

bitbake/bin/bitbake

@@ -27,7 +27,7 @@ from bb.main import bitbake_main, BitBakeConfigParameters, BBMainException
 
 bb.utils.check_system_locale()
 
-__version__ = "2.6.0"
+__version__ = "2.8.0"
 
 if __name__ == "__main__":
     if __version__ != bb.__version__:

bitbake/bin/bitbake-diffsigs

@@ -72,13 +72,16 @@ def find_siginfo_task(bbhandler, pn, taskname, sig1=None, sig2=None):
         elif sig2 not in sigfiles:
             logger.error('No sigdata files found matching %s %s with signature %s' % (pn, taskname, sig2))
             sys.exit(1)
-        latestfiles = [sigfiles[sig1], sigfiles[sig2]]
     else:
-        filedates = find_siginfo(bbhandler, pn, taskname)
-        latestfiles = sorted(filedates.keys(), key=lambda f: filedates[f])[-2:]
-        if not latestfiles:
+        sigfiles = find_siginfo(bbhandler, pn, taskname)
+        latestsigs = sorted(sigfiles.keys(), key=lambda h: sigfiles[h]['time'])[-2:]
+        if not latestsigs:
             logger.error('No sigdata files found matching %s %s' % (pn, taskname))
             sys.exit(1)
+        sig1 = latestsigs[0]
+        sig2 = latestsigs[1]
+
+    latestfiles = [sigfiles[sig1]['path'], sigfiles[sig2]['path']]
 
     return latestfiles
 
@@ -96,7 +99,7 @@ def recursecb(key, hash1, hash2):
     elif hash2 not in hashfiles:
         recout.append("Unable to find matching sigdata for %s with hash %s" % (key, hash2))
     else:
-        out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb, color=color)
+        out2 = bb.siggen.compare_sigfiles(hashfiles[hash1]['path'], hashfiles[hash2]['path'], recursecb, color=color)
         for change in out2:
             for line in change.splitlines():
                 recout.append('    ' + line)

bitbake/bin/bitbake-hashclient

@@ -14,6 +14,8 @@ import sys
 import threading
 import time
 import warnings
+import netrc
+import json
 warnings.simplefilter("default")
 
 try:
@@ -36,18 +38,42 @@ except ImportError:
 sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), 'lib'))
 
 import hashserv
+import bb.asyncrpc
 
 DEFAULT_ADDRESS = 'unix://./hashserve.sock'
 METHOD = 'stress.test.method'
 
+def print_user(u):
+    print(f"Username:    {u['username']}")
+    if "permissions" in u:
+        print("Permissions: " + " ".join(u["permissions"]))
+    if "token" in u:
+        print(f"Token:       {u['token']}")
+
 
 def main():
+    def handle_get(args, client):
+        result = client.get_taskhash(args.method, args.taskhash, all_properties=True)
+        if not result:
+            return 0
+
+        print(json.dumps(result, sort_keys=True, indent=4))
+        return 0
+
+    def handle_get_outhash(args, client):
+        result = client.get_outhash(args.method, args.outhash, args.taskhash)
+        if not result:
+            return 0
+
+        print(json.dumps(result, sort_keys=True, indent=4))
+        return 0
+
     def handle_stats(args, client):
         if args.reset:
             s = client.reset_stats()
         else:
             s = client.get_stats()
-        pprint.pprint(s)
+        print(json.dumps(s, sort_keys=True, indent=4))
         return 0
 
     def handle_stress(args, client):
@@ -56,25 +82,24 @@ def main():
             nonlocal missed_hashes
             nonlocal max_time
 
-            client = hashserv.create_client(args.address)
+            with hashserv.create_client(args.address) as client:
+                for i in range(args.requests):
+                    taskhash = hashlib.sha256()
+                    taskhash.update(args.taskhash_seed.encode('utf-8'))
+                    taskhash.update(str(i).encode('utf-8'))
 
-            for i in range(args.requests):
-                taskhash = hashlib.sha256()
-                taskhash.update(args.taskhash_seed.encode('utf-8'))
-                taskhash.update(str(i).encode('utf-8'))
+                    start_time = time.perf_counter()
+                    l = client.get_unihash(METHOD, taskhash.hexdigest())
+                    elapsed = time.perf_counter() - start_time
 
-                start_time = time.perf_counter()
-                l = client.get_unihash(METHOD, taskhash.hexdigest())
-                elapsed = time.perf_counter() - start_time
+                    with lock:
+                        if l:
+                            found_hashes += 1
+                        else:
+                            missed_hashes += 1
 
-                with lock:
-                    if l:
-                        found_hashes += 1
-                    else:
-                        missed_hashes += 1
-
-                    max_time = max(elapsed, max_time)
-                    pbar.update()
+                        max_time = max(elapsed, max_time)
+                        pbar.update()
 
         max_time = 0
         found_hashes = 0
@@ -126,12 +151,101 @@ def main():
         print("Removed %d rows" % (result["count"]))
         return 0
 
+    def handle_refresh_token(args, client):
+        r = client.refresh_token(args.username)
+        print_user(r)
+
+    def handle_set_user_permissions(args, client):
+        r = client.set_user_perms(args.username, args.permissions)
+        print_user(r)
+
+    def handle_get_user(args, client):
+        r = client.get_user(args.username)
+        print_user(r)
+
+    def handle_get_all_users(args, client):
+        users = client.get_all_users()
+        print("{username:20}| {permissions}".format(username="Username", permissions="Permissions"))
+        print(("-" * 20) + "+" + ("-" * 20))
+        for u in users:
+            print("{username:20}| {permissions}".format(username=u["username"], permissions=" ".join(u["permissions"])))
+
+    def handle_new_user(args, client):
+        r = client.new_user(args.username, args.permissions)
+        print_user(r)
+
+    def handle_delete_user(args, client):
+        r = client.delete_user(args.username)
+        print_user(r)
+
+    def handle_get_db_usage(args, client):
+        usage = client.get_db_usage()
+        print(usage)
+        tables = sorted(usage.keys())
+        print("{name:20}| {rows:20}".format(name="Table name", rows="Rows"))
+        print(("-" * 20) + "+" + ("-" * 20))
+        for t in tables:
+            print("{name:20}| {rows:<20}".format(name=t, rows=usage[t]["rows"]))
+        print()
+
+        total_rows = sum(t["rows"] for t in usage.values())
+        print(f"Total rows: {total_rows}")
+
+    def handle_get_db_query_columns(args, client):
+        columns = client.get_db_query_columns()
+        print("\n".join(sorted(columns)))
+
+    def handle_gc_status(args, client):
+        result = client.gc_status()
+        if not result["mark"]:
+            print("No Garbage collection in progress")
+            return 0
+
+        print("Current Mark: %s" % result["mark"])
+        print("Total hashes to keep: %d" % result["keep"])
+        print("Total hashes to remove: %s" % result["remove"])
+        return 0
+
+    def handle_gc_mark(args, client):
+        where = {k: v for k, v in args.where}
+        result = client.gc_mark(args.mark, where)
+        print("New hashes marked: %d" % result["count"])
+        return 0
+
+    def handle_gc_sweep(args, client):
+        result = client.gc_sweep(args.mark)
+        print("Removed %d rows" % result["count"])
+        return 0
+
+    def handle_unihash_exists(args, client):
+        result = client.unihash_exists(args.unihash)
+        if args.quiet:
+            return 0 if result else 1
+
+        print("true" if result else "false")
+        return 0
+
     parser = argparse.ArgumentParser(description='Hash Equivalence Client')
     parser.add_argument('--address', default=DEFAULT_ADDRESS, help='Server address (default "%(default)s")')
     parser.add_argument('--log', default='WARNING', help='Set logging level')
+    parser.add_argument('--login', '-l', metavar="USERNAME", help="Authenticate as USERNAME")
+    parser.add_argument('--password', '-p', metavar="TOKEN", help="Authenticate using token TOKEN")
+    parser.add_argument('--become', '-b', metavar="USERNAME", help="Impersonate user USERNAME (if allowed) when performing actions")
+    parser.add_argument('--no-netrc', '-n', action="store_false", dest="netrc", help="Do not use .netrc")
 
     subparsers = parser.add_subparsers()
 
+    get_parser = subparsers.add_parser('get', help="Get the unihash for a taskhash")
+    get_parser.add_argument("method", help="Method to query")
+    get_parser.add_argument("taskhash", help="Task hash to query")
+    get_parser.set_defaults(func=handle_get)
+
+    get_outhash_parser = subparsers.add_parser('get-outhash', help="Get output hash information")
+    get_outhash_parser.add_argument("method", help="Method to query")
+    get_outhash_parser.add_argument("outhash", help="Output hash to query")
+    get_outhash_parser.add_argument("taskhash", help="Task hash to query")
+    get_outhash_parser.set_defaults(func=handle_get_outhash)
+
     stats_parser = subparsers.add_parser('stats', help='Show server stats')
     stats_parser.add_argument('--reset', action='store_true',
                               help='Reset server stats')
@@ -159,6 +273,55 @@ def main():
     clean_unused_parser.add_argument("max_age", metavar="SECONDS", type=int, help="Remove unused entries older than SECONDS old")
     clean_unused_parser.set_defaults(func=handle_clean_unused)
 
+    refresh_token_parser = subparsers.add_parser('refresh-token', help="Refresh auth token")
+    refresh_token_parser.add_argument("--username", "-u", help="Refresh the token for another user (if authorized)")
+    refresh_token_parser.set_defaults(func=handle_refresh_token)
+
+    set_user_perms_parser = subparsers.add_parser('set-user-perms', help="Set new permissions for user")
+    set_user_perms_parser.add_argument("--username", "-u", help="Username", required=True)
+    set_user_perms_parser.add_argument("permissions", metavar="PERM", nargs="*", default=[], help="New permissions")
+    set_user_perms_parser.set_defaults(func=handle_set_user_permissions)
+
+    get_user_parser = subparsers.add_parser('get-user', help="Get user")
+    get_user_parser.add_argument("--username", "-u", help="Username")
+    get_user_parser.set_defaults(func=handle_get_user)
+
+    get_all_users_parser = subparsers.add_parser('get-all-users', help="List all users")
+    get_all_users_parser.set_defaults(func=handle_get_all_users)
+
+    new_user_parser = subparsers.add_parser('new-user', help="Create new user")
+    new_user_parser.add_argument("--username", "-u", help="Username", required=True)
+    new_user_parser.add_argument("permissions", metavar="PERM", nargs="*", default=[], help="New permissions")
+    new_user_parser.set_defaults(func=handle_new_user)
+
+    delete_user_parser = subparsers.add_parser('delete-user', help="Delete user")
+    delete_user_parser.add_argument("--username", "-u", help="Username", required=True)
+    delete_user_parser.set_defaults(func=handle_delete_user)
+
+    db_usage_parser = subparsers.add_parser('get-db-usage', help="Database Usage")
+    db_usage_parser.set_defaults(func=handle_get_db_usage)
+
+    db_query_columns_parser = subparsers.add_parser('get-db-query-columns', help="Show columns that can be used in database queries")
+    db_query_columns_parser.set_defaults(func=handle_get_db_query_columns)
+
+    gc_status_parser = subparsers.add_parser("gc-status", help="Show garbage collection status")
+    gc_status_parser.set_defaults(func=handle_gc_status)
+
+    gc_mark_parser = subparsers.add_parser('gc-mark', help="Mark hashes to be kept for garbage collection")
+    gc_mark_parser.add_argument("mark", help="Mark for this garbage collection operation")
+    gc_mark_parser.add_argument("--where", "-w", metavar="KEY VALUE", nargs=2, action="append", default=[],
+                             help="Keep entries in table where KEY == VALUE")
+    gc_mark_parser.set_defaults(func=handle_gc_mark)
+
+    gc_sweep_parser = subparsers.add_parser('gc-sweep', help="Perform garbage collection and delete any entries that are not marked")
+    gc_sweep_parser.add_argument("mark", help="Mark for this garbage collection operation")
+    gc_sweep_parser.set_defaults(func=handle_gc_sweep)
+
+    unihash_exists_parser = subparsers.add_parser('unihash-exists', help="Check if a unihash is known to the server")
+    unihash_exists_parser.add_argument("--quiet", action="store_true", help="Don't print status. Instead, exit with 0 if unihash exists and 1 if it does not")
+    unihash_exists_parser.add_argument("unihash", help="Unihash to check")
+    unihash_exists_parser.set_defaults(func=handle_unihash_exists)
+
     args = parser.parse_args()
 
     logger = logging.getLogger('hashserv')
@@ -172,11 +335,30 @@ def main():
     console.setLevel(level)
     logger.addHandler(console)
 
+    login = args.login
+    password = args.password
+
+    if login is None and args.netrc:
+        try:
+            n = netrc.netrc()
+            auth = n.authenticators(args.address)
+            if auth is not None:
+                login, _, password = auth
+        except FileNotFoundError:
+            pass
+        except netrc.NetrcParseError as e:
+            sys.stderr.write(f"Error parsing {e.filename}:{e.lineno}: {e.msg}\n")
+
     func = getattr(args, 'func', None)
     if func:
-        client = hashserv.create_client(args.address)
-
-        return func(args, client)
+        try:
+            with hashserv.create_client(args.address, login, password) as client:
+                if args.become:
+                    client.become_user(args.become)
+                return func(args, client)
+        except bb.asyncrpc.InvokeError as e:
+            print(f"ERROR: {e}")
+            return 1
 
     return 0
 

bitbake/bin/bitbake-hashserv

@@ -11,56 +11,161 @@ import logging
 import argparse
 import sqlite3
 import warnings
+
 warnings.simplefilter("default")
 
-sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), 'lib'))
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), "lib"))
 
 import hashserv
+from hashserv.server import DEFAULT_ANON_PERMS
 
 VERSION = "1.0.0"
 
-DEFAULT_BIND = 'unix://./hashserve.sock'
+DEFAULT_BIND = "unix://./hashserve.sock"
 
 
 def main():
-    parser = argparse.ArgumentParser(description='Hash Equivalence Reference Server. Version=%s' % VERSION,
-                                     epilog='''The bind address is the path to a unix domain socket if it is
-                                               prefixed with "unix://". Otherwise, it is an IP address
-                                               and port in form ADDRESS:PORT. To bind to all addresses, leave
-                                               the ADDRESS empty, e.g. "--bind :8686". To bind to a specific
-                                               IPv6 address, enclose the address in "[]", e.g.
-                                               "--bind [::1]:8686"'''
-                                     )
+    parser = argparse.ArgumentParser(
+        description="Hash Equivalence Reference Server. Version=%s" % VERSION,
+        formatter_class=argparse.RawTextHelpFormatter,
+        epilog="""
+The bind address may take one of the following formats:
+    unix://PATH         - Bind to unix domain socket at PATH
+    ws://ADDRESS:PORT   - Bind to websocket on ADDRESS:PORT
+    ADDRESS:PORT        - Bind to raw TCP socket on ADDRESS:PORT
 
-    parser.add_argument('-b', '--bind', default=DEFAULT_BIND, help='Bind address (default "%(default)s")')
-    parser.add_argument('-d', '--database', default='./hashserv.db', help='Database file (default "%(default)s")')
-    parser.add_argument('-l', '--log', default='WARNING', help='Set logging level')
-    parser.add_argument('-u', '--upstream', help='Upstream hashserv to pull hashes from')
-    parser.add_argument('-r', '--read-only', action='store_true', help='Disallow write operations from clients')
+To bind to all addresses, leave the ADDRESS empty, e.g. "--bind :8686" or
+"--bind ws://:8686". To bind to a specific IPv6 address, enclose the address in
+"[]", e.g. "--bind [::1]:8686" or "--bind ws://[::1]:8686"
+
+Note that the default Anonymous permissions are designed to not break existing
+server instances when upgrading, but are not particularly secure defaults. If
+you want to use authentication, it is recommended that you use "--anon-perms
+@read" to only give anonymous users read access, or "--anon-perms @none" to
+give un-authenticated users no access at all.
+
+Setting "--anon-perms @all" or "--anon-perms @user-admin" is not allowed, since
+this would allow anonymous users to manage all users accounts, which is a bad
+idea.
+
+If you are using user authentication, you should run your server in websockets
+mode with an SSL terminating load balancer in front of it (as this server does
+not implement SSL). Otherwise all usernames and passwords will be transmitted
+in the clear. When configured this way, clients can connect using a secure
+websocket, as in "wss://SERVER:PORT"
+
+The following permissions are supported by the server:
+
+    @none       - No permissions
+    @read       - The ability to read equivalent hashes from the server
+    @report     - The ability to report equivalent hashes to the server
+    @db-admin   - Manage the hash database(s). This includes cleaning the
+                  database, removing hashes, etc.
+    @user-admin - The ability to manage user accounts. This includes, creating
+                  users, deleting users, resetting login tokens, and assigning
+                  permissions.
+    @all        - All possible permissions, including any that may be added
+                  in the future
+        """,
+    )
+
+    parser.add_argument(
+        "-b",
+        "--bind",
+        default=os.environ.get("HASHSERVER_BIND", DEFAULT_BIND),
+        help='Bind address (default $HASHSERVER_BIND, "%(default)s")',
+    )
+    parser.add_argument(
+        "-d",
+        "--database",
+        default=os.environ.get("HASHSERVER_DB", "./hashserv.db"),
+        help='Database file (default $HASHSERVER_DB, "%(default)s")',
+    )
+    parser.add_argument(
+        "-l",
+        "--log",
+        default=os.environ.get("HASHSERVER_LOG_LEVEL", "WARNING"),
+        help='Set logging level (default $HASHSERVER_LOG_LEVEL, "%(default)s")',
+    )
+    parser.add_argument(
+        "-u",
+        "--upstream",
+        default=os.environ.get("HASHSERVER_UPSTREAM", None),
+        help="Upstream hashserv to pull hashes from ($HASHSERVER_UPSTREAM)",
+    )
+    parser.add_argument(
+        "-r",
+        "--read-only",
+        action="store_true",
+        help="Disallow write operations from clients ($HASHSERVER_READ_ONLY)",
+    )
+    parser.add_argument(
+        "--db-username",
+        default=os.environ.get("HASHSERVER_DB_USERNAME", None),
+        help="Database username ($HASHSERVER_DB_USERNAME)",
+    )
+    parser.add_argument(
+        "--db-password",
+        default=os.environ.get("HASHSERVER_DB_PASSWORD", None),
+        help="Database password ($HASHSERVER_DB_PASSWORD)",
+    )
+    parser.add_argument(
+        "--anon-perms",
+        metavar="PERM[,PERM[,...]]",
+        default=os.environ.get("HASHSERVER_ANON_PERMS", ",".join(DEFAULT_ANON_PERMS)),
+        help='Permissions to give anonymous users (default $HASHSERVER_ANON_PERMS, "%(default)s")',
+    )
+    parser.add_argument(
+        "--admin-user",
+        default=os.environ.get("HASHSERVER_ADMIN_USER", None),
+        help="Create default admin user with name ADMIN_USER ($HASHSERVER_ADMIN_USER)",
+    )
+    parser.add_argument(
+        "--admin-password",
+        default=os.environ.get("HASHSERVER_ADMIN_PASSWORD", None),
+        help="Create default admin user with password ADMIN_PASSWORD ($HASHSERVER_ADMIN_PASSWORD)",
+    )
 
     args = parser.parse_args()
 
-    logger = logging.getLogger('hashserv')
+    logger = logging.getLogger("hashserv")
 
     level = getattr(logging, args.log.upper(), None)
     if not isinstance(level, int):
-        raise ValueError('Invalid log level: %s' % args.log)
+        raise ValueError("Invalid log level: %s (Try ERROR/WARNING/INFO/DEBUG)" % args.log)
 
     logger.setLevel(level)
     console = logging.StreamHandler()
     console.setLevel(level)
     logger.addHandler(console)
 
-    server = hashserv.create_server(args.bind, args.database, upstream=args.upstream, read_only=args.read_only)
+    read_only = (os.environ.get("HASHSERVER_READ_ONLY", "0") == "1") or args.read_only
+    if "," in args.anon_perms:
+        anon_perms = args.anon_perms.split(",")
+    else:
+        anon_perms = args.anon_perms.split()
+
+    server = hashserv.create_server(
+        args.bind,
+        args.database,
+        upstream=args.upstream,
+        read_only=read_only,
+        db_username=args.db_username,
+        db_password=args.db_password,
+        anon_perms=anon_perms,
+        admin_username=args.admin_user,
+        admin_password=args.admin_password,
+    )
     server.serve_forever()
     return 0
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     try:
         ret = main()
     except Exception:
         ret = 1
         import traceback
+
         traceback.print_exc()
     sys.exit(ret)

bitbake/bin/bitbake-prserv

@@ -7,49 +7,77 @@
 
 import os
 import sys,logging
-import optparse
+import argparse
 import warnings
 warnings.simplefilter("default")
 
-sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)),'lib'))
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), "lib"))
 
 import prserv
 import prserv.serv
 
-__version__="1.0.0"
+VERSION = "1.1.0"
 
-PRHOST_DEFAULT='0.0.0.0'
+PRHOST_DEFAULT="0.0.0.0"
 PRPORT_DEFAULT=8585
 
 def main():
-    parser = optparse.OptionParser(
-        version="Bitbake PR Service Core version %s, %%prog version %s" % (prserv.__version__, __version__),
-        usage = "%prog < --start | --stop > [options]")
+    parser = argparse.ArgumentParser(
+        description="BitBake PR Server. Version=%s" % VERSION,
+        formatter_class=argparse.RawTextHelpFormatter)
 
-    parser.add_option("-f", "--file", help="database filename(default: prserv.sqlite3)", action="store",
-                      dest="dbfile", type="string", default="prserv.sqlite3")
-    parser.add_option("-l", "--log", help="log filename(default: prserv.log)", action="store",
-                      dest="logfile", type="string", default="prserv.log")
-    parser.add_option("--loglevel", help="logging level, i.e. CRITICAL, ERROR, WARNING, INFO, DEBUG",
-                      action = "store", type="string", dest="loglevel", default = "INFO")
-    parser.add_option("--start", help="start daemon",
-                      action="store_true", dest="start")
-    parser.add_option("--stop", help="stop daemon",
-                      action="store_true", dest="stop")
-    parser.add_option("--host", help="ip address to bind", action="store",
-                      dest="host", type="string", default=PRHOST_DEFAULT)
-    parser.add_option("--port", help="port number(default: 8585)", action="store",
-                      dest="port", type="int", default=PRPORT_DEFAULT)
-    parser.add_option("-r", "--read-only", help="open database in read-only mode",
-                      action="store_true")
+    parser.add_argument(
+        "-f",
+        "--file",
+        default="prserv.sqlite3",
+        help="database filename (default: prserv.sqlite3)",
+    )
+    parser.add_argument(
+        "-l",
+        "--log",
+        default="prserv.log",
+        help="log filename(default: prserv.log)",
+    )
+    parser.add_argument(
+        "--loglevel",
+        default="INFO",
+        help="logging level, i.e. CRITICAL, ERROR, WARNING, INFO, DEBUG",
+    )
+    parser.add_argument(
+        "--start",
+        action="store_true",
+        help="start daemon",
+    )
+    parser.add_argument(
+        "--stop",
+        action="store_true",
+        help="stop daemon",
+    )
+    parser.add_argument(
+        "--host",
+        help="ip address to bind",
+        default=PRHOST_DEFAULT,
+    )
+    parser.add_argument(
+        "--port",
+        type=int,
+        default=PRPORT_DEFAULT,
+        help="port number (default: 8585)",
+    )
+    parser.add_argument(
+        "-r",
+        "--read-only",
+        action="store_true",
+        help="open database in read-only mode",
+    )
 
-    options, args = parser.parse_args(sys.argv)
-    prserv.init_logger(os.path.abspath(options.logfile),options.loglevel)
+    args = parser.parse_args()
+    prserv.init_logger(os.path.abspath(args.log), args.loglevel)
 
-    if options.start:
-        ret=prserv.serv.start_daemon(options.dbfile, options.host, options.port,os.path.abspath(options.logfile), options.read_only)
-    elif options.stop:
-        ret=prserv.serv.stop_daemon(options.host, options.port)
+    if args.start:
+        ret=prserv.serv.start_daemon(args.file, args.host, args.port, os.path.abspath(args.log), args.read_only)
+    elif args.stop:
+        ret=prserv.serv.stop_daemon(args.host, args.port)
     else:
         ret=parser.print_help()
     return ret

bitbake/bin/bitbake-worker

@@ -183,7 +183,7 @@ def fork_off_task(cfg, data, databuilder, workerdata, extraconfigdata, runtask):
     if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not dry_run:
         fakeroot = True
         envvars = (runtask['fakerootenv'] or "").split()
-        for key, value in (var.split('=') for var in envvars):
+        for key, value in (var.split('=',1) for var in envvars):
             envbackup[key] = os.environ.get(key)
             os.environ[key] = value
             fakeenv[key] = value
@@ -195,7 +195,7 @@ def fork_off_task(cfg, data, databuilder, workerdata, extraconfigdata, runtask):
                         (fn, taskname, ', '.join(fakedirs)))
     else:
         envvars = (runtask['fakerootnoenv'] or "").split()
-        for key, value in (var.split('=') for var in envvars):
+        for key, value in (var.split('=',1) for var in envvars):
             envbackup[key] = os.environ.get(key)
             os.environ[key] = value
             fakeenv[key] = value
@@ -237,11 +237,13 @@ def fork_off_task(cfg, data, databuilder, workerdata, extraconfigdata, runtask):
             # Let SIGHUP exit as SIGTERM
             signal.signal(signal.SIGHUP, sigterm_handler)
 
-            # No stdin
-            newsi = os.open(os.devnull, os.O_RDWR)
-            os.dup2(newsi, sys.stdin.fileno())
+            # No stdin & stdout
+            # stdout is used as a status report channel and must not be used by child processes.
+            dumbio = os.open(os.devnull, os.O_RDWR)
+            os.dup2(dumbio, sys.stdin.fileno())
+            os.dup2(dumbio, sys.stdout.fileno())
 
-            if umask:
+            if umask is not None:
                 os.umask(umask)
 
             try:
@@ -305,6 +307,10 @@ def fork_off_task(cfg, data, databuilder, workerdata, extraconfigdata, runtask):
                 if not quieterrors:
                     logger.critical(traceback.format_exc())
                 os._exit(1)
+
+            sys.stdout.flush()
+            sys.stderr.flush()
+
             try:
                 if dry_run:
                     return 0
@@ -433,18 +439,30 @@ class BitbakeWorker(object):
                 while self.process_waitpid():
                     continue
 
-
     def handle_item(self, item, func):
-        if self.queue.startswith(b"<" + item + b">"):
-            index = self.queue.find(b"</" + item + b">")
-            while index != -1:
-                try:
-                    func(self.queue[(len(item) + 2):index])
-                except pickle.UnpicklingError:
-                    workerlog_write("Unable to unpickle data: %s\n" % ":".join("{:02x}".format(c) for c in self.queue))
-                    raise
-                self.queue = self.queue[(index + len(item) + 3):]
-                index = self.queue.find(b"</" + item + b">")
+        opening_tag = b"<" + item + b">"
+        if not self.queue.startswith(opening_tag):
+            return
+
+        tag_len = len(opening_tag)
+        if len(self.queue) < tag_len + 4:
+            # we need to receive more data
+            return
+        header = self.queue[tag_len:tag_len + 4]
+        payload_len = int.from_bytes(header, 'big')
+        # closing tag has length (tag_len + 1)
+        if len(self.queue) < tag_len * 2 + 1 + payload_len:
+            # we need to receive more data
+            return
+
+        index = self.queue.find(b"</" + item + b">")
+        if index != -1:
+            try:
+                func(self.queue[(tag_len + 4):index])
+            except pickle.UnpicklingError:
+                workerlog_write("Unable to unpickle data: %s\n" % ":".join("{:02x}".format(c) for c in self.queue))
+                raise
+            self.queue = self.queue[(index + len(b"</") + len(item) + len(b">")):]
 
     def handle_cookercfg(self, data):
         self.cookercfg = pickle.loads(data)

bitbake/bin/git-make-shallow

@@ -24,15 +24,17 @@ warnings.simplefilter("default")
 version = 1.0
 
 
+git_cmd = ['git', '-c', 'safe.bareRepository=all']
+
 def main():
     if sys.version_info < (3, 4, 0):
         sys.exit('Python 3.4 or greater is required')
 
-    git_dir = check_output(['git', 'rev-parse', '--git-dir']).rstrip()
+    git_dir = check_output(git_cmd + ['rev-parse', '--git-dir']).rstrip()
     shallow_file = os.path.join(git_dir, 'shallow')
     if os.path.exists(shallow_file):
         try:
-            check_output(['git', 'fetch', '--unshallow'])
+            check_output(git_cmd + ['fetch', '--unshallow'])
         except subprocess.CalledProcessError:
             try:
                 os.unlink(shallow_file)
@@ -41,21 +43,21 @@ def main():
                     raise
 
     args = process_args()
-    revs = check_output(['git', 'rev-list'] + args.revisions).splitlines()
+    revs = check_output(git_cmd + ['rev-list'] + args.revisions).splitlines()
 
     make_shallow(shallow_file, args.revisions, args.refs)
 
-    ref_revs = check_output(['git', 'rev-list'] + args.refs).splitlines()
+    ref_revs = check_output(git_cmd + ['rev-list'] + args.refs).splitlines()
     remaining_history = set(revs) & set(ref_revs)
     for rev in remaining_history:
-        if check_output(['git', 'rev-parse', '{}^@'.format(rev)]):
+        if check_output(git_cmd + ['rev-parse', '{}^@'.format(rev)]):
             sys.exit('Error: %s was not made shallow' % rev)
 
     filter_refs(args.refs)
 
     if args.shrink:
         shrink_repo(git_dir)
-        subprocess.check_call(['git', 'fsck', '--unreachable'])
+        subprocess.check_call(git_cmd + ['fsck', '--unreachable'])
 
 
 def process_args():
@@ -72,12 +74,12 @@ def process_args():
     args = parser.parse_args()
 
     if args.refs:
-        args.refs = check_output(['git', 'rev-parse', '--symbolic-full-name'] + args.refs).splitlines()
+        args.refs = check_output(git_cmd + ['rev-parse', '--symbolic-full-name'] + args.refs).splitlines()
     else:
         args.refs = get_all_refs(lambda r, t, tt: t == 'commit' or tt == 'commit')
 
     args.refs = list(filter(lambda r: not r.endswith('/HEAD'), args.refs))
-    args.revisions = check_output(['git', 'rev-parse'] + ['%s^{}' % i for i in args.revisions]).splitlines()
+    args.revisions = check_output(git_cmd + ['rev-parse'] + ['%s^{}' % i for i in args.revisions]).splitlines()
     return args
 
 
@@ -95,7 +97,7 @@ def make_shallow(shallow_file, revisions, refs):
 
 def get_all_refs(ref_filter=None):
     """Return all the existing refs in this repository, optionally filtering the refs."""
-    ref_output = check_output(['git', 'for-each-ref', '--format=%(refname)\t%(objecttype)\t%(*objecttype)'])
+    ref_output = check_output(git_cmd + ['for-each-ref', '--format=%(refname)\t%(objecttype)\t%(*objecttype)'])
     ref_split = [tuple(iter_extend(l.rsplit('\t'), 3)) for l in ref_output.splitlines()]
     if ref_filter:
         ref_split = (e for e in ref_split if ref_filter(*e))
@@ -113,7 +115,7 @@ def filter_refs(refs):
     all_refs = get_all_refs()
     to_remove = set(all_refs) - set(refs)
     if to_remove:
-        check_output(['xargs', '-0', '-n', '1', 'git', 'update-ref', '-d', '--no-deref'],
+        check_output(['xargs', '-0', '-n', '1'] + git_cmd + ['update-ref', '-d', '--no-deref'],
                      input=''.join(l + '\0' for l in to_remove))
 
 
@@ -126,7 +128,7 @@ def follow_history_intersections(revisions, refs):
         if rev in seen:
             continue
 
-        parents = check_output(['git', 'rev-parse', '%s^@' % rev]).splitlines()
+        parents = check_output(git_cmd + ['rev-parse', '%s^@' % rev]).splitlines()
 
         yield rev
         seen.add(rev)
@@ -134,12 +136,12 @@ def follow_history_intersections(revisions, refs):
         if not parents:
             continue
 
-        check_refs = check_output(['git', 'merge-base', '--independent'] + sorted(refs)).splitlines()
+        check_refs = check_output(git_cmd + ['merge-base', '--independent'] + sorted(refs)).splitlines()
         for parent in parents:
             for ref in check_refs:
                 print("Checking %s vs %s" % (parent, ref))
                 try:
-                    merge_base = check_output(['git', 'merge-base', parent, ref]).rstrip()
+                    merge_base = check_output(git_cmd + ['merge-base', parent, ref]).rstrip()
                 except subprocess.CalledProcessError:
                     continue
                 else:
@@ -159,14 +161,14 @@ def iter_except(func, exception, start=None):
 
 def shrink_repo(git_dir):
     """Shrink the newly shallow repository, removing the unreachable objects."""
-    subprocess.check_call(['git', 'reflog', 'expire', '--expire-unreachable=now', '--all'])
-    subprocess.check_call(['git', 'repack', '-ad'])
+    subprocess.check_call(git_cmd + ['reflog', 'expire', '--expire-unreachable=now', '--all'])
+    subprocess.check_call(git_cmd + ['repack', '-ad'])
     try:
         os.unlink(os.path.join(git_dir, 'objects', 'info', 'alternates'))
     except OSError as exc:
         if exc.errno != errno.ENOENT:
             raise
-    subprocess.check_call(['git', 'prune', '--expire', 'now'])
+    subprocess.check_call(git_cmd + ['prune', '--expire', 'now'])
 
 
 if __name__ == '__main__':

bitbake/bin/toaster

@@ -84,7 +84,7 @@ webserverStartAll()
     echo "Starting webserver..."
 
     $MANAGE runserver --noreload "$ADDR_PORT" \
-           </dev/null >>${BUILDDIR}/toaster_web.log 2>&1 \
+           </dev/null >>${TOASTER_LOGS_DIR}/web.log 2>&1 \
            & echo $! >${BUILDDIR}/.toastermain.pid
 
     sleep 1
@@ -181,6 +181,14 @@ WEBSERVER=1
 export TOASTER_BUILDSERVER=1
 ADDR_PORT="localhost:8000"
 TOASTERDIR=`dirname $BUILDDIR`
+# ${BUILDDIR}/toaster_logs/ became the default location for toaster logs
+# This is needed for implemented django-log-viewer: https://pypi.org/project/django-log-viewer/
+# If the directory does not exist, create it.
+TOASTER_LOGS_DIR="${BUILDDIR}/toaster_logs/"
+if [ ! -d $TOASTER_LOGS_DIR ]
+then
+    mkdir $TOASTER_LOGS_DIR
+fi
 unset CMD
 for param in $*; do
     case $param in
@@ -299,7 +307,7 @@ case $CMD in
         export BITBAKE_UI='toasterui'
         if [ $TOASTER_BUILDSERVER -eq 1 ] ; then
             $MANAGE runbuilds \
-               </dev/null >>${BUILDDIR}/toaster_runbuilds.log 2>&1 \
+               </dev/null >>${TOASTER_LOGS_DIR}/toaster_runbuilds.log 2>&1 \
                & echo $! >${BUILDDIR}/.runbuilds.pid
         else
             echo "Toaster build server not started."

bitbake/bin/toaster-eventreplay

@@ -30,79 +30,23 @@ sys.path.insert(0, join(dirname(dirname(abspath(__file__))), 'lib'))
 
 import bb.cooker
 from bb.ui import toasterui
-
-class EventPlayer:
-    """Emulate a connection to a bitbake server."""
-
-    def __init__(self, eventfile, variables):
-        self.eventfile = eventfile
-        self.variables = variables
-        self.eventmask = []
-
-    def waitEvent(self, _timeout):
-        """Read event from the file."""
-        line = self.eventfile.readline().strip()
-        if not line:
-            return
-        try:
-            event_str = json.loads(line)['vars'].encode('utf-8')
-            event = pickle.loads(codecs.decode(event_str, 'base64'))
-            event_name = "%s.%s" % (event.__module__, event.__class__.__name__)
-            if event_name not in self.eventmask:
-                return
-            return event
-        except ValueError as err:
-            print("Failed loading ", line)
-            raise err
-
-    def runCommand(self, command_line):
-        """Emulate running a command on the server."""
-        name = command_line[0]
-
-        if name == "getVariable":
-            var_name = command_line[1]
-            variable = self.variables.get(var_name)
-            if variable:
-                return variable['v'], None
-            return None, "Missing variable %s" % var_name
-
-        elif name == "getAllKeysWithFlags":
-            dump = {}
-            flaglist = command_line[1]
-            for key, val in self.variables.items():
-                try:
-                    if not key.startswith("__"):
-                        dump[key] = {
-                            'v': val['v'],
-                            'history' : val['history'],
-                        }
-                        for flag in flaglist:
-                            dump[key][flag] = val[flag]
-                except Exception as err:
-                    print(err)
-            return (dump, None)
-
-        elif name == 'setEventMask':
-            self.eventmask = command_line[-1]
-            return True, None
-
-        else:
-            raise Exception("Command %s not implemented" % command_line[0])
-
-    def getEventHandle(self):
-        """
-        This method is called by toasterui.
-        The return value is passed to self.runCommand but not used there.
-        """
-        pass
+from bb.ui import eventreplay
 
 def main(argv):
     with open(argv[-1]) as eventfile:
         # load variables from the first line
-        variables = json.loads(eventfile.readline().strip())['allvariables']
-
+        variables = None
+        while line := eventfile.readline().strip():
+            try:
+                variables = json.loads(line)['allvariables']
+                break
+            except (KeyError, json.JSONDecodeError):
+                continue
+        if not variables:
+            sys.exit("Cannot find allvariables entry in event log file %s" % argv[-1])
+        eventfile.seek(0)
         params = namedtuple('ConfigParams', ['observe_only'])(True)
-        player = EventPlayer(eventfile, variables)
+        player = eventreplay.EventPlayer(eventfile, variables)
 
         return toasterui.main(player, player, params)
 

bitbake/contrib/vim/syntax/bitbake.vim

@@ -63,13 +63,14 @@ syn region bbVarFlagFlag        matchgroup=bbArrayBrackets start="\[" end="\]\s*
 
 " Includes and requires
 syn keyword bbInclude           inherit include require contained 
-syn match bbIncludeRest         ".*$" contained contains=bbString,bbVarDeref
+syn match bbIncludeRest         ".*$" contained contains=bbString,bbVarDeref,bbVarPyValue
 syn match bbIncludeLine         "^\(inherit\|include\|require\)\s\+" contains=bbInclude nextgroup=bbIncludeRest
 
 " Add taks and similar
 syn keyword bbStatement         addtask deltask addhandler after before EXPORT_FUNCTIONS contained
-syn match bbStatementRest       ".*$" skipwhite contained contains=bbStatement
-syn match bbStatementLine       "^\(addtask\|deltask\|addhandler\|after\|before\|EXPORT_FUNCTIONS\)\s\+" contains=bbStatement nextgroup=bbStatementRest
+syn match bbStatementRest       /[^\\]*$/ skipwhite contained contains=bbStatement,bbVarDeref,bbVarPyValue
+syn region bbStatementRestCont  start=/.*\\$/ end=/^[^\\]*$/ contained contains=bbStatement,bbVarDeref,bbVarPyValue,bbContinue keepend
+syn match bbStatementLine       "^\(addtask\|deltask\|addhandler\|after\|before\|EXPORT_FUNCTIONS\)\s\+" contains=bbStatement nextgroup=bbStatementRest,bbStatementRestCont
 
 " OE Important Functions
 syn keyword bbOEFunctions       do_fetch do_unpack do_patch do_configure do_compile do_stage do_install do_package contained
@@ -122,6 +123,7 @@ hi def link bbPyFlag            Type
 hi def link bbPyDef             Statement
 hi def link bbStatement         Statement
 hi def link bbStatementRest     Identifier
+hi def link bbStatementRestCont Identifier
 hi def link bbOEFunctions       Special
 hi def link bbVarPyValue        PreProc
 hi def link bbOverrideOperator  Operator

bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.rst

@@ -586,10 +586,11 @@ or possibly those defined in the metadata/signature handler itself. The
 simplest parameter to pass is "none", which causes a set of signature
 information to be written out into ``STAMPS_DIR`` corresponding to the
 targets specified. The other currently available parameter is
-"printdiff", which causes BitBake to try to establish the closest
+"printdiff", which causes BitBake to try to establish the most recent
 signature match it can (e.g. in the sstate cache) and then run
-``bitbake-diffsigs`` over the matches to determine the stamps and delta
-where these two stamp trees diverge.
+compare the matched signatures to determine the stamps and delta
+where these two stamp trees diverge. This can be used to determine why
+tasks need to be re-run in situations where that is not expected.
 
 .. note::
 

bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables-context.rst

@@ -0,0 +1,91 @@
+.. SPDX-License-Identifier: CC-BY-2.5
+
+================
+Variable Context
+================
+
+|
+
+Variables might only have an impact or can be used in certain contexts. Some
+should only be used in global files like ``.conf``, while others are intended only
+for local files like ``.bb``. This chapter aims to describe some important variable
+contexts.
+
+.. _ref-varcontext-configuration:
+
+BitBake's own configuration
+===========================
+
+Variables starting with ``BB_`` usually configure the behaviour of BitBake itself.
+For example, one could configure:
+
+- System resources, like disk space to be used (:term:`BB_DISKMON_DIRS`),
+  or the number of tasks to be run in parallel by BitBake (:term:`BB_NUMBER_THREADS`).
+
+- How the fetchers shall behave, e.g., :term:`BB_FETCH_PREMIRRORONLY` is used
+  by BitBake to determine if BitBake's fetcher shall search only
+  :term:`PREMIRRORS` for files.
+
+Those variables are usually configured globally.
+
+BitBake configuration
+=====================
+
+There are variables:
+
+- Like :term:`B` or :term:`T`, that are used to specify directories used by
+  BitBake during the build of a particular recipe. Those variables are
+  specified in ``bitbake.conf``. Some, like :term:`B`, are quite often
+  overwritten in recipes.
+
+- Starting with ``FAKEROOT``, to configure how the ``fakeroot`` command is
+  handled. Those are usually set by ``bitbake.conf`` and might get adapted in a
+  ``bbclass``.
+
+- Detailing where BitBake will store and fetch information from, for
+  data reuse between build runs like :term:`CACHE`, :term:`DL_DIR` or
+  :term:`PERSISTENT_DIR`. Those are usually global.
+
+
+Layers and files
+================
+
+Variables starting with ``LAYER`` configure how BitBake handles layers.
+Additionally, variables starting with ``BB`` configure how layers and files are
+handled. For example:
+
+- :term:`LAYERDEPENDS` is used to configure on which layers a given layer
+  depends.
+
+- The configured layers are contained in :term:`BBLAYERS` and files in
+  :term:`BBFILES`.
+
+Those variables are often used in the files ``layer.conf`` and ``bblayers.conf``.
+
+Recipes and packages
+====================
+
+Variables handling recipes and packages can be split into:
+
+- :term:`PN`, :term:`PV` or :term:`PF` for example, contain information about
+  the name or revision of a recipe or package. Usually, the default set in
+  ``bitbake.conf`` is used, but those are from time to time overwritten in
+  recipes.
+
+- :term:`SUMMARY`, :term:`DESCRIPTION`, :term:`LICENSE` or :term:`HOMEPAGE`
+  contain the expected information and should be set specifically for every
+  recipe.
+
+- In recipes, variables are also used to control build and runtime
+  dependencies between recipes/packages with other recipes/packages. The
+  most common should be: :term:`PROVIDES`, :term:`RPROVIDES`, :term:`DEPENDS`,
+  and :term:`RDEPENDS`.
+
+- There are further variables starting with ``SRC`` that specify the sources in
+  a recipe like :term:`SRC_URI` or :term:`SRCDATE`. Those are also usually set
+  in recipes.
+
+- Which version or provider of a recipe should be given preference when
+  multiple recipes would provide the same item, is controlled by variables
+  starting with ``PREFERRED_``. Those are normally set in the configuration
+  files of a ``MACHINE`` or ``DISTRO``.

bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst

@@ -432,6 +432,15 @@ overview of their function and contents.
       ``ConfigParsed`` event can set the variable to trigger the re-parse.
       You must be careful to avoid recursive loops with this functionality.
 
+   :term:`BB_LOADFACTOR_MAX`
+      Setting this to a value will cause BitBake to check the system load
+      average before executing new tasks. If the load average is above the
+      the number of CPUs multipled by this factor, no new task will be started
+      unless there is no task executing. A value of "1.5" has been found to
+      work reasonably. This is helpful for systems which don't have pressure
+      regulation enabled, which is more granular. Pressure values take
+      precedence over loadfactor.
+
    :term:`BB_LOGCONFIG`
       Specifies the name of a config file that contains the user logging
       configuration. See

bitbake/doc/index.rst

@@ -13,6 +13,7 @@ BitBake User Manual
    bitbake-user-manual/bitbake-user-manual-intro
    bitbake-user-manual/bitbake-user-manual-execution
    bitbake-user-manual/bitbake-user-manual-metadata
+   bitbake-user-manual/bitbake-user-manual-ref-variables-context
    bitbake-user-manual/bitbake-user-manual-fetching
    bitbake-user-manual/bitbake-user-manual-ref-variables
    bitbake-user-manual/bitbake-user-manual-hello

bitbake/lib/bb/__init__.py

@@ -9,12 +9,19 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-__version__ = "2.6.0"
+__version__ = "2.8.0"
 
 import sys
 if sys.version_info < (3, 8, 0):
     raise RuntimeError("Sorry, python 3.8.0 or later is required for this version of bitbake")
 
+if sys.version_info < (3, 10, 0):
+    # With python 3.8 and 3.9, we see errors of "libgcc_s.so.1 must be installed for pthread_cancel to work"
+    # https://stackoverflow.com/questions/64797838/libgcc-s-so-1-must-be-installed-for-pthread-cancel-to-work
+    # https://bugs.ams1.psf.io/issue42888
+    # so ensure libgcc_s is loaded early on
+    import ctypes
+    libgcc_s = ctypes.CDLL('libgcc_s.so.1')
 
 class BBHandledException(Exception):
     """
@@ -29,6 +36,7 @@ class BBHandledException(Exception):
 
 import os
 import logging
+from collections import namedtuple
 
 
 class NullHandler(logging.Handler):
@@ -220,3 +228,14 @@ def deprecate_import(current, modulename, fromlist, renames = None):
 
         setattr(sys.modules[current], newname, newobj)
 
+TaskData = namedtuple("TaskData", [
+    "pn",
+    "taskname",
+    "fn",
+    "deps",
+    "provides",
+    "taskhash",
+    "unihash",
+    "hashfn",
+    "taskhash_deps",
+])

bitbake/lib/bb/asyncrpc/__init__.py

@@ -4,30 +4,13 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-import itertools
-import json
 
-# The Python async server defaults to a 64K receive buffer, so we hardcode our
-# maximum chunk size. It would be better if the client and server reported to
-# each other what the maximum chunk sizes were, but that will slow down the
-# connection setup with a round trip delay so I'd rather not do that unless it
-# is necessary
-DEFAULT_MAX_CHUNK = 32 * 1024
-
-
-def chunkify(msg, max_chunk):
-    if len(msg) < max_chunk - 1:
-        yield ''.join((msg, "\n"))
-    else:
-        yield ''.join((json.dumps({
-                'chunk-stream': None
-            }), "\n"))
-
-        args = [iter(msg)] * (max_chunk - 1)
-        for m in map(''.join, itertools.zip_longest(*args, fillvalue='')):
-            yield ''.join(itertools.chain(m, "\n"))
-        yield "\n"
-
-
-from .client import AsyncClient, Client
-from .serv import AsyncServer, AsyncServerConnection, ClientError, ServerError
+from .client import AsyncClient, Client, ClientPool
+from .serv import AsyncServer, AsyncServerConnection
+from .connection import DEFAULT_MAX_CHUNK
+from .exceptions import (
+    ClientError,
+    ServerError,
+    ConnectionClosedError,
+    InvokeError,
+)

bitbake/lib/bb/asyncrpc/client.py

@@ -10,22 +10,59 @@ import json
 import os
 import socket
 import sys
-from . import chunkify, DEFAULT_MAX_CHUNK
+import re
+import contextlib
+from threading import Thread
+from .connection import StreamConnection, WebsocketConnection, DEFAULT_MAX_CHUNK
+from .exceptions import ConnectionClosedError, InvokeError
 
+UNIX_PREFIX = "unix://"
+WS_PREFIX = "ws://"
+WSS_PREFIX = "wss://"
+
+ADDR_TYPE_UNIX = 0
+ADDR_TYPE_TCP = 1
+ADDR_TYPE_WS = 2
+
+def parse_address(addr):
+    if addr.startswith(UNIX_PREFIX):
+        return (ADDR_TYPE_UNIX, (addr[len(UNIX_PREFIX) :],))
+    elif addr.startswith(WS_PREFIX) or addr.startswith(WSS_PREFIX):
+        return (ADDR_TYPE_WS, (addr,))
+    else:
+        m = re.match(r"\[(?P<host>[^\]]*)\]:(?P<port>\d+)$", addr)
+        if m is not None:
+            host = m.group("host")
+            port = m.group("port")
+        else:
+            host, port = addr.split(":")
+
+        return (ADDR_TYPE_TCP, (host, int(port)))
 
 class AsyncClient(object):
-    def __init__(self, proto_name, proto_version, logger, timeout=30):
-        self.reader = None
-        self.writer = None
+    def __init__(
+        self,
+        proto_name,
+        proto_version,
+        logger,
+        timeout=30,
+        server_headers=False,
+        headers={},
+    ):
+        self.socket = None
         self.max_chunk = DEFAULT_MAX_CHUNK
         self.proto_name = proto_name
         self.proto_version = proto_version
         self.logger = logger
         self.timeout = timeout
+        self.needs_server_headers = server_headers
+        self.server_headers = {}
+        self.headers = headers
 
     async def connect_tcp(self, address, port):
         async def connect_sock():
-            return await asyncio.open_connection(address, port)
+            reader, writer = await asyncio.open_connection(address, port)
+            return StreamConnection(reader, writer, self.timeout, self.max_chunk)
 
         self._connect_sock = connect_sock
 
@@ -40,27 +77,59 @@ class AsyncClient(object):
                 sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0)
                 sock.connect(os.path.basename(path))
             finally:
-               os.chdir(cwd)
-            return await asyncio.open_unix_connection(sock=sock)
+                os.chdir(cwd)
+            reader, writer = await asyncio.open_unix_connection(sock=sock)
+            return StreamConnection(reader, writer, self.timeout, self.max_chunk)
+
+        self._connect_sock = connect_sock
+
+    async def connect_websocket(self, uri):
+        import websockets
+
+        async def connect_sock():
+            websocket = await websockets.connect(uri, ping_interval=None)
+            return WebsocketConnection(websocket, self.timeout)
 
         self._connect_sock = connect_sock
 
     async def setup_connection(self):
-        s = '%s %s\n\n' % (self.proto_name, self.proto_version)
-        self.writer.write(s.encode("utf-8"))
-        await self.writer.drain()
+        # Send headers
+        await self.socket.send("%s %s" % (self.proto_name, self.proto_version))
+        await self.socket.send(
+            "needs-headers: %s" % ("true" if self.needs_server_headers else "false")
+        )
+        for k, v in self.headers.items():
+            await self.socket.send("%s: %s" % (k, v))
+
+        # End of headers
+        await self.socket.send("")
+
+        self.server_headers = {}
+        if self.needs_server_headers:
+            while True:
+                line = await self.socket.recv()
+                if not line:
+                    # End headers
+                    break
+                tag, value = line.split(":", 1)
+                self.server_headers[tag.lower()] = value.strip()
+
+    async def get_header(self, tag, default):
+        await self.connect()
+        return self.server_headers.get(tag, default)
 
     async def connect(self):
-        if self.reader is None or self.writer is None:
-            (self.reader, self.writer) = await self._connect_sock()
+        if self.socket is None:
+            self.socket = await self._connect_sock()
             await self.setup_connection()
 
-    async def close(self):
-        self.reader = None
+    async def disconnect(self):
+        if self.socket is not None:
+            await self.socket.close()
+            self.socket = None
 
-        if self.writer is not None:
-            self.writer.close()
-            self.writer = None
+    async def close(self):
+        await self.disconnect()
 
     async def _send_wrapper(self, proc):
         count = 0
@@ -71,6 +140,7 @@ class AsyncClient(object):
             except (
                 OSError,
                 ConnectionError,
+                ConnectionClosedError,
                 json.JSONDecodeError,
                 UnicodeDecodeError,
             ) as e:
@@ -82,49 +152,27 @@ class AsyncClient(object):
                 await self.close()
                 count += 1
 
-    async def send_message(self, msg):
-        async def get_line():
-            try:
-                line = await asyncio.wait_for(self.reader.readline(), self.timeout)
-            except asyncio.TimeoutError:
-                raise ConnectionError("Timed out waiting for server")
-
-            if not line:
-                raise ConnectionError("Connection closed")
-
-            line = line.decode("utf-8")
-
-            if not line.endswith("\n"):
-                raise ConnectionError("Bad message %r" % (line))
-
-            return line
+    def check_invoke_error(self, msg):
+        if isinstance(msg, dict) and "invoke-error" in msg:
+            raise InvokeError(msg["invoke-error"]["message"])
 
+    async def invoke(self, msg):
         async def proc():
-            for c in chunkify(json.dumps(msg), self.max_chunk):
-                self.writer.write(c.encode("utf-8"))
-            await self.writer.drain()
+            await self.socket.send_message(msg)
+            return await self.socket.recv_message()
 
-            l = await get_line()
-
-            m = json.loads(l)
-            if m and "chunk-stream" in m:
-                lines = []
-                while True:
-                    l = (await get_line()).rstrip("\n")
-                    if not l:
-                        break
-                    lines.append(l)
-
-                m = json.loads("".join(lines))
-
-            return m
-
-        return await self._send_wrapper(proc)
+        result = await self._send_wrapper(proc)
+        self.check_invoke_error(result)
+        return result
 
     async def ping(self):
-        return await self.send_message(
-            {'ping': {}}
-        )
+        return await self.invoke({"ping": {}})
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, exc_type, exc_value, traceback):
+        await self.close()
 
 
 class Client(object):
@@ -142,7 +190,7 @@ class Client(object):
         # required (but harmless) with it.
         asyncio.set_event_loop(self.loop)
 
-        self._add_methods('connect_tcp', 'ping')
+        self._add_methods("connect_tcp", "ping")
 
     @abc.abstractmethod
     def _get_async_client(self):
@@ -171,8 +219,95 @@ class Client(object):
     def max_chunk(self, value):
         self.client.max_chunk = value
 
-    def close(self):
+    def disconnect(self):
         self.loop.run_until_complete(self.client.close())
-        if sys.version_info >= (3, 6):
+
+    def close(self):
+        if self.loop:
+            self.loop.run_until_complete(self.client.close())
+            if sys.version_info >= (3, 6):
+                self.loop.run_until_complete(self.loop.shutdown_asyncgens())
+            self.loop.close()
+        self.loop = None
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.close()
+        return False
+
+
+class ClientPool(object):
+    def __init__(self, max_clients):
+        self.avail_clients = []
+        self.num_clients = 0
+        self.max_clients = max_clients
+        self.loop = None
+        self.client_condition = None
+
+    @abc.abstractmethod
+    async def _new_client(self):
+        raise NotImplementedError("Must be implemented in derived class")
+
+    def close(self):
+        if self.client_condition:
+            self.client_condition = None
+
+        if self.loop:
+            self.loop.run_until_complete(self.__close_clients())
             self.loop.run_until_complete(self.loop.shutdown_asyncgens())
-        self.loop.close()
+            self.loop.close()
+            self.loop = None
+
+    def run_tasks(self, tasks):
+        if not self.loop:
+            self.loop = asyncio.new_event_loop()
+
+        thread = Thread(target=self.__thread_main, args=(tasks,))
+        thread.start()
+        thread.join()
+
+    @contextlib.asynccontextmanager
+    async def get_client(self):
+        async with self.client_condition:
+            if self.avail_clients:
+                client = self.avail_clients.pop()
+            elif self.num_clients < self.max_clients:
+                self.num_clients += 1
+                client = await self._new_client()
+            else:
+                while not self.avail_clients:
+                    await self.client_condition.wait()
+                client = self.avail_clients.pop()
+
+        try:
+            yield client
+        finally:
+            async with self.client_condition:
+                self.avail_clients.append(client)
+                self.client_condition.notify()
+
+    def __thread_main(self, tasks):
+        async def process_task(task):
+            async with self.get_client() as client:
+                await task(client)
+
+        asyncio.set_event_loop(self.loop)
+        if not self.client_condition:
+            self.client_condition = asyncio.Condition()
+        tasks = [process_task(t) for t in tasks]
+        self.loop.run_until_complete(asyncio.gather(*tasks))
+
+    async def __close_clients(self):
+        for c in self.avail_clients:
+            await c.close()
+        self.avail_clients = []
+        self.num_clients = 0
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.close()
+        return False

bitbake/lib/bb/asyncrpc/connection.py

@@ -0,0 +1,146 @@
+#
+# Copyright BitBake Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import asyncio
+import itertools
+import json
+from datetime import datetime
+from .exceptions import ClientError, ConnectionClosedError
+
+
+# The Python async server defaults to a 64K receive buffer, so we hardcode our
+# maximum chunk size. It would be better if the client and server reported to
+# each other what the maximum chunk sizes were, but that will slow down the
+# connection setup with a round trip delay so I'd rather not do that unless it
+# is necessary
+DEFAULT_MAX_CHUNK = 32 * 1024
+
+
+def chunkify(msg, max_chunk):
+    if len(msg) < max_chunk - 1:
+        yield "".join((msg, "\n"))
+    else:
+        yield "".join((json.dumps({"chunk-stream": None}), "\n"))
+
+        args = [iter(msg)] * (max_chunk - 1)
+        for m in map("".join, itertools.zip_longest(*args, fillvalue="")):
+            yield "".join(itertools.chain(m, "\n"))
+        yield "\n"
+
+
+def json_serialize(obj):
+    if isinstance(obj, datetime):
+        return obj.isoformat()
+    raise TypeError("Type %s not serializeable" % type(obj))
+
+
+class StreamConnection(object):
+    def __init__(self, reader, writer, timeout, max_chunk=DEFAULT_MAX_CHUNK):
+        self.reader = reader
+        self.writer = writer
+        self.timeout = timeout
+        self.max_chunk = max_chunk
+
+    @property
+    def address(self):
+        return self.writer.get_extra_info("peername")
+
+    async def send_message(self, msg):
+        for c in chunkify(json.dumps(msg, default=json_serialize), self.max_chunk):
+            self.writer.write(c.encode("utf-8"))
+        await self.writer.drain()
+
+    async def recv_message(self):
+        l = await self.recv()
+
+        m = json.loads(l)
+        if not m:
+            return m
+
+        if "chunk-stream" in m:
+            lines = []
+            while True:
+                l = await self.recv()
+                if not l:
+                    break
+                lines.append(l)
+
+            m = json.loads("".join(lines))
+
+        return m
+
+    async def send(self, msg):
+        self.writer.write(("%s\n" % msg).encode("utf-8"))
+        await self.writer.drain()
+
+    async def recv(self):
+        if self.timeout < 0:
+            line = await self.reader.readline()
+        else:
+            try:
+                line = await asyncio.wait_for(self.reader.readline(), self.timeout)
+            except asyncio.TimeoutError:
+                raise ConnectionError("Timed out waiting for data")
+
+        if not line:
+            raise ConnectionClosedError("Connection closed")
+
+        line = line.decode("utf-8")
+
+        if not line.endswith("\n"):
+            raise ConnectionError("Bad message %r" % (line))
+
+        return line.rstrip()
+
+    async def close(self):
+        self.reader = None
+        if self.writer is not None:
+            self.writer.close()
+            self.writer = None
+
+
+class WebsocketConnection(object):
+    def __init__(self, socket, timeout):
+        self.socket = socket
+        self.timeout = timeout
+
+    @property
+    def address(self):
+        return ":".join(str(s) for s in self.socket.remote_address)
+
+    async def send_message(self, msg):
+        await self.send(json.dumps(msg, default=json_serialize))
+
+    async def recv_message(self):
+        m = await self.recv()
+        return json.loads(m)
+
+    async def send(self, msg):
+        import websockets.exceptions
+
+        try:
+            await self.socket.send(msg)
+        except websockets.exceptions.ConnectionClosed:
+            raise ConnectionClosedError("Connection closed")
+
+    async def recv(self):
+        import websockets.exceptions
+
+        try:
+            if self.timeout < 0:
+                return await self.socket.recv()
+
+            try:
+                return await asyncio.wait_for(self.socket.recv(), self.timeout)
+            except asyncio.TimeoutError:
+                raise ConnectionError("Timed out waiting for data")
+        except websockets.exceptions.ConnectionClosed:
+            raise ConnectionClosedError("Connection closed")
+
+    async def close(self):
+        if self.socket is not None:
+            await self.socket.close()
+            self.socket = None

bitbake/lib/bb/asyncrpc/exceptions.py

@@ -0,0 +1,21 @@
+#
+# Copyright BitBake Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+
+class ClientError(Exception):
+    pass
+
+
+class InvokeError(Exception):
+    pass
+
+
+class ServerError(Exception):
+    pass
+
+
+class ConnectionClosedError(Exception):
+    pass

bitbake/lib/bb/asyncrpc/serv.py

@@ -12,241 +12,333 @@ import signal
 import socket
 import sys
 import multiprocessing
-from . import chunkify, DEFAULT_MAX_CHUNK
+import logging
+from .connection import StreamConnection, WebsocketConnection
+from .exceptions import ClientError, ServerError, ConnectionClosedError, InvokeError
 
 
-class ClientError(Exception):
-    pass
-
-
-class ServerError(Exception):
-    pass
+class ClientLoggerAdapter(logging.LoggerAdapter):
+    def process(self, msg, kwargs):
+        return f"[Client {self.extra['address']}] {msg}", kwargs
 
 
 class AsyncServerConnection(object):
-    def __init__(self, reader, writer, proto_name, logger):
-        self.reader = reader
-        self.writer = writer
+    # If a handler returns this object (e.g. `return self.NO_RESPONSE`), no
+    # return message will be automatically be sent back to the client
+    NO_RESPONSE = object()
+
+    def __init__(self, socket, proto_name, logger):
+        self.socket = socket
         self.proto_name = proto_name
-        self.max_chunk = DEFAULT_MAX_CHUNK
         self.handlers = {
-            'chunk-stream': self.handle_chunk,
-            'ping': self.handle_ping,
+            "ping": self.handle_ping,
         }
-        self.logger = logger
+        self.logger = ClientLoggerAdapter(
+            logger,
+            {
+                "address": socket.address,
+            },
+        )
+        self.client_headers = {}
+
+    async def close(self):
+        await self.socket.close()
+
+    async def handle_headers(self, headers):
+        return {}
 
     async def process_requests(self):
         try:
-            self.addr = self.writer.get_extra_info('peername')
-            self.logger.debug('Client %r connected' % (self.addr,))
+            self.logger.info("Client %r connected" % (self.socket.address,))
 
             # Read protocol and version
-            client_protocol = await self.reader.readline()
+            client_protocol = await self.socket.recv()
             if not client_protocol:
                 return
 
-            (client_proto_name, client_proto_version) = client_protocol.decode('utf-8').rstrip().split()
+            (client_proto_name, client_proto_version) = client_protocol.split()
             if client_proto_name != self.proto_name:
-                self.logger.debug('Rejecting invalid protocol %s' % (self.proto_name))
+                self.logger.debug("Rejecting invalid protocol %s" % (self.proto_name))
                 return
 
-            self.proto_version = tuple(int(v) for v in client_proto_version.split('.'))
+            self.proto_version = tuple(int(v) for v in client_proto_version.split("."))
             if not self.validate_proto_version():
-                self.logger.debug('Rejecting invalid protocol version %s' % (client_proto_version))
+                self.logger.debug(
+                    "Rejecting invalid protocol version %s" % (client_proto_version)
+                )
                 return
 
-            # Read headers. Currently, no headers are implemented, so look for
-            # an empty line to signal the end of the headers
+            # Read headers
+            self.client_headers = {}
             while True:
-                line = await self.reader.readline()
-                if not line:
-                    return
-
-                line = line.decode('utf-8').rstrip()
-                if not line:
+                header = await self.socket.recv()
+                if not header:
+                    # Empty line. End of headers
                     break
+                tag, value = header.split(":", 1)
+                self.client_headers[tag.lower()] = value.strip()
+
+            if self.client_headers.get("needs-headers", "false") == "true":
+                for k, v in (await self.handle_headers(self.client_headers)).items():
+                    await self.socket.send("%s: %s" % (k, v))
+                await self.socket.send("")
 
             # Handle messages
             while True:
-                d = await self.read_message()
+                d = await self.socket.recv_message()
                 if d is None:
                     break
-                await self.dispatch_message(d)
-                await self.writer.drain()
-        except ClientError as e:
+                try:
+                    response = await self.dispatch_message(d)
+                except InvokeError as e:
+                    await self.socket.send_message(
+                        {"invoke-error": {"message": str(e)}}
+                    )
+                    break
+
+                if response is not self.NO_RESPONSE:
+                    await self.socket.send_message(response)
+
+        except ConnectionClosedError as e:
+            self.logger.info(str(e))
+        except (ClientError, ConnectionError) as e:
             self.logger.error(str(e))
         finally:
-            self.writer.close()
+            await self.close()
 
     async def dispatch_message(self, msg):
         for k in self.handlers.keys():
             if k in msg:
-                self.logger.debug('Handling %s' % k)
-                await self.handlers[k](msg[k])
-                return
+                self.logger.debug("Handling %s" % k)
+                return await self.handlers[k](msg[k])
 
         raise ClientError("Unrecognized command %r" % msg)
 
-    def write_message(self, msg):
-        for c in chunkify(json.dumps(msg), self.max_chunk):
-            self.writer.write(c.encode('utf-8'))
-
-    async def read_message(self):
-        l = await self.reader.readline()
-        if not l:
-            return None
-
-        try:
-            message = l.decode('utf-8')
-
-            if not message.endswith('\n'):
-                return None
-
-            return json.loads(message)
-        except (json.JSONDecodeError, UnicodeDecodeError) as e:
-            self.logger.error('Bad message from client: %r' % message)
-            raise e
-
-    async def handle_chunk(self, request):
-        lines = []
-        try:
-            while True:
-                l = await self.reader.readline()
-                l = l.rstrip(b"\n").decode("utf-8")
-                if not l:
-                    break
-                lines.append(l)
-
-            msg = json.loads(''.join(lines))
-        except (json.JSONDecodeError, UnicodeDecodeError) as e:
-            self.logger.error('Bad message from client: %r' % lines)
-            raise e
-
-        if 'chunk-stream' in msg:
-            raise ClientError("Nested chunks are not allowed")
-
-        await self.dispatch_message(msg)
-
     async def handle_ping(self, request):
-        response = {'alive': True}
-        self.write_message(response)
+        return {"alive": True}
+
+
+class StreamServer(object):
+    def __init__(self, handler, logger):
+        self.handler = handler
+        self.logger = logger
+        self.closed = False
+
+    async def handle_stream_client(self, reader, writer):
+        # writer.transport.set_write_buffer_limits(0)
+        socket = StreamConnection(reader, writer, -1)
+        if self.closed:
+            await socket.close()
+            return
+
+        await self.handler(socket)
+
+    async def stop(self):
+        self.closed = True
+
+
+class TCPStreamServer(StreamServer):
+    def __init__(self, host, port, handler, logger):
+        super().__init__(handler, logger)
+        self.host = host
+        self.port = port
+
+    def start(self, loop):
+        self.server = loop.run_until_complete(
+            asyncio.start_server(self.handle_stream_client, self.host, self.port)
+        )
+
+        for s in self.server.sockets:
+            self.logger.debug("Listening on %r" % (s.getsockname(),))
+            # Newer python does this automatically. Do it manually here for
+            # maximum compatibility
+            s.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1)
+            s.setsockopt(socket.SOL_TCP, socket.TCP_QUICKACK, 1)
+
+            # Enable keep alives. This prevents broken client connections
+            # from persisting on the server for long periods of time.
+            s.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
+            s.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 30)
+            s.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, 15)
+            s.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, 4)
+
+        name = self.server.sockets[0].getsockname()
+        if self.server.sockets[0].family == socket.AF_INET6:
+            self.address = "[%s]:%d" % (name[0], name[1])
+        else:
+            self.address = "%s:%d" % (name[0], name[1])
+
+        return [self.server.wait_closed()]
+
+    async def stop(self):
+        await super().stop()
+        self.server.close()
+
+    def cleanup(self):
+        pass
+
+
+class UnixStreamServer(StreamServer):
+    def __init__(self, path, handler, logger):
+        super().__init__(handler, logger)
+        self.path = path
+
+    def start(self, loop):
+        cwd = os.getcwd()
+        try:
+            # Work around path length limits in AF_UNIX
+            os.chdir(os.path.dirname(self.path))
+            self.server = loop.run_until_complete(
+                asyncio.start_unix_server(
+                    self.handle_stream_client, os.path.basename(self.path)
+                )
+            )
+        finally:
+            os.chdir(cwd)
+
+        self.logger.debug("Listening on %r" % self.path)
+        self.address = "unix://%s" % os.path.abspath(self.path)
+        return [self.server.wait_closed()]
+
+    async def stop(self):
+        await super().stop()
+        self.server.close()
+
+    def cleanup(self):
+        os.unlink(self.path)
+
+
+class WebsocketsServer(object):
+    def __init__(self, host, port, handler, logger):
+        self.host = host
+        self.port = port
+        self.handler = handler
+        self.logger = logger
+
+    def start(self, loop):
+        import websockets.server
+
+        self.server = loop.run_until_complete(
+            websockets.server.serve(
+                self.client_handler,
+                self.host,
+                self.port,
+                ping_interval=None,
+            )
+        )
+
+        for s in self.server.sockets:
+            self.logger.debug("Listening on %r" % (s.getsockname(),))
+
+            # Enable keep alives. This prevents broken client connections
+            # from persisting on the server for long periods of time.
+            s.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
+            s.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 30)
+            s.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, 15)
+            s.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, 4)
+
+        name = self.server.sockets[0].getsockname()
+        if self.server.sockets[0].family == socket.AF_INET6:
+            self.address = "ws://[%s]:%d" % (name[0], name[1])
+        else:
+            self.address = "ws://%s:%d" % (name[0], name[1])
+
+        return [self.server.wait_closed()]
+
+    async def stop(self):
+        self.server.close()
+
+    def cleanup(self):
+        pass
+
+    async def client_handler(self, websocket):
+        socket = WebsocketConnection(websocket, -1)
+        await self.handler(socket)
 
 
 class AsyncServer(object):
     def __init__(self, logger):
-        self._cleanup_socket = None
         self.logger = logger
-        self.start = None
-        self.address = None
         self.loop = None
+        self.run_tasks = []
 
     def start_tcp_server(self, host, port):
-        def start_tcp():
-            self.server = self.loop.run_until_complete(
-                asyncio.start_server(self.handle_client, host, port)
-            )
-
-            for s in self.server.sockets:
-                self.logger.debug('Listening on %r' % (s.getsockname(),))
-                # Newer python does this automatically. Do it manually here for
-                # maximum compatibility
-                s.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1)
-                s.setsockopt(socket.SOL_TCP, socket.TCP_QUICKACK, 1)
-
-                # Enable keep alives. This prevents broken client connections
-                # from persisting on the server for long periods of time.
-                s.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
-                s.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 30)
-                s.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, 15)
-                s.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, 4)
-
-            name = self.server.sockets[0].getsockname()
-            if self.server.sockets[0].family == socket.AF_INET6:
-                self.address = "[%s]:%d" % (name[0], name[1])
-            else:
-                self.address = "%s:%d" % (name[0], name[1])
-
-        self.start = start_tcp
+        self.server = TCPStreamServer(host, port, self._client_handler, self.logger)
 
     def start_unix_server(self, path):
-        def cleanup():
-            os.unlink(path)
+        self.server = UnixStreamServer(path, self._client_handler, self.logger)
 
-        def start_unix():
-            cwd = os.getcwd()
-            try:
-                # Work around path length limits in AF_UNIX
-                os.chdir(os.path.dirname(path))
-                self.server = self.loop.run_until_complete(
-                    asyncio.start_unix_server(self.handle_client, os.path.basename(path))
-                )
-            finally:
-                os.chdir(cwd)
+    def start_websocket_server(self, host, port):
+        self.server = WebsocketsServer(host, port, self._client_handler, self.logger)
 
-            self.logger.debug('Listening on %r' % path)
-
-            self._cleanup_socket = cleanup
-            self.address = "unix://%s" % os.path.abspath(path)
-
-        self.start = start_unix
-
-    @abc.abstractmethod
-    def accept_client(self, reader, writer):
-        pass
-
-    async def handle_client(self, reader, writer):
-        # writer.transport.set_write_buffer_limits(0)
+    async def _client_handler(self, socket):
+        address = socket.address
         try:
-            client = self.accept_client(reader, writer)
+            client = self.accept_client(socket)
             await client.process_requests()
         except Exception as e:
             import traceback
-            self.logger.error('Error from client: %s' % str(e), exc_info=True)
-            traceback.print_exc()
-            writer.close()
-        self.logger.debug('Client disconnected')
 
-    def run_loop_forever(self):
-        try:
-            self.loop.run_forever()
-        except KeyboardInterrupt:
-            pass
+            self.logger.error(
+                "Error from client %s: %s" % (address, str(e)), exc_info=True
+            )
+            traceback.print_exc()
+        finally:
+            self.logger.debug("Client %s disconnected", address)
+            await socket.close()
+
+    @abc.abstractmethod
+    def accept_client(self, socket):
+        pass
+
+    async def stop(self):
+        self.logger.debug("Stopping server")
+        await self.server.stop()
+
+    def start(self):
+        tasks = self.server.start(self.loop)
+        self.address = self.server.address
+        return tasks
 
     def signal_handler(self):
         self.logger.debug("Got exit signal")
-        self.loop.stop()
+        self.loop.create_task(self.stop())
 
-    def _serve_forever(self):
+    def _serve_forever(self, tasks):
         try:
             self.loop.add_signal_handler(signal.SIGTERM, self.signal_handler)
+            self.loop.add_signal_handler(signal.SIGINT, self.signal_handler)
+            self.loop.add_signal_handler(signal.SIGQUIT, self.signal_handler)
             signal.pthread_sigmask(signal.SIG_UNBLOCK, [signal.SIGTERM])
 
-            self.run_loop_forever()
-            self.server.close()
+            self.loop.run_until_complete(asyncio.gather(*tasks))
 
-            self.loop.run_until_complete(self.server.wait_closed())
-            self.logger.debug('Server shutting down')
+            self.logger.debug("Server shutting down")
         finally:
-            if self._cleanup_socket is not None:
-                self._cleanup_socket()
+            self.server.cleanup()
 
     def serve_forever(self):
         """
         Serve requests in the current process
         """
+        self._create_loop()
+        tasks = self.start()
+        self._serve_forever(tasks)
+        self.loop.close()
+
+    def _create_loop(self):
         # Create loop and override any loop that may have existed in
         # a parent process.  It is possible that the usecases of
         # serve_forever might be constrained enough to allow using
         # get_event_loop here, but better safe than sorry for now.
         self.loop = asyncio.new_event_loop()
         asyncio.set_event_loop(self.loop)
-        self.start()
-        self._serve_forever()
 
-    def serve_as_process(self, *, prefunc=None, args=()):
+    def serve_as_process(self, *, prefunc=None, args=(), log_level=None):
         """
         Serve requests in a child process
         """
+
         def run(queue):
             # Create loop and override any loop that may have existed
             # in a parent process.  Without doing this and instead
@@ -259,18 +351,22 @@ class AsyncServer(object):
             # more general, though, as any potential use of asyncio in
             # Cooker could create a loop that needs to replaced in this
             # new process.
-            self.loop = asyncio.new_event_loop()
-            asyncio.set_event_loop(self.loop)
+            self._create_loop()
             try:
-                self.start()
+                self.address = None
+                tasks = self.start()
             finally:
+                # Always put the server address to wake up the parent task
                 queue.put(self.address)
                 queue.close()
 
             if prefunc is not None:
                 prefunc(self, *args)
 
-            self._serve_forever()
+            if log_level is not None:
+                self.logger.setLevel(log_level)
+
+            self._serve_forever(tasks)
 
             if sys.version_info >= (3, 6):
                 self.loop.run_until_complete(self.loop.shutdown_asyncgens())

bitbake/lib/bb/cache.py

@@ -344,9 +344,7 @@ def virtualfn2realfn(virtualfn):
     """
     mc = ""
     if virtualfn.startswith('mc:') and virtualfn.count(':') >= 2:
-        elems = virtualfn.split(':')
-        mc = elems[1]
-        virtualfn = ":".join(elems[2:])
+        (_, mc, virtualfn) = virtualfn.split(':', 2)
 
     fn = virtualfn
     cls = ""
@@ -369,7 +367,7 @@ def realfn2virtual(realfn, cls, mc):
 
 def variant2virtual(realfn, variant):
     """
-    Convert a real filename + the associated subclass keyword to a virtual filename
+    Convert a real filename + a variant to a virtual filename
     """
     if variant == "":
         return realfn

bitbake/lib/bb/codeparser.py

@@ -62,6 +62,7 @@ def check_indent(codestr):
 modulecode_deps = {}
 
 def add_module_functions(fn, functions, namespace):
+    import os
     fstat = os.stat(fn)
     fixedhash = fn + ":" + str(fstat.st_size) +  ":" + str(fstat.st_mtime)
     for f in functions:
@@ -71,6 +72,11 @@ def add_module_functions(fn, functions, namespace):
             parser.parse_python(None, filename=fn, lineno=1, fixedhash=fixedhash+f)
             #bb.warn("Cached %s" % f)
         except KeyError:
+            targetfn = inspect.getsourcefile(functions[f])
+            if fn != targetfn:
+                # Skip references to other modules outside this file
+                #bb.warn("Skipping %s" % name)
+                continue
             lines, lineno = inspect.getsourcelines(functions[f])
             src = "".join(lines)
             parser.parse_python(src, filename=fn, lineno=lineno, fixedhash=fixedhash+f)
@@ -81,14 +87,14 @@ def add_module_functions(fn, functions, namespace):
             if e in functions:
                 execs.remove(e)
                 execs.add(namespace + "." + e)
-        modulecode_deps[name] = [parser.references.copy(), execs, parser.var_execs.copy(), parser.contains.copy()]
+        modulecode_deps[name] = [parser.references.copy(), execs, parser.var_execs.copy(), parser.contains.copy(), parser.extra]
         #bb.warn("%s: %s\nRefs:%s Execs: %s %s %s" % (name, fn, parser.references, parser.execs, parser.var_execs, parser.contains))
 
 def update_module_dependencies(d):
     for mod in modulecode_deps:
         excludes = set((d.getVarFlag(mod, "vardepsexclude") or "").split())
         if excludes:
-            modulecode_deps[mod] = [modulecode_deps[mod][0] - excludes, modulecode_deps[mod][1] - excludes, modulecode_deps[mod][2] - excludes, modulecode_deps[mod][3]]
+            modulecode_deps[mod] = [modulecode_deps[mod][0] - excludes, modulecode_deps[mod][1] - excludes, modulecode_deps[mod][2] - excludes, modulecode_deps[mod][3], modulecode_deps[mod][4]]
 
 # A custom getstate/setstate using tuples is actually worth 15% cachesize by
 # avoiding duplication of the attribute names!
@@ -111,21 +117,22 @@ class SetCache(object):
 codecache = SetCache()
 
 class pythonCacheLine(object):
-    def __init__(self, refs, execs, contains):
+    def __init__(self, refs, execs, contains, extra):
         self.refs = codecache.internSet(refs)
         self.execs = codecache.internSet(execs)
         self.contains = {}
         for c in contains:
             self.contains[c] = codecache.internSet(contains[c])
+        self.extra = extra
 
     def __getstate__(self):
-        return (self.refs, self.execs, self.contains)
+        return (self.refs, self.execs, self.contains, self.extra)
 
     def __setstate__(self, state):
-        (refs, execs, contains) = state
-        self.__init__(refs, execs, contains)
+        (refs, execs, contains, extra) = state
+        self.__init__(refs, execs, contains, extra)
     def __hash__(self):
-        l = (hash(self.refs), hash(self.execs))
+        l = (hash(self.refs), hash(self.execs), hash(self.extra))
         for c in sorted(self.contains.keys()):
             l = l + (c, hash(self.contains[c]))
         return hash(l)
@@ -154,7 +161,7 @@ class CodeParserCache(MultiProcessCache):
     # so that an existing cache gets invalidated. Additionally you'll need
     # to increment __cache_version__ in cache.py in order to ensure that old
     # recipe caches don't trigger "Taskhash mismatch" errors.
-    CACHE_VERSION = 11
+    CACHE_VERSION = 12
 
     def __init__(self):
         MultiProcessCache.__init__(self)
@@ -168,8 +175,8 @@ class CodeParserCache(MultiProcessCache):
         self.pythoncachelines = {}
         self.shellcachelines = {}
 
-    def newPythonCacheLine(self, refs, execs, contains):
-        cacheline = pythonCacheLine(refs, execs, contains)
+    def newPythonCacheLine(self, refs, execs, contains, extra):
+        cacheline = pythonCacheLine(refs, execs, contains, extra)
         h = hash(cacheline)
         if h in self.pythoncachelines:
             return self.pythoncachelines[h]
@@ -257,17 +264,17 @@ class PythonParser():
         if name and (name.endswith(self.getvars) or name.endswith(self.getvarflags) or name in self.containsfuncs or name in self.containsanyfuncs):
             if isinstance(node.args[0], ast.Constant) and isinstance(node.args[0].value, str):
                 varname = node.args[0].value
-                if name in self.containsfuncs and isinstance(node.args[1], ast.Str):
+                if name in self.containsfuncs and isinstance(node.args[1], ast.Constant):
                     if varname not in self.contains:
                         self.contains[varname] = set()
-                    self.contains[varname].add(node.args[1].s)
-                elif name in self.containsanyfuncs and isinstance(node.args[1], ast.Str):
+                    self.contains[varname].add(node.args[1].value)
+                elif name in self.containsanyfuncs and isinstance(node.args[1], ast.Constant):
                     if varname not in self.contains:
                         self.contains[varname] = set()
-                    self.contains[varname].update(node.args[1].s.split())
+                    self.contains[varname].update(node.args[1].value.split())
                 elif name.endswith(self.getvarflags):
-                    if isinstance(node.args[1], ast.Str):
-                        self.references.add('%s[%s]' % (varname, node.args[1].s))
+                    if isinstance(node.args[1], ast.Constant):
+                        self.references.add('%s[%s]' % (varname, node.args[1].value))
                     else:
                         self.warn(node.func, node.args[1])
                 else:
@@ -275,8 +282,8 @@ class PythonParser():
             else:
                 self.warn(node.func, node.args[0])
         elif name and name.endswith(".expand"):
-            if isinstance(node.args[0], ast.Str):
-                value = node.args[0].s
+            if isinstance(node.args[0], ast.Constant):
+                value = node.args[0].value
                 d = bb.data.init()
                 parser = d.expandWithRefs(value, self.name)
                 self.references |= parser.references
@@ -286,8 +293,8 @@ class PythonParser():
                         self.contains[varname] = set()
                     self.contains[varname] |= parser.contains[varname]
         elif name in self.execfuncs:
-            if isinstance(node.args[0], ast.Str):
-                self.var_execs.add(node.args[0].s)
+            if isinstance(node.args[0], ast.Constant):
+                self.var_execs.add(node.args[0].value)
             else:
                 self.warn(node.func, node.args[0])
         elif name and isinstance(node.func, (ast.Name, ast.Attribute)):
@@ -337,6 +344,7 @@ class PythonParser():
             self.contains = {}
             for i in codeparsercache.pythoncache[h].contains:
                 self.contains[i] = set(codeparsercache.pythoncache[h].contains[i])
+            self.extra = codeparsercache.pythoncache[h].extra
             return
 
         if h in codeparsercache.pythoncacheextras:
@@ -345,6 +353,7 @@ class PythonParser():
             self.contains = {}
             for i in codeparsercache.pythoncacheextras[h].contains:
                 self.contains[i] = set(codeparsercache.pythoncacheextras[h].contains[i])
+            self.extra = codeparsercache.pythoncacheextras[h].extra
             return
 
         if fixedhash and not node:
@@ -363,8 +372,11 @@ class PythonParser():
                 self.visit_Call(n)
 
         self.execs.update(self.var_execs)
+        self.extra = None
+        if fixedhash:
+            self.extra = bbhash(str(node))
 
-        codeparsercache.pythoncacheextras[h] = codeparsercache.newPythonCacheLine(self.references, self.execs, self.contains)
+        codeparsercache.pythoncacheextras[h] = codeparsercache.newPythonCacheLine(self.references, self.execs, self.contains, self.extra)
 
 class ShellParser():
     def __init__(self, name, log):

bitbake/lib/bb/command.py

@@ -550,8 +550,8 @@ class CommandsSync:
         and return a datastore object representing the environment
         for the recipe.
         """
-        fn = params[0]
-        mc = bb.runqueue.mc_from_tid(fn)
+        virtualfn = params[0]
+        (fn, cls, mc) = bb.cache.virtualfn2realfn(virtualfn)
         appends = params[1]
         appendlist = params[2]
         if len(params) > 3:
@@ -574,10 +574,10 @@ class CommandsSync:
         if config_data:
             # We have to use a different function here if we're passing in a datastore
             # NOTE: we took a copy above, so we don't do it here again
-            envdata = command.cooker.databuilder._parse_recipe(config_data, fn, appendfiles, mc, layername)['']
+            envdata = command.cooker.databuilder._parse_recipe(config_data, fn, appendfiles, mc, layername)[cls]
         else:
             # Use the standard path
-            envdata = command.cooker.databuilder.parseRecipe(fn, appendfiles, layername)
+            envdata = command.cooker.databuilder.parseRecipe(virtualfn, appendfiles, layername)
         idx = command.remotedatastores.store(envdata)
         return DataStoreConnectionHandle(idx)
     parseRecipeFile.readonly = True
@@ -777,6 +777,7 @@ class CommandsAsync:
         (mc, pn) = bb.runqueue.split_mc(params[0])
         taskname = params[1]
         sigs = params[2]
+        bb.siggen.check_siggen_version(bb.siggen)
         res = bb.siggen.find_siginfo(pn, taskname, sigs, command.cooker.databuilder.mcdata[mc])
         bb.event.fire(bb.event.FindSigInfoResult(res), command.cooker.databuilder.mcdata[mc])
         command.finishAsyncCommand()

bitbake/lib/bb/cooker.py

@@ -102,12 +102,15 @@ class CookerFeatures(object):
 
 class EventWriter:
     def __init__(self, cooker, eventfile):
-        self.file_inited = None
         self.cooker = cooker
         self.eventfile = eventfile
         self.event_queue = []
 
-    def write_event(self, event):
+    def write_variables(self):
+        with open(self.eventfile, "a") as f:
+            f.write("%s\n" % json.dumps({ "allvariables" : self.cooker.getAllKeysWithFlags(["doc", "func"])}))
+
+    def send(self, event):
         with open(self.eventfile, "a") as f:
             try:
                 str_event = codecs.encode(pickle.dumps(event), 'base64').decode('utf-8')
@@ -117,28 +120,6 @@ class EventWriter:
                 import traceback
                 print(err, traceback.format_exc())
 
-    def send(self, event):
-        if self.file_inited:
-            # we have the file, just write the event
-            self.write_event(event)
-        else:
-            # init on bb.event.BuildStarted
-            name = "%s.%s" % (event.__module__, event.__class__.__name__)
-            if name in ("bb.event.BuildStarted", "bb.cooker.CookerExit"):
-                with open(self.eventfile, "w") as f:
-                    f.write("%s\n" % json.dumps({ "allvariables" : self.cooker.getAllKeysWithFlags(["doc", "func"])}))
-
-                self.file_inited = True
-
-                # write pending events
-                for evt in self.event_queue:
-                    self.write_event(evt)
-
-                # also write the current event
-                self.write_event(event)
-            else:
-                # queue all events until the file is inited
-                self.event_queue.append(event)
 
 #============================================================================#
 # BBCooker
@@ -303,6 +284,10 @@ class BBCooker:
         self.data_hash = self.databuilder.data_hash
         self.extraconfigdata = {}
 
+        eventlog = self.data.getVar("BB_DEFAULT_EVENTLOG")
+        if not self.configuration.writeeventlog and eventlog:
+            self.setupEventLog(eventlog)
+
         if consolelog:
             self.data.setVar("BB_CONSOLELOG", consolelog)
 
@@ -330,13 +315,13 @@ class BBCooker:
                 dbfile = (self.data.getVar("PERSISTENT_DIR") or self.data.getVar("CACHE")) + "/hashserv.db"
                 upstream = self.data.getVar("BB_HASHSERVE_UPSTREAM") or None
                 if upstream:
-                    import socket
                     try:
-                        sock = socket.create_connection(upstream.split(":"), 5)
-                        sock.close()
-                    except socket.error as e:
+                        with hashserv.create_client(upstream) as client:
+                            client.ping()
+                    except (ConnectionError, ImportError) as e:
                         bb.warn("BB_HASHSERVE_UPSTREAM is not valid, unable to connect hash equivalence server at '%s': %s"
                                  % (upstream, repr(e)))
+                        upstream = None
 
                 self.hashservaddr = "unix://%s/hashserve.sock" % self.data.getVar("TOPDIR")
                 self.hashserv = hashserv.create_server(
@@ -345,7 +330,7 @@ class BBCooker:
                     sync=False,
                     upstream=upstream,
                 )
-                self.hashserv.serve_as_process()
+                self.hashserv.serve_as_process(log_level=logging.WARNING)
             for mc in self.databuilder.mcdata:
                 self.databuilder.mcorigdata[mc].setVar("BB_HASHSERVE", self.hashservaddr)
                 self.databuilder.mcdata[mc].setVar("BB_HASHSERVE", self.hashservaddr)
@@ -409,6 +394,19 @@ class BBCooker:
 
         self._parsecache_set(False)
 
+    def setupEventLog(self, eventlog):
+        if self.eventlog and self.eventlog[0] != eventlog:
+            bb.event.unregister_UIHhandler(self.eventlog[1])
+            self.eventlog = None
+        if not self.eventlog or self.eventlog[0] != eventlog:
+            # we log all events to a file if so directed
+            # register the log file writer as UI Handler
+            if not os.path.exists(os.path.dirname(eventlog)):
+                bb.utils.mkdirhier(os.path.dirname(eventlog))
+            writer = EventWriter(self, eventlog)
+            EventLogWriteHandler = namedtuple('EventLogWriteHandler', ['event'])
+            self.eventlog = (eventlog, bb.event.register_UIHhandler(EventLogWriteHandler(writer)), writer)
+
     def updateConfigOpts(self, options, environment, cmdline):
         self.ui_cmdline = cmdline
         clean = True
@@ -428,14 +426,7 @@ class BBCooker:
                 setattr(self.configuration, o, options[o])
 
         if self.configuration.writeeventlog:
-            if self.eventlog and self.eventlog[0] != self.configuration.writeeventlog:
-                bb.event.unregister_UIHhandler(self.eventlog[1])
-            if not self.eventlog or self.eventlog[0] != self.configuration.writeeventlog:
-                # we log all events to a file if so directed
-                # register the log file writer as UI Handler
-                writer = EventWriter(self, self.configuration.writeeventlog)
-                EventLogWriteHandler = namedtuple('EventLogWriteHandler', ['event'])
-                self.eventlog = (self.configuration.writeeventlog, bb.event.register_UIHhandler(EventLogWriteHandler(writer)))
+            self.setupEventLog(self.configuration.writeeventlog)
 
         bb.msg.loggerDefaultLogLevel = self.configuration.default_loglevel
         bb.msg.loggerDefaultDomains = self.configuration.debug_domains
@@ -1395,6 +1386,8 @@ class BBCooker:
         buildname = self.databuilder.mcdata[mc].getVar("BUILDNAME")
         if fireevents:
             bb.event.fire(bb.event.BuildStarted(buildname, [item]), self.databuilder.mcdata[mc])
+            if self.eventlog:
+                self.eventlog[2].write_variables()
             bb.event.enable_heartbeat()
 
         # Execute the runqueue
@@ -1466,7 +1459,6 @@ class BBCooker:
 
                     if t in task or getAllTaskSignatures:
                         try:
-                            rq.rqdata.prepare_task_hash(tid)
                             sig.append([pn, t, rq.rqdata.get_task_unihash(tid)])
                         except KeyError:
                             sig.append(self.getTaskSignatures(target, [t])[0])
@@ -1538,6 +1530,8 @@ class BBCooker:
 
         for mc in self.multiconfigs:
             bb.event.fire(bb.event.BuildStarted(buildname, ntargets), self.databuilder.mcdata[mc])
+        if self.eventlog:
+            self.eventlog[2].write_variables()
         bb.event.enable_heartbeat()
 
         rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist)
@@ -1548,7 +1542,13 @@ class BBCooker:
 
 
     def getAllKeysWithFlags(self, flaglist):
+        def dummy_autorev(d):
+            return
+
         dump = {}
+        # Horrible but for now we need to avoid any sideeffects of autorev being called
+        saved = bb.fetch2.get_autorev
+        bb.fetch2.get_autorev = dummy_autorev
         for k in self.data.keys():
             try:
                 expand = True
@@ -1568,6 +1568,7 @@ class BBCooker:
                             dump[k][d] = None
             except Exception as e:
                 print(e)
+        bb.fetch2.get_autorev = saved
         return dump
 
 
@@ -1787,7 +1788,7 @@ class CookerCollectFiles(object):
             for ignored in ('SCCS', 'CVS', '.svn'):
                 if ignored in dirs:
                     dirs.remove(ignored)
-            found += [os.path.join(dir, f) for f in files if (f.endswith(['.bb', '.bbappend']))]
+            found += [os.path.join(dir, f) for f in files if (f.endswith(('.bb', '.bbappend')))]
 
         return found
 

bitbake/lib/bb/cookerdata.py

@@ -503,8 +503,8 @@ class CookerDataBuilder(object):
 
         if appends:
             bb_data.setVar('__BBAPPEND', " ".join(appends))
-        bb_data = bb.parse.handle(bbfile, bb_data)
-        return bb_data
+
+        return bb.parse.handle(bbfile, bb_data)
 
     def parseRecipeVariants(self, bbfile, appends, virtonly=False, mc=None, layername=None):
         """
@@ -516,8 +516,7 @@ class CookerDataBuilder(object):
             (bbfile, virtual, mc) = bb.cache.virtualfn2realfn(bbfile)
             bb_data = self.mcdata[mc].createCopy()
             bb_data.setVar("__ONLYFINALISE", virtual or "default")
-            datastores = self._parse_recipe(bb_data, bbfile, appends, mc, layername)
-            return datastores
+            return self._parse_recipe(bb_data, bbfile, appends, mc, layername)
 
         if mc is not None:
             bb_data = self.mcdata[mc].createCopy()
@@ -543,5 +542,5 @@ class CookerDataBuilder(object):
         """
         logger.debug("Parsing %s (full)" % virtualfn)
         (fn, virtual, mc) = bb.cache.virtualfn2realfn(virtualfn)
-        bb_data = self.parseRecipeVariants(virtualfn, appends, virtonly=True, layername=layername)
-        return bb_data[virtual]
+        datastores = self.parseRecipeVariants(virtualfn, appends, virtonly=True, layername=layername)
+        return datastores[virtual]

bitbake/lib/bb/data.py

@@ -293,7 +293,7 @@ def build_dependencies(key, keys, mod_funcs, shelldeps, varflagsexcl, ignored_va
         if key in mod_funcs:
             exclusions = set()
             moddep = bb.codeparser.modulecode_deps[key]
-            value = handle_contains("", moddep[3], exclusions, d)
+            value = handle_contains(moddep[4], moddep[3], exclusions, d)
             return frozenset((moddep[0] | keys & moddep[1]) - ignored_vars), value
 
         if key[-1] == ']':

bitbake/lib/bb/data_smart.py

@@ -272,12 +272,9 @@ class VariableHistory(object):
             return
         if 'op' not in loginfo or not loginfo['op']:
             loginfo['op'] = 'set'
-        if 'detail' in loginfo:
-            loginfo['detail'] = str(loginfo['detail'])
         if 'variable' not in loginfo or 'file' not in loginfo:
             raise ValueError("record() missing variable or file.")
         var = loginfo['variable']
-
         if var not in self.variables:
             self.variables[var] = []
         if not isinstance(self.variables[var], list):
@@ -336,7 +333,8 @@ class VariableHistory(object):
                     flag = '[%s] ' % (event['flag'])
                 else:
                     flag = ''
-                o.write("#   %s %s:%s%s\n#     %s\"%s\"\n" % (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n#     ', event['detail'])))
+                o.write("#   %s %s:%s%s\n#     %s\"%s\"\n" % \
+                    (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n#     ', str(event['detail']))))
             if len(history) > 1:
                 o.write("# pre-expansion value:\n")
                 o.write('#   "%s"\n' % (commentVal))
@@ -390,7 +388,7 @@ class VariableHistory(object):
             if isset and event['op'] == 'set?':
                 continue
             isset = True
-            items = d.expand(event['detail']).split()
+            items = d.expand(str(event['detail'])).split()
             for item in items:
                 # This is a little crude but is belt-and-braces to avoid us
                 # having to handle every possible operation type specifically

bitbake/lib/bb/event.py

@@ -257,14 +257,15 @@ def register(name, handler, mask=None, filename=None, lineno=None, data=None):
         # handle string containing python code
         if isinstance(handler, str):
             tmp = "def %s(e, d):\n%s" % (name, handler)
+            # Inject empty lines to make code match lineno in filename
+            if lineno is not None:
+                tmp = "\n" * (lineno-1) + tmp
             try:
                 code = bb.methodpool.compile_cache(tmp)
                 if not code:
                     if filename is None:
                         filename = "%s(e, d)" % name
                     code = compile(tmp, filename, "exec", ast.PyCF_ONLY_AST)
-                    if lineno is not None:
-                        ast.increment_lineno(code, lineno-1)
                     code = compile(code, filename, "exec")
                     bb.methodpool.compile_cache_add(tmp, code)
             except SyntaxError:

bitbake/lib/bb/fetch2/__init__.py

@@ -290,12 +290,12 @@ class URI(object):
 
     def _param_str_split(self, string, elmdelim, kvdelim="="):
         ret = collections.OrderedDict()
-        for k, v in [x.split(kvdelim, 1) for x in string.split(elmdelim) if x]:
+        for k, v in [x.split(kvdelim, 1) if kvdelim in x else (x, None) for x in string.split(elmdelim) if x]:
             ret[k] = v
         return ret
 
     def _param_str_join(self, dict_, elmdelim, kvdelim="="):
-        return elmdelim.join([kvdelim.join([k, v]) for k, v in dict_.items()])
+        return elmdelim.join([kvdelim.join([k, v]) if v else k for k, v in dict_.items()])
 
     @property
     def hostport(self):
@@ -872,7 +872,10 @@ FETCH_EXPORT_VARS = ['HOME', 'PATH',
                      'AWS_PROFILE',
                      'AWS_ACCESS_KEY_ID',
                      'AWS_SECRET_ACCESS_KEY',
+                     'AWS_ROLE_ARN',
+                     'AWS_WEB_IDENTITY_TOKEN_FILE',
                      'AWS_DEFAULT_REGION',
+                     'AWS_SESSION_TOKEN',
                      'GIT_CACHE_PATH',
                      'REMOTE_CONTAINERS_IPC',
                      'SSL_CERT_DIR']
@@ -940,7 +943,10 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None):
         elif e.stderr:
             output = "output:\n%s" % e.stderr
         else:
-            output = "no output"
+            if log:
+                output = "see logfile for output"
+            else:
+                output = "no output"
         error_message = "Fetch command %s failed with exit code %s, %s" % (e.command, e.exitcode, output)
     except bb.process.CmdError as e:
         error_message = "Fetch command %s could not be run:\n%s" % (e.command, e.msg)
@@ -1112,7 +1118,8 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
             logger.debug("Mirror fetch failure for url %s (original url: %s)" % (ud.url, origud.url))
             logger.debug(str(e))
         try:
-            ud.method.clean(ud, ld)
+            if ud.method.cleanup_upon_failure():
+                ud.method.clean(ud, ld)
         except UnboundLocalError:
             pass
         return False
@@ -1437,6 +1444,12 @@ class FetchMethod(object):
         """
         return False
 
+    def cleanup_upon_failure(self):
+        """
+        When a fetch fails, should clean() be called?
+        """
+        return True
+
     def verify_donestamp(self, ud, d):
         """
         Verify the donestamp file
@@ -1882,7 +1895,7 @@ class Fetch(object):
                             logger.debug(str(e))
                         firsterr = e
                         # Remove any incomplete fetch
-                        if not verified_stamp:
+                        if not verified_stamp and m.cleanup_upon_failure():
                             m.clean(ud, self.d)
                         logger.debug("Trying MIRRORS")
                         mirrors = mirror_from_string(self.d.getVar('MIRRORS'))
@@ -1945,7 +1958,7 @@ class Fetch(object):
                     ret = m.try_mirrors(self, ud, self.d, mirrors, True)
 
             if not ret:
-                raise FetchError("URL %s doesn't work" % u, u)
+                raise FetchError("URL doesn't work", u)
 
     def unpack(self, root, urls=None):
         """

bitbake/lib/bb/fetch2/gcp.py

@@ -23,6 +23,7 @@ import urllib.parse, urllib.error
 from bb.fetch2 import FetchMethod
 from bb.fetch2 import FetchError
 from bb.fetch2 import logger
+from bb.fetch2 import runfetchcmd
 
 class GCP(FetchMethod):
     """
@@ -47,6 +48,7 @@ class GCP(FetchMethod):
             ud.basename = os.path.basename(ud.path)
 
         ud.localfile = d.expand(urllib.parse.unquote(ud.basename))
+        ud.basecmd = "gsutil stat"
 
     def get_gcp_client(self):
         from google.cloud import storage
@@ -61,7 +63,8 @@ class GCP(FetchMethod):
         if self.gcp_client is None:
             self.get_gcp_client()
 
-        bb.fetch2.check_network_access(d, "gsutil stat", ud.url)
+        bb.fetch2.check_network_access(d, ud.basecmd, f"gs://{ud.host}{ud.path}")
+        runfetchcmd("%s %s" % (ud.basecmd, f"gs://{ud.host}{ud.path}"), d)
 
         # Path sometimes has leading slash, so strip it
         path = ud.path.lstrip("/")
@@ -88,7 +91,8 @@ class GCP(FetchMethod):
         if self.gcp_client is None:
             self.get_gcp_client()
 
-        bb.fetch2.check_network_access(d, "gsutil stat", ud.url)
+        bb.fetch2.check_network_access(d, ud.basecmd, f"gs://{ud.host}{ud.path}")
+        runfetchcmd("%s %s" % (ud.basecmd, f"gs://{ud.host}{ud.path}"), d)
 
         # Path sometimes has leading slash, so strip it
         path = ud.path.lstrip("/")

bitbake/lib/bb/fetch2/git.py

@@ -48,10 +48,23 @@ Supported SRC_URI options are:
    instead of branch.
    The default is "0", set nobranch=1 if needed.
 
+- subpath
+   Limit the checkout to a specific subpath of the tree.
+   By default, checkout the whole tree, set subpath=<path> if needed
+
+- destsuffix
+   The name of the path in which to place the checkout.
+   By default, the path is git/, set destsuffix=<suffix> if needed
+
 - usehead
    For local git:// urls to use the current branch HEAD as the revision for use with
    AUTOREV. Implies nobranch.
 
+- lfs
+    Enable the checkout to use LFS for large files. This will download all LFS files
+    in the download step, as the unpack step does not have network access.
+    The default is "1", set lfs=0 to skip.
+
 """
 
 # Copyright (C) 2005 Richard Purdie
@@ -74,6 +87,7 @@ from contextlib import contextmanager
 from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import runfetchcmd
 from   bb.fetch2 import logger
+from   bb.fetch2 import trusted_network
 
 
 sha1_re = re.compile(r'^[0-9a-f]{40}$')
@@ -136,6 +150,9 @@ class Git(FetchMethod):
     def supports_checksum(self, urldata):
         return False
 
+    def cleanup_upon_failure(self):
+        return False
+
     def urldata_init(self, ud, d):
         """
         init git specific variable within url data
@@ -245,7 +262,7 @@ class Git(FetchMethod):
             for name in ud.names:
                 ud.unresolvedrev[name] = 'HEAD'
 
-        ud.basecmd = d.getVar("FETCHCMD_git") or "git -c gc.autoDetach=false -c core.pager=cat"
+        ud.basecmd = d.getVar("FETCHCMD_git") or "git -c gc.autoDetach=false -c core.pager=cat -c safe.bareRepository=all"
 
         write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0"
         ud.write_tarballs = write_tarballs != "0" or ud.rebaseable
@@ -260,7 +277,7 @@ class Git(FetchMethod):
                     ud.unresolvedrev[name] = ud.revisions[name]
                 ud.revisions[name] = self.latest_revision(ud, d, name)
 
-        gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_'))
+        gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_').replace('(', '_').replace(')', '_'))
         if gitsrcname.startswith('.'):
             gitsrcname = gitsrcname[1:]
 
@@ -311,7 +328,10 @@ class Git(FetchMethod):
         return ud.clonedir
 
     def need_update(self, ud, d):
-        return self.clonedir_need_update(ud, d) or self.shallow_tarball_need_update(ud) or self.tarball_need_update(ud)
+        return self.clonedir_need_update(ud, d) \
+                or self.shallow_tarball_need_update(ud) \
+                or self.tarball_need_update(ud) \
+                or self.lfs_need_update(ud, d)
 
     def clonedir_need_update(self, ud, d):
         if not os.path.exists(ud.clonedir):
@@ -323,6 +343,15 @@ class Git(FetchMethod):
                 return True
         return False
 
+    def lfs_need_update(self, ud, d):
+        if self.clonedir_need_update(ud, d):
+            return True
+
+        for name in ud.names:
+            if not self._lfs_objects_downloaded(ud, d, name, ud.clonedir):
+                return True
+        return False
+
     def clonedir_need_shallow_revs(self, ud, d):
         for rev in ud.shallow_revs:
             try:
@@ -342,6 +371,16 @@ class Git(FetchMethod):
         # is not possible
         if bb.utils.to_boolean(d.getVar("BB_FETCH_PREMIRRORONLY")):
             return True
+        # If the url is not in trusted network, that is, BB_NO_NETWORK is set to 0
+        # and BB_ALLOWED_NETWORKS does not contain the host that ud.url uses, then
+        # we need to try premirrors first as using upstream is destined to fail.
+        if not trusted_network(d, ud.url):
+            return True
+        # the following check is to ensure incremental fetch in downloads, this is
+        # because the premirror might be old and does not contain the new rev required,
+        # and this will cause a total removal and new clone. So if we can reach to
+        # network, we prefer upstream over premirror, though the premirror might contain
+        # the new rev.
         if os.path.exists(ud.clonedir):
             return False
         return True
@@ -362,7 +401,11 @@ class Git(FetchMethod):
             else:
                 tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
                 runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=tmpdir)
-                fetch_cmd = "LANG=C %s fetch -f --progress %s " % (ud.basecmd, shlex.quote(tmpdir))
+                output = runfetchcmd("%s remote" % ud.basecmd, d, quiet=True, workdir=ud.clonedir)
+                if 'mirror' in output:
+                    runfetchcmd("%s remote rm mirror" % ud.basecmd, d, workdir=ud.clonedir)
+                runfetchcmd("%s remote add --mirror=fetch mirror %s" % (ud.basecmd, tmpdir), d, workdir=ud.clonedir)
+                fetch_cmd = "LANG=C %s fetch -f --update-head-ok  --progress mirror " % (ud.basecmd)
                 runfetchcmd(fetch_cmd, d, workdir=ud.clonedir)
         repourl = self._get_repo_url(ud)
 
@@ -439,7 +482,7 @@ class Git(FetchMethod):
             if missing_rev:
                 raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev)
 
-        if self._contains_lfs(ud, d, ud.clonedir) and self._need_lfs(ud):
+        if self.lfs_need_update(ud, d):
             # Unpack temporary working copy, use it to run 'git checkout' to force pre-fetching
             # of all LFS blobs needed at the srcrev.
             #
@@ -462,8 +505,8 @@ class Git(FetchMethod):
                 # Only do this if the unpack resulted in a .git/lfs directory being
                 # created; this only happens if at least one blob needed to be
                 # downloaded.
-                if os.path.exists(os.path.join(tmpdir, "git", ".git", "lfs")):
-                    runfetchcmd("tar -cf - lfs | tar -xf - -C %s" % ud.clonedir, d, workdir="%s/git/.git" % tmpdir)
+                if os.path.exists(os.path.join(ud.destdir, ".git", "lfs")):
+                    runfetchcmd("tar -cf - lfs | tar -xf - -C %s" % ud.clonedir, d, workdir="%s/.git" % ud.destdir)
 
     def build_mirror_data(self, ud, d):
 
@@ -501,7 +544,7 @@ class Git(FetchMethod):
 
             logger.info("Creating tarball of git repository")
             with create_atomic(ud.fullmirror) as tfile:
-                mtime = runfetchcmd("git log --all -1 --format=%cD", d,
+                mtime = runfetchcmd("{} log --all -1 --format=%cD".format(ud.basecmd), d,
                         quiet=True, workdir=ud.clonedir)
                 runfetchcmd("tar -czf %s --owner oe:0 --group oe:0 --mtime \"%s\" ."
                         % (tfile, mtime), d, workdir=ud.clonedir)
@@ -629,6 +672,8 @@ class Git(FetchMethod):
                 raise bb.fetch2.FetchError("Repository %s has LFS content, install git-lfs on host to download (or set lfs=0 to ignore it)" % (repourl))
             elif not need_lfs:
                 bb.note("Repository %s has LFS content but it is not being fetched" % (repourl))
+            else:
+                runfetchcmd("%s lfs install --local" % ud.basecmd, d, workdir=destdir)
 
         if not ud.nocheckout:
             if subpath:
@@ -680,6 +725,35 @@ class Git(FetchMethod):
             raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output))
         return output.split()[0] != "0"
 
+    def _lfs_objects_downloaded(self, ud, d, name, wd):
+        """
+        Verifies whether the LFS objects for requested revisions have already been downloaded
+        """
+        # Bail out early if this repository doesn't use LFS
+        if not self._need_lfs(ud) or not self._contains_lfs(ud, d, wd):
+            return True
+
+        # The Git LFS specification specifies ([1]) the LFS folder layout so it should be safe to check for file
+        # existence.
+        # [1] https://github.com/git-lfs/git-lfs/blob/main/docs/spec.md#intercepting-git
+        cmd = "%s lfs ls-files -l %s" \
+                % (ud.basecmd, ud.revisions[name])
+        output = runfetchcmd(cmd, d, quiet=True, workdir=wd).rstrip()
+        # Do not do any further matching if no objects are managed by LFS
+        if not output:
+            return True
+
+        # Match all lines beginning with the hexadecimal OID
+        oid_regex = re.compile("^(([a-fA-F0-9]{2})([a-fA-F0-9]{2})[A-Fa-f0-9]+)")
+        for line in output.split("\n"):
+            oid = re.search(oid_regex, line)
+            if not oid:
+                bb.warn("git lfs ls-files output '%s' did not match expected format." % line)
+            if not os.path.exists(os.path.join(wd, "lfs", "objects", oid.group(2), oid.group(3), oid.group(1))):
+                return False
+
+        return True
+
     def _need_lfs(self, ud):
         return ud.parm.get("lfs", "1") == "1"
 
@@ -688,8 +762,11 @@ class Git(FetchMethod):
         Check if the repository has 'lfs' (large file) content
         """
 
-        # The bare clonedir doesn't use the remote names; it has the branch immediately.
-        if wd == ud.clonedir:
+        if ud.nobranch:
+            # If no branch is specified, use the current git commit
+            refname = self._build_revision(ud, d, ud.names[0])
+        elif wd == ud.clonedir:
+            # The bare clonedir doesn't use the remote names; it has the branch immediately.
             refname = ud.branches[ud.names[0]]
         else:
             refname = "origin/%s" % ud.branches[ud.names[0]]
@@ -794,38 +871,42 @@ class Git(FetchMethod):
         """
         pupver = ('', '')
 
-        tagregex = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or r"(?P<pver>([0-9][\.|_]?)+)")
         try:
             output = self._lsremote(ud, d, "refs/tags/*")
         except (bb.fetch2.FetchError, bb.fetch2.NetworkAccess) as e:
             bb.note("Could not list remote: %s" % str(e))
             return pupver
 
+        rev_tag_re = re.compile(r"([0-9a-f]{40})\s+refs/tags/(.*)")
+        pver_re = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or r"(?P<pver>([0-9][\.|_]?)+)")
+        nonrel_re = re.compile(r"(alpha|beta|rc|final)+")
+
         verstring = ""
-        revision = ""
         for line in output.split("\n"):
             if not line:
                 break
 
-            tag_head = line.split("/")[-1]
+            m = rev_tag_re.match(line)
+            if not m:
+                continue
+
+            (revision, tag) = m.groups()
+
             # Ignore non-released branches
-            m = re.search(r"(alpha|beta|rc|final)+", tag_head)
-            if m:
+            if nonrel_re.search(tag):
                 continue
 
             # search for version in the line
-            tag = tagregex.search(tag_head)
-            if tag is None:
+            m = pver_re.search(tag)
+            if not m:
                 continue
 
-            tag = tag.group('pver')
-            tag = tag.replace("_", ".")
+            pver = m.group('pver').replace("_", ".")
 
-            if verstring and bb.utils.vercmp(("0", tag, ""), ("0", verstring, "")) < 0:
+            if verstring and bb.utils.vercmp(("0", pver, ""), ("0", verstring, "")) < 0:
                 continue
 
-            verstring = tag
-            revision = line.split()[0]
+            verstring = pver
             pupver = (verstring, revision)
 
         return pupver

bitbake/lib/bb/fetch2/wget.py

@@ -87,7 +87,10 @@ class Wget(FetchMethod):
         if not ud.localfile:
             ud.localfile = d.expand(urllib.parse.unquote(ud.host + ud.path).replace("/", "."))
 
-        self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -t 2 -T 30 --passive-ftp"
+        self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -t 2 -T 30"
+
+        if ud.type == 'ftp' or ud.type == 'ftps':
+            self.basecmd += " --passive-ftp"
 
         if not self.check_certs(d):
             self.basecmd += " --no-check-certificate"
@@ -105,7 +108,8 @@ class Wget(FetchMethod):
 
         fetchcmd = self.basecmd
 
-        localpath = os.path.join(d.getVar("DL_DIR"), ud.localfile) + ".tmp"
+        dldir = os.path.realpath(d.getVar("DL_DIR"))
+        localpath = os.path.join(dldir, ud.localfile) + ".tmp"
         bb.utils.mkdirhier(os.path.dirname(localpath))
         fetchcmd += " -O %s" % shlex.quote(localpath)
 
@@ -125,12 +129,21 @@ class Wget(FetchMethod):
         uri = ud.url.split(";")[0]
         if os.path.exists(ud.localpath):
             # file exists, but we didnt complete it.. trying again..
-            fetchcmd += d.expand(" -c -P ${DL_DIR} '%s'" % uri)
+            fetchcmd += " -c -P " + dldir + " '" + uri + "'"
         else:
-            fetchcmd += d.expand(" -P ${DL_DIR} '%s'" % uri)
+            fetchcmd += " -P " + dldir + " '" + uri + "'"
 
         self._runwget(ud, d, fetchcmd, False)
 
+        # Sanity check since wget can pretend it succeed when it didn't
+        # Also, this used to happen if sourceforge sent us to the mirror page
+        if not os.path.exists(localpath):
+            raise FetchError("The fetch command returned success for url %s but %s doesn't exist?!" % (uri, localpath), uri)
+
+        if os.path.getsize(localpath) == 0:
+            os.remove(localpath)
+            raise FetchError("The fetch of %s resulted in a zero size file?! Deleting and failing since this isn't right." % (uri), uri)
+
         # Try and verify any checksum now, meaning if it isn't correct, we don't remove the
         # original file, which might be a race (imagine two recipes referencing the same
         # source, one with an incorrect checksum)
@@ -140,15 +153,6 @@ class Wget(FetchMethod):
         # Our lock prevents multiple writers but mirroring code may grab incomplete files
         os.rename(localpath, localpath[:-4])
 
-        # Sanity check since wget can pretend it succeed when it didn't
-        # Also, this used to happen if sourceforge sent us to the mirror page
-        if not os.path.exists(ud.localpath):
-            raise FetchError("The fetch command returned success for url %s but %s doesn't exist?!" % (uri, ud.localpath), uri)
-
-        if os.path.getsize(ud.localpath) == 0:
-            os.remove(ud.localpath)
-            raise FetchError("The fetch of %s resulted in a zero size file?! Deleting and failing since this isn't right." % (uri), uri)
-
         return True
 
     def checkstatus(self, fetch, ud, d, try_again=True):
@@ -375,7 +379,7 @@ class Wget(FetchMethod):
                     return self.checkstatus(fetch, ud, d, False)
                 else:
                     # debug for now to avoid spamming the logs in e.g. remote sstate searches
-                    logger.debug2("checkstatus() urlopen failed: %s" % e)
+                    logger.debug2("checkstatus() urlopen failed for %s: %s" % (uri,e))
                     return False
 
         return True

bitbake/lib/bb/main.py

@@ -217,7 +217,9 @@ def create_bitbake_parser():
                              "execution. The SIGNATURE_HANDLER parameter is passed to the "
                              "handler. Two common values are none and printdiff but the handler "
                              "may define more/less. none means only dump the signature, printdiff"
-                             " means compare the dumped signature with the cached one.")
+                             " means recursively compare the dumped signature with the most recent"
+                             " one in a local build or sstate cache (can be used to find out why tasks re-run"
+                             " when that is not expected)")
 
     exec_group.add_argument("--revisions-changed", action="store_true",
                         help="Set the exit code depending on whether upstream floating "

bitbake/lib/bb/msg.py

@@ -230,7 +230,7 @@ def logger_create(name, output=sys.stderr, level=logging.INFO, preserve_handlers
     console = logging.StreamHandler(output)
     console.addFilter(bb.msg.LogFilterShowOnce())
     format = bb.msg.BBLogFormatter("%(levelname)s: %(message)s")
-    if color == 'always' or (color == 'auto' and output.isatty()):
+    if color == 'always' or (color == 'auto' and output.isatty() and os.environ.get('NO_COLOR', '') == ''):
         format.enable_color()
     console.setFormatter(format)
     if preserve_handlers:

bitbake/lib/bb/parse/__init__.py

@@ -49,20 +49,23 @@ class SkipPackage(SkipRecipe):
 __mtime_cache = {}
 def cached_mtime(f):
     if f not in __mtime_cache:
-        __mtime_cache[f] = os.stat(f)[stat.ST_MTIME]
+        res = os.stat(f)
+        __mtime_cache[f] = (res.st_mtime_ns, res.st_size, res.st_ino)
     return __mtime_cache[f]
 
 def cached_mtime_noerror(f):
     if f not in __mtime_cache:
         try:
-            __mtime_cache[f] = os.stat(f)[stat.ST_MTIME]
+            res = os.stat(f)
+            __mtime_cache[f] = (res.st_mtime_ns, res.st_size, res.st_ino)
         except OSError:
             return 0
     return __mtime_cache[f]
 
 def check_mtime(f, mtime):
     try:
-        current_mtime = os.stat(f)[stat.ST_MTIME]
+        res = os.stat(f)
+        current_mtime = (res.st_mtime_ns, res.st_size, res.st_ino)
         __mtime_cache[f] = current_mtime
     except OSError:
         current_mtime = 0
@@ -70,7 +73,8 @@ def check_mtime(f, mtime):
 
 def update_mtime(f):
     try:
-        __mtime_cache[f] = os.stat(f)[stat.ST_MTIME]
+        res = os.stat(f)
+        __mtime_cache[f] = (res.st_mtime_ns, res.st_size, res.st_ino)
     except OSError:
         if f in __mtime_cache:
             del __mtime_cache[f]

bitbake/lib/bb/parse/ast.py

@@ -211,10 +211,12 @@ class ExportFuncsNode(AstNode):
 
     def eval(self, data):
 
+        sentinel = "    # Export function set\n"
         for func in self.n:
             calledfunc = self.classname + "_" + func
 
-            if data.getVar(func, False) and not data.getVarFlag(func, 'export_func', False):
+            basevar = data.getVar(func, False)
+            if basevar and sentinel not in basevar:
                 continue
 
             if data.getVar(func, False):
@@ -231,12 +233,11 @@ class ExportFuncsNode(AstNode):
             data.setVarFlag(func, "lineno", 1)
 
             if data.getVarFlag(calledfunc, "python", False):
-                data.setVar(func, "    bb.build.exec_func('" + calledfunc + "', d)\n", parsing=True)
+                data.setVar(func, sentinel + "    bb.build.exec_func('" + calledfunc + "', d)\n", parsing=True)
             else:
                 if "-" in self.classname:
                    bb.fatal("The classname %s contains a dash character and is calling an sh function %s using EXPORT_FUNCTIONS. Since a dash is illegal in sh function names, this cannot work, please rename the class or don't use EXPORT_FUNCTIONS." % (self.classname, calledfunc))
-                data.setVar(func, "    " + calledfunc + "\n", parsing=True)
-            data.setVarFlag(func, 'export_func', '1')
+                data.setVar(func, sentinel + "    " + calledfunc + "\n", parsing=True)
 
 class AddTaskNode(AstNode):
     def __init__(self, filename, lineno, func, before, after):
@@ -313,6 +314,16 @@ class InheritNode(AstNode):
     def eval(self, data):
         bb.parse.BBHandler.inherit(self.classes, self.filename, self.lineno, data)
 
+class InheritDeferredNode(AstNode):
+    def __init__(self, filename, lineno, classes):
+        AstNode.__init__(self, filename, lineno)
+        self.inherit = (classes, filename, lineno)
+
+    def eval(self, data):
+        inherits = data.getVar('__BBDEFINHERITS', False) or []
+        inherits.append(self.inherit)
+        data.setVar('__BBDEFINHERITS', inherits)
+
 def handleInclude(statements, filename, lineno, m, force):
     statements.append(IncludeNode(filename, lineno, m.group(1), force))
 
@@ -363,6 +374,10 @@ def handleInherit(statements, filename, lineno, m):
     classes = m.group(1)
     statements.append(InheritNode(filename, lineno, classes))
 
+def handleInheritDeferred(statements, filename, lineno, m):
+    classes = m.group(1)
+    statements.append(InheritDeferredNode(filename, lineno, classes))
+
 def runAnonFuncs(d):
     code = []
     for funcname in d.getVar("__BBANONFUNCS", False) or []:
@@ -429,6 +444,14 @@ def multi_finalize(fn, d):
         logger.debug("Appending .bbappend file %s to %s", append, fn)
         bb.parse.BBHandler.handle(append, d, True)
 
+    while True:
+        inherits = d.getVar('__BBDEFINHERITS', False) or []
+        if not inherits:
+            break
+        inherit, filename, lineno = inherits.pop(0)
+        d.setVar('__BBDEFINHERITS', inherits)
+        bb.parse.BBHandler.inherit(inherit, filename, lineno, d, deferred=True)
+
     onlyfinalise = d.getVar("__ONLYFINALISE", False)
 
     safe_d = d

bitbake/lib/bb/parse/parse_py/BBHandler.py

@@ -21,6 +21,7 @@ from .ConfHandler import include, init
 
 __func_start_regexp__    = re.compile(r"(((?P<py>python(?=(\s|\()))|(?P<fr>fakeroot(?=\s)))\s*)*(?P<func>[\w\.\-\+\{\}\$:]+)?\s*\(\s*\)\s*{$" )
 __inherit_regexp__       = re.compile(r"inherit\s+(.+)" )
+__inherit_def_regexp__   = re.compile(r"inherit_defer\s+(.+)" )
 __export_func_regexp__   = re.compile(r"EXPORT_FUNCTIONS\s+(.+)" )
 __addtask_regexp__       = re.compile(r"addtask\s+(?P<func>\w+)\s*((before\s*(?P<before>((.*(?=after))|(.*))))|(after\s*(?P<after>((.*(?=before))|(.*)))))*")
 __deltask_regexp__       = re.compile(r"deltask\s+(.+)")
@@ -33,6 +34,7 @@ __infunc__ = []
 __inpython__ = False
 __body__   = []
 __classname__ = ""
+__residue__ = []
 
 cached_statements = {}
 
@@ -40,8 +42,10 @@ def supports(fn, d):
     """Return True if fn has a supported extension"""
     return os.path.splitext(fn)[-1] in [".bb", ".bbclass", ".inc"]
 
-def inherit(files, fn, lineno, d):
+def inherit(files, fn, lineno, d, deferred=False):
     __inherit_cache = d.getVar('__inherit_cache', False) or []
+    #if "${" in files and not deferred:
+    #    bb.warn("%s:%s has non deferred conditional inherit" % (fn, lineno))
     files = d.expand(files).split()
     for file in files:
         classtype = d.getVar("__bbclasstype", False)
@@ -77,7 +81,7 @@ def inherit(files, fn, lineno, d):
             __inherit_cache = d.getVar('__inherit_cache', False) or []
 
 def get_statements(filename, absolute_filename, base_name):
-    global cached_statements
+    global cached_statements, __residue__, __body__
 
     try:
         return cached_statements[absolute_filename]
@@ -97,6 +101,11 @@ def get_statements(filename, absolute_filename, base_name):
             # add a blank line to close out any python definition
             feeder(lineno, "", filename, base_name, statements, eof=True)
 
+        if __residue__:
+            raise ParseError("Unparsed lines %s: %s" % (filename, str(__residue__)), filename, lineno)
+        if __body__:
+            raise ParseError("Unparsed lines from unclosed function %s: %s" % (filename, str(__body__)), filename, lineno)
+
         if filename.endswith(".bbclass") or filename.endswith(".inc"):
             cached_statements[absolute_filename] = statements
         return statements
@@ -265,6 +274,11 @@ def feeder(lineno, s, fn, root, statements, eof=False):
         ast.handleInherit(statements, fn, lineno, m)
         return
 
+    m = __inherit_def_regexp__.match(s)
+    if m:
+        ast.handleInheritDeferred(statements, fn, lineno, m)
+        return
+
     return ConfHandler.feeder(lineno, s, fn, statements, conffile=False)
 
 # Add us to the handlers list

bitbake/lib/bb/runqueue.py

@@ -157,7 +157,7 @@ class RunQueueScheduler(object):
             (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
             self.stamps[tid] = bb.parse.siggen.stampfile_mcfn(taskname, taskfn, extrainfo=False)
             if tid in self.rq.runq_buildable:
-                self.buildable.append(tid)
+                self.buildable.add(tid)
 
         self.rev_prio_map = None
         self.is_pressure_usable()
@@ -220,6 +220,16 @@ class RunQueueScheduler(object):
                 bb.note("Pressure status changed to CPU: %s, IO: %s, Mem: %s (CPU: %s/%s, IO: %s/%s, Mem: %s/%s) - using %s/%s bitbake threads" % (pressure_state + pressure_values + (len(self.rq.runq_running.difference(self.rq.runq_complete)), self.rq.number_tasks)))
             self.pressure_state = pressure_state
             return (exceeds_cpu_pressure or exceeds_io_pressure or exceeds_memory_pressure)
+        elif self.rq.max_loadfactor:
+            limit = False
+            loadfactor = float(os.getloadavg()[0]) / os.cpu_count()
+            # bb.warn("Comparing %s to %s" % (loadfactor, self.rq.max_loadfactor))
+            if loadfactor > self.rq.max_loadfactor:
+                limit = True
+            if hasattr(self, "loadfactor_limit") and limit != self.loadfactor_limit:
+                bb.note("Load average limiting set to %s as load average: %s - using %s/%s bitbake threads" % (limit, loadfactor, len(self.rq.runq_running.difference(self.rq.runq_complete)), self.rq.number_tasks))
+            self.loadfactor_limit = limit
+            return limit
         return False
 
     def next_buildable_task(self):
@@ -270,11 +280,11 @@ class RunQueueScheduler(object):
         best = None
         bestprio = None
         for tid in buildable:
-            taskname = taskname_from_tid(tid)
-            if taskname in skip_buildable and skip_buildable[taskname] >= int(self.skip_maxthread[taskname]):
-                continue
             prio = self.rev_prio_map[tid]
             if bestprio is None or bestprio > prio:
+                taskname = taskname_from_tid(tid)
+                if taskname in skip_buildable and skip_buildable[taskname] >= int(self.skip_maxthread[taskname]):
+                    continue
                 stamp = self.stamps[tid]
                 if stamp in self.rq.build_stamps.values():
                     continue
@@ -1004,25 +1014,32 @@ class RunQueueData:
         # Handle --runall
         if self.cooker.configuration.runall:
             # re-run the mark_active and then drop unused tasks from new list
-            reduced_tasklist = set(self.runtaskentries.keys())
-            for tid in list(self.runtaskentries.keys()):
-                if tid not in runq_build:
-                   reduced_tasklist.remove(tid)
-            runq_build = {}
 
-            for task in self.cooker.configuration.runall:
-                if not task.startswith("do_"):
-                    task = "do_{0}".format(task)
+            runall_tids = set()
+            added = True
+            while added:
+                reduced_tasklist = set(self.runtaskentries.keys())
+                for tid in list(self.runtaskentries.keys()):
+                    if tid not in runq_build:
+                       reduced_tasklist.remove(tid)
+                runq_build = {}
+
+                orig = runall_tids
                 runall_tids = set()
-                for tid in reduced_tasklist:
-                    wanttid = "{0}:{1}".format(fn_from_tid(tid), task)
-                    if wanttid in self.runtaskentries:
-                        runall_tids.add(wanttid)
+                for task in self.cooker.configuration.runall:
+                    if not task.startswith("do_"):
+                        task = "do_{0}".format(task)
+                    for tid in reduced_tasklist:
+                        wanttid = "{0}:{1}".format(fn_from_tid(tid), task)
+                        if wanttid in self.runtaskentries:
+                            runall_tids.add(wanttid)
 
-                for tid in list(runall_tids):
-                    mark_active(tid, 1)
-                    if self.cooker.configuration.force:
-                        invalidate_task(tid, False)
+                    for tid in list(runall_tids):
+                        mark_active(tid, 1)
+                        self.target_tids.append(tid)
+                        if self.cooker.configuration.force:
+                            invalidate_task(tid, False)
+                added = runall_tids - orig
 
         delcount = set()
         for tid in list(self.runtaskentries.keys()):
@@ -1256,27 +1273,41 @@ class RunQueueData:
 
         bb.parse.siggen.set_setscene_tasks(self.runq_setscene_tids)
 
+        starttime = time.time()
+        lasttime = starttime
+
         # Iterate over the task list and call into the siggen code
         dealtwith = set()
         todeal = set(self.runtaskentries)
         while todeal:
+            ready = set()
             for tid in todeal.copy():
                 if not (self.runtaskentries[tid].depends - dealtwith):
-                    dealtwith.add(tid)
-                    todeal.remove(tid)
-                    self.prepare_task_hash(tid)
-                bb.event.check_for_interrupts(self.cooker.data)
+                    self.runtaskentries[tid].taskhash_deps = bb.parse.siggen.prep_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches)
+                    # get_taskhash for a given tid *must* be called before get_unihash* below
+                    self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches)
+                    ready.add(tid)
+            unihashes = bb.parse.siggen.get_unihashes(ready)
+            for tid in ready:
+                dealtwith.add(tid)
+                todeal.remove(tid)
+                self.runtaskentries[tid].unihash = unihashes[tid]
+
+            bb.event.check_for_interrupts(self.cooker.data)
+
+            if time.time() > (lasttime + 30):
+                lasttime = time.time()
+                hashequiv_logger.verbose("Initial setup loop progress: %s of %s in %s" % (len(todeal), len(self.runtaskentries), lasttime - starttime))
+
+        endtime = time.time()
+        if (endtime-starttime > 60):
+            hashequiv_logger.verbose("Initial setup loop took: %s" % (endtime-starttime))
 
         bb.parse.siggen.writeout_file_checksum_cache()
 
         #self.dump_data()
         return len(self.runtaskentries)
 
-    def prepare_task_hash(self, tid):
-        bb.parse.siggen.prep_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches)
-        self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches)
-        self.runtaskentries[tid].unihash = bb.parse.siggen.get_unihash(tid)
-
     def dump_data(self):
         """
         Dump some debug information on the internal data structures
@@ -1318,6 +1349,16 @@ class RunQueue:
         self.worker = {}
         self.fakeworker = {}
 
+    @staticmethod
+    def send_pickled_data(worker, data, name):
+        msg = bytearray()
+        msg.extend(b"<" + name.encode() + b">")
+        pickled_data = pickle.dumps(data)
+        msg.extend(len(pickled_data).to_bytes(4, 'big'))
+        msg.extend(pickled_data)
+        msg.extend(b"</" + name.encode() + b">")
+        worker.stdin.write(msg)
+
     def _start_worker(self, mc, fakeroot = False, rqexec = None):
         logger.debug("Starting bitbake-worker")
         magic = "decafbad"
@@ -1332,7 +1373,7 @@ class RunQueue:
             fakerootcmd = shlex.split(mcdata.getVar("FAKEROOTCMD"))
             fakerootenv = (mcdata.getVar("FAKEROOTBASEENV") or "").split()
             env = os.environ.copy()
-            for key, value in (var.split('=') for var in fakerootenv):
+            for key, value in (var.split('=',1) for var in fakerootenv):
                 env[key] = value
             worker = subprocess.Popen(fakerootcmd + [sys.executable, workerscript, magic], stdout=subprocess.PIPE, stdin=subprocess.PIPE, env=env)
             fakerootlogs = self.rqdata.dataCaches[mc].fakerootlogs
@@ -1355,9 +1396,9 @@ class RunQueue:
             "umask" : self.cfgData.getVar("BB_DEFAULT_UMASK"),
         }
 
-        worker.stdin.write(b"<cookerconfig>" + pickle.dumps(self.cooker.configuration) + b"</cookerconfig>")
-        worker.stdin.write(b"<extraconfigdata>" + pickle.dumps(self.cooker.extraconfigdata) + b"</extraconfigdata>")
-        worker.stdin.write(b"<workerdata>" + pickle.dumps(workerdata) + b"</workerdata>")
+        RunQueue.send_pickled_data(worker, self.cooker.configuration, "cookerconfig")
+        RunQueue.send_pickled_data(worker, self.cooker.extraconfigdata, "extraconfigdata")
+        RunQueue.send_pickled_data(worker, workerdata, "workerdata")
         worker.stdin.flush()
 
         return RunQueueWorker(worker, workerpipe)
@@ -1367,7 +1408,7 @@ class RunQueue:
             return
         logger.debug("Teardown for bitbake-worker")
         try:
-           worker.process.stdin.write(b"<quit></quit>")
+           RunQueue.send_pickled_data(worker.process, b"", "quit")
            worker.process.stdin.flush()
            worker.process.stdin.close()
         except IOError:
@@ -1379,12 +1420,12 @@ class RunQueue:
             continue
         worker.pipe.close()
 
-    def start_worker(self):
+    def start_worker(self, rqexec):
         if self.worker:
             self.teardown_workers()
         self.teardown = False
         for mc in self.rqdata.dataCaches:
-            self.worker[mc] = self._start_worker(mc)
+            self.worker[mc] = self._start_worker(mc, False, rqexec)
 
     def start_fakeworker(self, rqexec, mc):
         if not mc in self.fakeworker:
@@ -1544,6 +1585,9 @@ class RunQueue:
                                          ('bb.event.HeartbeatEvent',), data=self.cfgData)
                  self.dm_event_handler_registered = True
 
+            self.rqdata.init_progress_reporter.next_stage()
+            self.rqexe = RunQueueExecute(self)
+
             dump = self.cooker.configuration.dump_signatures
             if dump:
                 self.rqdata.init_progress_reporter.finish()
@@ -1555,10 +1599,8 @@ class RunQueue:
                 self.state = runQueueComplete
 
         if self.state is runQueueSceneInit:
-            self.rqdata.init_progress_reporter.next_stage()
-            self.start_worker()
-            self.rqdata.init_progress_reporter.next_stage()
-            self.rqexe = RunQueueExecute(self)
+            self.start_worker(self.rqexe)
+            self.rqdata.init_progress_reporter.finish()
 
             # If we don't have any setscene functions, skip execution
             if not self.rqdata.runq_setscene_tids:
@@ -1673,6 +1715,17 @@ class RunQueue:
         return
 
     def print_diffscenetasks(self):
+        def get_root_invalid_tasks(task, taskdepends, valid, noexec, visited_invalid):
+            invalidtasks = []
+            for t in taskdepends[task].depends:
+                if t not in valid and t not in visited_invalid:
+                    invalidtasks.extend(get_root_invalid_tasks(t, taskdepends, valid, noexec, visited_invalid))
+                    visited_invalid.add(t)
+
+            direct_invalid = [t for t in taskdepends[task].depends if t not in valid]
+            if not direct_invalid and task not in noexec:
+                invalidtasks = [task]
+            return invalidtasks
 
         noexec = []
         tocheck = set()
@@ -1706,46 +1759,49 @@ class RunQueue:
                     valid_new.add(dep)
 
         invalidtasks = set()
-        for tid in self.rqdata.runtaskentries:
-            if tid not in valid_new and tid not in noexec:
-                invalidtasks.add(tid)
 
-        found = set()
-        processed = set()
-        for tid in invalidtasks:
+        toptasks = set(["{}:{}".format(t[3], t[2]) for t in self.rqdata.targets])
+        for tid in toptasks:
             toprocess = set([tid])
             while toprocess:
                 next = set()
+                visited_invalid = set()
                 for t in toprocess:
-                    for dep in self.rqdata.runtaskentries[t].depends:
-                        if dep in invalidtasks:
-                            found.add(tid)
-                        if dep not in processed:
-                            processed.add(dep)
+                    if t not in valid_new and t not in noexec:
+                        invalidtasks.update(get_root_invalid_tasks(t, self.rqdata.runtaskentries, valid_new, noexec, visited_invalid))
+                        continue
+                    if t in self.rqdata.runq_setscene_tids:
+                        for dep in self.rqexe.sqdata.sq_deps[t]:
                             next.add(dep)
+                        continue
+
+                    for dep in self.rqdata.runtaskentries[t].depends:
+                        next.add(dep)
+
                 toprocess = next
-                if tid in found:
-                    toprocess = set()
 
         tasklist = []
-        for tid in invalidtasks.difference(found):
+        for tid in invalidtasks:
             tasklist.append(tid)
 
         if tasklist:
             bb.plain("The differences between the current build and any cached tasks start at the following tasks:\n" + "\n".join(tasklist))
 
-        return invalidtasks.difference(found)
+        return invalidtasks
 
     def write_diffscenetasks(self, invalidtasks):
+        bb.siggen.check_siggen_version(bb.siggen)
 
         # Define recursion callback
         def recursecb(key, hash1, hash2):
             hashes = [hash1, hash2]
+            bb.debug(1, "Recursively looking for recipe {} hashes {}".format(key, hashes))
             hashfiles = bb.siggen.find_siginfo(key, None, hashes, self.cfgData)
+            bb.debug(1, "Found hashfiles:\n{}".format(hashfiles))
 
             recout = []
             if len(hashfiles) == 2:
-                out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb)
+                out2 = bb.siggen.compare_sigfiles(hashfiles[hash1]['path'], hashfiles[hash2]['path'], recursecb)
                 recout.extend(list('    ' + l for l in out2))
             else:
                 recout.append("Unable to find matching sigdata for %s with hashes %s or %s" % (key, hash1, hash2))
@@ -1756,20 +1812,25 @@ class RunQueue:
         for tid in invalidtasks:
             (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
             pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn]
-            h = self.rqdata.runtaskentries[tid].hash
+            h = self.rqdata.runtaskentries[tid].unihash
+            bb.debug(1, "Looking for recipe {} task {}".format(pn, taskname))
             matches = bb.siggen.find_siginfo(pn, taskname, [], self.cooker.databuilder.mcdata[mc])
+            bb.debug(1, "Found hashfiles:\n{}".format(matches))
             match = None
-            for m in matches:
-                if h in m:
-                    match = m
+            for m in matches.values():
+                if h in m['path']:
+                    match = m['path']
             if match is None:
-                bb.fatal("Can't find a task we're supposed to have written out? (hash: %s)?" % h)
+                bb.fatal("Can't find a task we're supposed to have written out? (hash: %s tid: %s)?" % (h, tid))
             matches = {k : v for k, v in iter(matches.items()) if h not in k}
+            matches_local = {k : v for k, v in iter(matches.items()) if h not in k and not v['sstate']}
+            if matches_local:
+                matches = matches_local
             if matches:
-                latestmatch = sorted(matches.keys(), key=lambda f: matches[f])[-1]
+                latestmatch = matches[sorted(matches.keys(), key=lambda h: matches[h]['time'])[-1]]['path']
                 prevh = __find_sha256__.search(latestmatch).group(0)
                 output = bb.siggen.compare_sigfiles(latestmatch, match, recursecb)
-                bb.plain("\nTask %s:%s couldn't be used from the cache because:\n  We need hash %s, closest matching task was %s\n  " % (pn, taskname, h, prevh) + '\n  '.join(output))
+                bb.plain("\nTask %s:%s couldn't be used from the cache because:\n  We need hash %s, most recent matching task was %s\n  " % (pn, taskname, h, prevh) + '\n  '.join(output))
 
 
 class RunQueueExecute:
@@ -1785,6 +1846,7 @@ class RunQueueExecute:
         self.max_cpu_pressure = self.cfgData.getVar("BB_PRESSURE_MAX_CPU")
         self.max_io_pressure = self.cfgData.getVar("BB_PRESSURE_MAX_IO")
         self.max_memory_pressure = self.cfgData.getVar("BB_PRESSURE_MAX_MEMORY")
+        self.max_loadfactor = self.cfgData.getVar("BB_LOADFACTOR_MAX")
 
         self.sq_buildable = set()
         self.sq_running = set()
@@ -1802,6 +1864,8 @@ class RunQueueExecute:
         self.build_stamps2 = []
         self.failed_tids = []
         self.sq_deferred = {}
+        self.sq_needed_harddeps = set()
+        self.sq_harddep_deferred = set()
 
         self.stampcache = {}
 
@@ -1811,11 +1875,6 @@ class RunQueueExecute:
 
         self.stats = RunQueueStats(len(self.rqdata.runtaskentries), len(self.rqdata.runq_setscene_tids))
 
-        for mc in rq.worker:
-            rq.worker[mc].pipe.setrunqueueexec(self)
-        for mc in rq.fakeworker:
-            rq.fakeworker[mc].pipe.setrunqueueexec(self)
-
         if self.number_tasks <= 0:
              bb.fatal("Invalid BB_NUMBER_THREADS %s" % self.number_tasks)
 
@@ -1841,6 +1900,11 @@ class RunQueueExecute:
                 bb.fatal("Invalid BB_PRESSURE_MAX_MEMORY %s, minimum value is %s." % (self.max_memory_pressure, lower_limit))
             if self.max_memory_pressure > upper_limit:
                 bb.warn("Your build will be largely unregulated since BB_PRESSURE_MAX_MEMORY is set to %s. It is very unlikely that such high pressure will be experienced." % (self.max_io_pressure))
+
+        if self.max_loadfactor:
+            self.max_loadfactor = float(self.max_loadfactor)
+            if self.max_loadfactor <= 0:
+                bb.fatal("Invalid BB_LOADFACTOR_MAX %s, needs to be greater than zero." % (self.max_loadfactor))
             
         # List of setscene tasks which we've covered
         self.scenequeue_covered = set()
@@ -1851,11 +1915,6 @@ class RunQueueExecute:
         self.tasks_notcovered = set()
         self.scenequeue_notneeded = set()
 
-        # We can't skip specified target tasks which aren't setscene tasks
-        self.cantskip = set(self.rqdata.target_tids)
-        self.cantskip.difference_update(self.rqdata.runq_setscene_tids)
-        self.cantskip.intersection_update(self.rqdata.runtaskentries)
-
         schedulers = self.get_schedulers()
         for scheduler in schedulers:
             if self.scheduler == scheduler.name:
@@ -1868,7 +1927,25 @@ class RunQueueExecute:
 
         #if self.rqdata.runq_setscene_tids:
         self.sqdata = SQData()
-        build_scenequeue_data(self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self)
+        build_scenequeue_data(self.sqdata, self.rqdata, self)
+
+        update_scenequeue_data(self.sqdata.sq_revdeps, self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self, summary=True)
+
+        # Compute a list of 'stale' sstate tasks where the current hash does not match the one
+        # in any stamp files. Pass the list out to metadata as an event.
+        found = {}
+        for tid in self.rqdata.runq_setscene_tids:
+            (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
+            stamps = bb.build.find_stale_stamps(taskname, taskfn)
+            if stamps:
+                if mc not in found:
+                    found[mc] = {}
+                found[mc][tid] = stamps
+        for mc in found:
+            event = bb.event.StaleSetSceneTasks(found[mc])
+            bb.event.fire(event, self.cooker.databuilder.mcdata[mc])
+
+        self.build_taskdepdata_cache()
 
     def runqueue_process_waitpid(self, task, status, fakerootlog=None):
 
@@ -1894,14 +1971,14 @@ class RunQueueExecute:
     def finish_now(self):
         for mc in self.rq.worker:
             try:
-                self.rq.worker[mc].process.stdin.write(b"<finishnow></finishnow>")
+                RunQueue.send_pickled_data(self.rq.worker[mc].process, b"", "finishnow")
                 self.rq.worker[mc].process.stdin.flush()
             except IOError:
                 # worker must have died?
                 pass
         for mc in self.rq.fakeworker:
             try:
-                self.rq.fakeworker[mc].process.stdin.write(b"<finishnow></finishnow>")
+                RunQueue.send_pickled_data(self.rq.fakeworker[mc].process, b"", "finishnow")
                 self.rq.fakeworker[mc].process.stdin.flush()
             except IOError:
                 # worker must have died?
@@ -2117,8 +2194,11 @@ class RunQueueExecute:
         if not self.sqdone and self.can_start_task():
             # Find the next setscene to run
             for nexttask in self.sorted_setscene_tids:
-                if nexttask in self.sq_buildable and nexttask not in self.sq_running and self.sqdata.stamps[nexttask] not in self.build_stamps.values():
-                    if nexttask not in self.sqdata.unskippable and self.sqdata.sq_revdeps[nexttask] and self.sqdata.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sqdata.sq_revdeps[nexttask]):
+                if nexttask in self.sq_buildable and nexttask not in self.sq_running and self.sqdata.stamps[nexttask] not in self.build_stamps.values() and nexttask not in self.sq_harddep_deferred:
+                    if nexttask not in self.sqdata.unskippable and self.sqdata.sq_revdeps[nexttask] and \
+                            nexttask not in self.sq_needed_harddeps and \
+                            self.sqdata.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and \
+                            self.check_dependencies(nexttask, self.sqdata.sq_revdeps[nexttask]):
                         if nexttask not in self.rqdata.target_tids:
                             logger.debug2("Skipping setscene for task %s" % nexttask)
                             self.sq_task_skip(nexttask)
@@ -2126,6 +2206,19 @@ class RunQueueExecute:
                             if nexttask in self.sq_deferred:
                                 del self.sq_deferred[nexttask]
                             return True
+                    if nexttask in self.sqdata.sq_harddeps_rev and not self.sqdata.sq_harddeps_rev[nexttask].issubset(self.scenequeue_covered | self.scenequeue_notcovered):
+                        logger.debug2("Deferring %s due to hard dependencies" % nexttask)
+                        updated = False
+                        for dep in self.sqdata.sq_harddeps_rev[nexttask]:
+                            if dep not in self.sq_needed_harddeps:
+                                logger.debug2("Enabling task %s as it is a hard dependency" % dep)
+                                self.sq_buildable.add(dep)
+                                self.sq_needed_harddeps.add(dep)
+                                updated = True
+                        self.sq_harddep_deferred.add(nexttask)
+                        if updated:
+                            return True
+                        continue
                     # If covered tasks are running, need to wait for them to complete
                     for t in self.sqdata.sq_covered_tasks[nexttask]:
                         if t in self.runq_running and t not in self.runq_complete:
@@ -2196,10 +2289,10 @@ class RunQueueExecute:
             if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run:
                 if not mc in self.rq.fakeworker:
                     self.rq.start_fakeworker(self, mc)
-                self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps(runtask) + b"</runtask>")
+                RunQueue.send_pickled_data(self.rq.fakeworker[mc].process, runtask, "runtask")
                 self.rq.fakeworker[mc].process.stdin.flush()
             else:
-                self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps(runtask) + b"</runtask>")
+                RunQueue.send_pickled_data(self.rq.worker[mc].process, runtask, "runtask")
                 self.rq.worker[mc].process.stdin.flush()
 
             self.build_stamps[task] = bb.parse.siggen.stampfile_mcfn(taskname, taskfn, extrainfo=False)
@@ -2297,10 +2390,10 @@ class RunQueueExecute:
                         self.rq.state = runQueueFailed
                         self.stats.taskFailed()
                         return True
-                self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps(runtask) + b"</runtask>")
+                RunQueue.send_pickled_data(self.rq.fakeworker[mc].process, runtask, "runtask")
                 self.rq.fakeworker[mc].process.stdin.flush()
             else:
-                self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps(runtask) + b"</runtask>")
+                RunQueue.send_pickled_data(self.rq.worker[mc].process, runtask, "runtask")
                 self.rq.worker[mc].process.stdin.flush()
 
             self.build_stamps[task] = bb.parse.siggen.stampfile_mcfn(taskname, taskfn, extrainfo=False)
@@ -2354,6 +2447,25 @@ class RunQueueExecute:
             ret.add(dep)
         return ret
 
+    # Build the individual cache entries in advance once to save time
+    def build_taskdepdata_cache(self):
+        taskdepdata_cache = {}
+        for task in self.rqdata.runtaskentries:
+            (mc, fn, taskname, taskfn) = split_tid_mcfn(task)
+            taskdepdata_cache[task] = bb.TaskData(
+                pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn],
+                taskname = taskname,
+                fn = fn,
+                deps = self.filtermcdeps(task, mc, self.rqdata.runtaskentries[task].depends),
+                provides = self.rqdata.dataCaches[mc].fn_provides[taskfn],
+                taskhash = self.rqdata.runtaskentries[task].hash,
+                unihash = self.rqdata.runtaskentries[task].unihash,
+                hashfn = self.rqdata.dataCaches[mc].hashfn[taskfn],
+                taskhash_deps = self.rqdata.runtaskentries[task].taskhash_deps,
+            )
+
+        self.taskdepdata_cache = taskdepdata_cache
+
     # We filter out multiconfig dependencies from taskdepdata we pass to the tasks
     # as most code can't handle them
     def build_taskdepdata(self, task):
@@ -2365,16 +2477,11 @@ class RunQueueExecute:
         while next:
             additional = []
             for revdep in next:
-                (mc, fn, taskname, taskfn) = split_tid_mcfn(revdep)
-                pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn]
-                deps = self.rqdata.runtaskentries[revdep].depends
-                provides = self.rqdata.dataCaches[mc].fn_provides[taskfn]
-                taskhash = self.rqdata.runtaskentries[revdep].hash
-                unihash = self.rqdata.runtaskentries[revdep].unihash
-                deps = self.filtermcdeps(task, mc, deps)
-                hashfn = self.rqdata.dataCaches[mc].hashfn[taskfn]
-                taskdepdata[revdep] = [pn, taskname, fn, deps, provides, taskhash, unihash, hashfn]
-                for revdep2 in deps:
+                self.taskdepdata_cache[revdep] = self.taskdepdata_cache[revdep]._replace(
+                    unihash=self.rqdata.runtaskentries[revdep].unihash
+                )
+                taskdepdata[revdep] = self.taskdepdata_cache[revdep]
+                for revdep2 in self.taskdepdata_cache[revdep].deps:
                     if revdep2 not in taskdepdata:
                         additional.append(revdep2)
             next = additional
@@ -2388,7 +2495,7 @@ class RunQueueExecute:
             return
 
         notcovered = set(self.scenequeue_notcovered)
-        notcovered |= self.cantskip
+        notcovered |= self.sqdata.cantskip
         for tid in self.scenequeue_notcovered:
             notcovered |= self.sqdata.sq_covered_tasks[tid]
         notcovered |= self.sqdata.unskippable.difference(self.rqdata.runq_setscene_tids)
@@ -2468,17 +2575,28 @@ class RunQueueExecute:
             elif self.rqdata.runtaskentries[p].depends.isdisjoint(total):
                 next.add(p)
 
+        starttime = time.time()
+        lasttime = starttime
+
         # When an item doesn't have dependencies in total, we can process it. Drop items from total when handled
         while next:
             current = next.copy()
             next = set()
+            ready = {}
             for tid in current:
                 if self.rqdata.runtaskentries[p].depends and not self.rqdata.runtaskentries[tid].depends.isdisjoint(total):
                     continue
+                # get_taskhash for a given tid *must* be called before get_unihash* below
+                ready[tid] = bb.parse.siggen.get_taskhash(tid, self.rqdata.runtaskentries[tid].depends, self.rqdata.dataCaches)
+
+            unihashes = bb.parse.siggen.get_unihashes(ready.keys())
+
+            for tid in ready:
                 orighash = self.rqdata.runtaskentries[tid].hash
-                newhash = bb.parse.siggen.get_taskhash(tid, self.rqdata.runtaskentries[tid].depends, self.rqdata.dataCaches)
+                newhash = ready[tid]
                 origuni = self.rqdata.runtaskentries[tid].unihash
-                newuni = bb.parse.siggen.get_unihash(tid)
+                newuni = unihashes[tid]
+
                 # FIXME, need to check it can come from sstate at all for determinism?
                 remapped = False
                 if newuni == origuni:
@@ -2499,12 +2617,21 @@ class RunQueueExecute:
                 next |= self.rqdata.runtaskentries[tid].revdeps
                 total.remove(tid)
                 next.intersection_update(total)
+                bb.event.check_for_interrupts(self.cooker.data)
+
+                if time.time() > (lasttime + 30):
+                    lasttime = time.time()
+                    hashequiv_logger.verbose("Rehash loop slow progress: %s in %s" % (len(total), lasttime - starttime))
+
+        endtime = time.time()
+        if (endtime-starttime > 60):
+            hashequiv_logger.verbose("Rehash loop took more than 60s: %s" % (endtime-starttime))
 
         if changed:
             for mc in self.rq.worker:
-                self.rq.worker[mc].process.stdin.write(b"<newtaskhashes>" + pickle.dumps(bb.parse.siggen.get_taskhashes()) + b"</newtaskhashes>")
+                RunQueue.send_pickled_data(self.rq.worker[mc].process, bb.parse.siggen.get_taskhashes(), "newtaskhashes")
             for mc in self.rq.fakeworker:
-                self.rq.fakeworker[mc].process.stdin.write(b"<newtaskhashes>" + pickle.dumps(bb.parse.siggen.get_taskhashes()) + b"</newtaskhashes>")
+                RunQueue.send_pickled_data(self.rq.fakeworker[mc].process, bb.parse.siggen.get_taskhashes(), "newtaskhashes")
 
             hashequiv_logger.debug(pprint.pformat("Tasks changed:\n%s" % (changed)))
 
@@ -2574,8 +2701,8 @@ class RunQueueExecute:
         update_tasks2 = []
         for tid in update_tasks:
             harddepfail = False
-            for t in self.sqdata.sq_harddeps:
-                if tid in self.sqdata.sq_harddeps[t] and t in self.scenequeue_notcovered:
+            for t in self.sqdata.sq_harddeps_rev[tid]:
+                if t in self.scenequeue_notcovered:
                     harddepfail = True
                     break
             if not harddepfail and self.sqdata.sq_revdeps[tid].issubset(self.scenequeue_covered | self.scenequeue_notcovered):
@@ -2607,12 +2734,14 @@ class RunQueueExecute:
 
         if changed:
             self.stats.updateCovered(len(self.scenequeue_covered), len(self.scenequeue_notcovered))
+            self.sq_needed_harddeps = set()
+            self.sq_harddep_deferred = set()
             self.holdoff_need_update = True
 
     def scenequeue_updatecounters(self, task, fail=False):
 
-        for dep in sorted(self.sqdata.sq_deps[task]):
-            if fail and task in self.sqdata.sq_harddeps and dep in self.sqdata.sq_harddeps[task]:
+        if fail and task in self.sqdata.sq_harddeps:
+            for dep in sorted(self.sqdata.sq_harddeps[task]):
                 if dep in self.scenequeue_covered or dep in self.scenequeue_notcovered:
                     # dependency could be already processed, e.g. noexec setscene task
                     continue
@@ -2622,6 +2751,7 @@ class RunQueueExecute:
                 logger.debug2("%s was unavailable and is a hard dependency of %s so skipping" % (task, dep))
                 self.sq_task_failoutright(dep)
                 continue
+        for dep in sorted(self.sqdata.sq_deps[task]):
             if self.sqdata.sq_revdeps[dep].issubset(self.scenequeue_covered | self.scenequeue_notcovered):
                 if dep not in self.sq_buildable:
                     self.sq_buildable.add(dep)
@@ -2640,6 +2770,13 @@ class RunQueueExecute:
                         new.add(dep)
             next = new
 
+        # If this task was one which other setscene tasks have a hard dependency upon, we need
+        # to walk through the hard dependencies and allow execution of those which have completed dependencies.
+        if task in self.sqdata.sq_harddeps:
+            for dep in self.sq_harddep_deferred.copy():
+                if self.sqdata.sq_harddeps_rev[dep].issubset(self.scenequeue_covered | self.scenequeue_notcovered):
+                    self.sq_harddep_deferred.remove(dep)
+
         self.stats.updateCovered(len(self.scenequeue_covered), len(self.scenequeue_notcovered))
         self.holdoff_need_update = True
 
@@ -2708,13 +2845,19 @@ class RunQueueExecute:
             additional = []
             for revdep in next:
                 (mc, fn, taskname, taskfn) = split_tid_mcfn(revdep)
-                pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn]
                 deps = getsetscenedeps(revdep)
-                provides = self.rqdata.dataCaches[mc].fn_provides[taskfn]
-                taskhash = self.rqdata.runtaskentries[revdep].hash
-                unihash = self.rqdata.runtaskentries[revdep].unihash
-                hashfn = self.rqdata.dataCaches[mc].hashfn[taskfn]
-                taskdepdata[revdep] = [pn, taskname, fn, deps, provides, taskhash, unihash, hashfn]
+
+                taskdepdata[revdep] = bb.TaskData(
+                    pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn],
+                    taskname = taskname,
+                    fn = fn,
+                    deps = deps,
+                    provides = self.rqdata.dataCaches[mc].fn_provides[taskfn],
+                    taskhash = self.rqdata.runtaskentries[revdep].hash,
+                    unihash = self.rqdata.runtaskentries[revdep].unihash,
+                    hashfn = self.rqdata.dataCaches[mc].hashfn[taskfn],
+                    taskhash_deps = self.rqdata.runtaskentries[revdep].taskhash_deps,
+                )
                 for revdep2 in deps:
                     if revdep2 not in taskdepdata:
                         additional.append(revdep2)
@@ -2758,6 +2901,7 @@ class SQData(object):
         self.sq_revdeps = {}
         # Injected inter-setscene task dependencies
         self.sq_harddeps = {}
+        self.sq_harddeps_rev = {}
         # Cache of stamp files so duplicates can't run in parallel
         self.stamps = {}
         # Setscene tasks directly depended upon by the build
@@ -2767,12 +2911,17 @@ class SQData(object):
         # A list of normal tasks a setscene task covers
         self.sq_covered_tasks = {}
 
-def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
+def build_scenequeue_data(sqdata, rqdata, sqrq):
 
     sq_revdeps = {}
     sq_revdeps_squash = {}
     sq_collated_deps = {}
 
+    # We can't skip specified target tasks which aren't setscene tasks
+    sqdata.cantskip = set(rqdata.target_tids)
+    sqdata.cantskip.difference_update(rqdata.runq_setscene_tids)
+    sqdata.cantskip.intersection_update(rqdata.runtaskentries)
+
     # We need to construct a dependency graph for the setscene functions. Intermediate
     # dependencies between the setscene tasks only complicate the code. This code
     # therefore aims to collapse the huge runqueue dependency tree into a smaller one
@@ -2841,7 +2990,7 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
     for tid in rqdata.runtaskentries:
         if not rqdata.runtaskentries[tid].revdeps:
             sqdata.unskippable.add(tid)
-    sqdata.unskippable |= sqrq.cantskip
+    sqdata.unskippable |= sqdata.cantskip
     while new:
         new = False
         orig = sqdata.unskippable.copy()
@@ -2880,6 +3029,7 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
         idepends = rqdata.taskData[mc].taskentries[realtid].idepends
         sqdata.stamps[tid] = bb.parse.siggen.stampfile_mcfn(taskname, taskfn, extrainfo=False)
 
+        sqdata.sq_harddeps_rev[tid] = set()
         for (depname, idependtask) in idepends:
 
             if depname not in rqdata.taskData[mc].build_targets:
@@ -2892,20 +3042,15 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
             if deptid not in rqdata.runtaskentries:
                 bb.msg.fatal("RunQueue", "Task %s depends upon non-existent task %s:%s" % (realtid, depfn, idependtask))
 
+            logger.debug2("Adding hard setscene dependency %s for %s" % (deptid, tid))
+
             if not deptid in sqdata.sq_harddeps:
                 sqdata.sq_harddeps[deptid] = set()
             sqdata.sq_harddeps[deptid].add(tid)
-
-            sq_revdeps_squash[tid].add(deptid)
-            # Have to zero this to avoid circular dependencies
-            sq_revdeps_squash[deptid] = set()
+            sqdata.sq_harddeps_rev[tid].add(deptid)
 
     rqdata.init_progress_reporter.next_stage()
 
-    for task in sqdata.sq_harddeps:
-        for dep in sqdata.sq_harddeps[task]:
-            sq_revdeps_squash[dep].add(task)
-
     rqdata.init_progress_reporter.next_stage()
 
     #for tid in sq_revdeps_squash:
@@ -2932,7 +3077,7 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
         if not sqdata.sq_revdeps[tid]:
             sqrq.sq_buildable.add(tid)
 
-    rqdata.init_progress_reporter.finish()
+    rqdata.init_progress_reporter.next_stage()
 
     sqdata.noexec = set()
     sqdata.stamppresent = set()
@@ -2951,22 +3096,6 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
                 sqrq.sq_deferred[tid] = sqdata.hashes[h]
                 bb.debug(1, "Deferring %s after %s" % (tid, sqdata.hashes[h]))
 
-    update_scenequeue_data(sqdata.sq_revdeps, sqdata, rqdata, rq, cooker, stampcache, sqrq, summary=True)
-
-    # Compute a list of 'stale' sstate tasks where the current hash does not match the one
-    # in any stamp files. Pass the list out to metadata as an event.
-    found = {}
-    for tid in rqdata.runq_setscene_tids:
-        (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
-        stamps = bb.build.find_stale_stamps(taskname, taskfn)
-        if stamps:
-            if mc not in found:
-                found[mc] = {}
-            found[mc][tid] = stamps
-    for mc in found:
-        event = bb.event.StaleSetSceneTasks(found[mc])
-        bb.event.fire(event, cooker.databuilder.mcdata[mc])
-
 def check_setscene_stamps(tid, rqdata, rq, stampcache, noexecstamp=False):
 
     (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
@@ -3167,9 +3296,6 @@ class runQueuePipe():
         self.rqexec = rqexec
         self.fakerootlogs = fakerootlogs
 
-    def setrunqueueexec(self, rqexec):
-        self.rqexec = rqexec
-
     def read(self):
         for workers, name in [(self.rq.worker, "Worker"), (self.rq.fakeworker, "Fakeroot")]:
             for worker in workers.values():

bitbake/lib/bb/server/process.py

@@ -402,6 +402,22 @@ class ProcessServer():
                 serverlog("".join(msg))
 
     def idle_thread(self):
+        if self.cooker.configuration.profile:
+            try:
+                import cProfile as profile
+            except:
+                import profile
+            prof = profile.Profile()
+
+            ret = profile.Profile.runcall(prof, self.idle_thread_internal)
+
+            prof.dump_stats("profile-mainloop.log")
+            bb.utils.process_profilelog("profile-mainloop.log")
+            serverlog("Raw profiling information saved to profile-mainloop.log and processed statistics to profile-mainloop.log.processed")
+        else:
+            self.idle_thread_internal()
+
+    def idle_thread_internal(self):
         def remove_idle_func(function):
             with bb.utils.lock_timeout(self._idlefuncsLock):
                 del self._idlefuns[function]
@@ -500,12 +516,18 @@ class ServerCommunicator():
         self.recv = recv
 
     def runCommand(self, command):
-        self.connection.send(command)
+        try:
+            self.connection.send(command)
+        except BrokenPipeError as e:
+            raise BrokenPipeError("bitbake-server might have died or been forcibly stopped, ie. OOM killed") from e
         if not self.recv.poll(30):
             logger.info("No reply from server in 30s (for command %s at %s)" % (command[0], currenttime()))
             if not self.recv.poll(30):
                 raise ProcessTimeout("Timeout while waiting for a reply from the bitbake server (60s at %s)" % currenttime())
-        ret, exc = self.recv.get()
+        try:
+            ret, exc = self.recv.get()
+        except EOFError as e:
+            raise EOFError("bitbake-server might have died or been forcibly stopped, ie. OOM killed") from e
         # Should probably turn all exceptions in exc back into exceptions?
         # For now, at least handle BBHandledException
         if exc and ("BBHandledException" in exc or "SystemExit" in exc):

bitbake/lib/bb/siggen.py

@@ -15,6 +15,7 @@ import difflib
 import simplediff
 import json
 import types
+from contextlib import contextmanager
 import bb.compress.zstd
 from bb.checksum import FileChecksumCache
 from bb import runqueue
@@ -24,6 +25,24 @@ import hashserv.client
 logger = logging.getLogger('BitBake.SigGen')
 hashequiv_logger = logging.getLogger('BitBake.SigGen.HashEquiv')
 
+#find_siginfo and find_siginfo_version are set by the metadata siggen
+# The minimum version of the find_siginfo function we need
+find_siginfo_minversion = 2
+
+HASHSERV_ENVVARS = [
+    "SSL_CERT_DIR",
+    "SSL_CERT_FILE",
+    "NO_PROXY",
+    "HTTPS_PROXY",
+    "HTTP_PROXY"
+]
+
+def check_siggen_version(siggen):
+    if not hasattr(siggen, "find_siginfo_version"):
+        bb.fatal("Siggen from metadata (OE-Core?) is too old, please update it (no version found)")
+    if siggen.find_siginfo_version < siggen.find_siginfo_minversion:
+        bb.fatal("Siggen from metadata (OE-Core?) is too old, please update it (%s vs %s)" % (siggen.find_siginfo_version, siggen.find_siginfo_minversion))
+
 class SetEncoder(json.JSONEncoder):
     def default(self, obj):
         if isinstance(obj, set) or isinstance(obj, frozenset):
@@ -92,9 +111,18 @@ class SignatureGenerator(object):
             if flag:
                 self.datacaches[mc].stamp_extrainfo[mcfn][t] = flag
 
+    def get_cached_unihash(self, tid):
+        return None
+
     def get_unihash(self, tid):
+        unihash = self.get_cached_unihash(tid)
+        if unihash:
+            return unihash
         return self.taskhash[tid]
 
+    def get_unihashes(self, tids):
+        return {tid: self.get_unihash(tid) for tid in tids}
+
     def prep_taskhash(self, tid, deps, dataCaches):
         return
 
@@ -353,7 +381,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
             self.taints[tid] = taint
             logger.warning("%s is tainted from a forced run" % tid)
 
-        return
+        return set(dep for _, dep in self.runtaskdeps[tid])
 
     def get_taskhash(self, tid, deps, dataCaches):
 
@@ -511,32 +539,86 @@ class SignatureGeneratorBasicHash(SignatureGeneratorBasic):
 class SignatureGeneratorUniHashMixIn(object):
     def __init__(self, data):
         self.extramethod = {}
+        # NOTE: The cache only tracks hashes that exist. Hashes that don't
+        # exist are always queries from the server since it is possible for
+        # hashes to appear over time, but much less likely for them to
+        # disappear
+        self.unihash_exists_cache = set()
+        self.username = None
+        self.password = None
+        self.env = {}
+
+        origenv = data.getVar("BB_ORIGENV")
+        for e in HASHSERV_ENVVARS:
+            value = data.getVar(e)
+            if not value and origenv:
+                value = origenv.getVar(e)
+            if value:
+                self.env[e] = value
         super().__init__(data)
 
     def get_taskdata(self):
-        return (self.server, self.method, self.extramethod) + super().get_taskdata()
+        return (self.server, self.method, self.extramethod, self.max_parallel, self.username, self.password, self.env) + super().get_taskdata()
 
     def set_taskdata(self, data):
-        self.server, self.method, self.extramethod = data[:3]
-        super().set_taskdata(data[3:])
+        self.server, self.method, self.extramethod, self.max_parallel, self.username, self.password, self.env = data[:7]
+        super().set_taskdata(data[7:])
 
+    def get_hashserv_creds(self):
+        if self.username and self.password:
+            return {
+                "username": self.username,
+                "password": self.password,
+            }
+
+        return {}
+
+    @contextmanager
+    def _client_env(self):
+        orig_env = os.environ.copy()
+        try:
+            for k, v in self.env.items():
+                os.environ[k] = v
+
+            yield
+        finally:
+            for k, v in self.env.items():
+                if k in orig_env:
+                    os.environ[k] = orig_env[k]
+                else:
+                    del os.environ[k]
+
+    @contextmanager
     def client(self):
-        if getattr(self, '_client', None) is None:
-            self._client = hashserv.create_client(self.server)
-        return self._client
+        with self._client_env():
+            if getattr(self, '_client', None) is None:
+                self._client = hashserv.create_client(self.server, **self.get_hashserv_creds())
+            yield self._client
+
+    @contextmanager
+    def client_pool(self):
+        with self._client_env():
+            if getattr(self, '_client_pool', None) is None:
+                self._client_pool = hashserv.client.ClientPool(self.server, self.max_parallel, **self.get_hashserv_creds())
+            yield self._client_pool
 
     def reset(self, data):
-        if getattr(self, '_client', None) is not None:
-            self._client.close()
-            self._client = None 
+        self.__close_clients()
         return super().reset(data)
 
     def exit(self):
-        if getattr(self, '_client', None) is not None:
-            self._client.close()
-            self._client = None
+        self.__close_clients()
         return super().exit()
 
+    def __close_clients(self):
+        with self._client_env():
+            if getattr(self, '_client', None) is not None:
+                self._client.close()
+                self._client = None
+            if getattr(self, '_client_pool', None) is not None:
+                self._client_pool.close()
+                self._client_pool = None
+
     def get_stampfile_hash(self, tid):
         if tid in self.taskhash:
             # If a unique hash is reported, use it as the stampfile hash. This
@@ -568,7 +650,7 @@ class SignatureGeneratorUniHashMixIn(object):
             return None
         return unihash
 
-    def get_unihash(self, tid):
+    def get_cached_unihash(self, tid):
         taskhash = self.taskhash[tid]
 
         # If its not a setscene task we can return
@@ -583,40 +665,108 @@ class SignatureGeneratorUniHashMixIn(object):
             self.unihash[tid] = unihash
             return unihash
 
-        # In the absence of being able to discover a unique hash from the
-        # server, make it be equivalent to the taskhash. The unique "hash" only
-        # really needs to be a unique string (not even necessarily a hash), but
-        # making it match the taskhash has a few advantages:
-        #
-        # 1) All of the sstate code that assumes hashes can be the same
-        # 2) It provides maximal compatibility with builders that don't use
-        #    an equivalency server
-        # 3) The value is easy for multiple independent builders to derive the
-        #    same unique hash from the same input. This means that if the
-        #    independent builders find the same taskhash, but it isn't reported
-        #    to the server, there is a better chance that they will agree on
-        #    the unique hash.
-        unihash = taskhash
+        return None
 
-        try:
-            method = self.method
-            if tid in self.extramethod:
-                method = method + self.extramethod[tid]
-            data = self.client().get_unihash(method, self.taskhash[tid])
-            if data:
-                unihash = data
+    def _get_method(self, tid):
+        method = self.method
+        if tid in self.extramethod:
+            method = method + self.extramethod[tid]
+
+        return method
+
+    def unihashes_exist(self, query):
+        if len(query) == 0:
+            return {}
+
+        uncached_query = {}
+        result = {}
+        for key, unihash in query.items():
+            if unihash in self.unihash_exists_cache:
+                result[key] = True
+            else:
+                uncached_query[key] = unihash
+
+        if self.max_parallel <= 1 or len(uncached_query) <= 1:
+            # No parallelism required. Make the query serially with the single client
+            with self.client() as client:
+                uncached_result = {
+                    key: client.unihash_exists(value) for key, value in uncached_query.items()
+                }
+        else:
+            with self.client_pool() as client_pool:
+                uncached_result = client_pool.unihashes_exist(uncached_query)
+
+        for key, exists in uncached_result.items():
+            if exists:
+                self.unihash_exists_cache.add(query[key])
+            result[key] = exists
+
+        return result
+
+    def get_unihash(self, tid):
+        return self.get_unihashes([tid])[tid]
+
+    def get_unihashes(self, tids):
+        """
+        For a iterable of tids, returns a dictionary that maps each tid to a
+        unihash
+        """
+        result = {}
+        queries = {}
+        query_result = {}
+
+        for tid in tids:
+            unihash = self.get_cached_unihash(tid)
+            if unihash:
+                result[tid] = unihash
+            else:
+                queries[tid] = (self._get_method(tid), self.taskhash[tid])
+
+        if len(queries) == 0:
+            return result
+
+        if self.max_parallel <= 1 or len(queries) <= 1:
+            # No parallelism required. Make the query using a single client
+            with self.client() as client:
+                keys = list(queries.keys())
+                unihashes = client.get_unihash_batch(queries[k] for k in keys)
+
+                for idx, k in enumerate(keys):
+                    query_result[k] = unihashes[idx]
+        else:
+            with self.client_pool() as client_pool:
+                query_result = client_pool.get_unihashes(queries)
+
+        for tid, unihash in query_result.items():
+            # In the absence of being able to discover a unique hash from the
+            # server, make it be equivalent to the taskhash. The unique "hash" only
+            # really needs to be a unique string (not even necessarily a hash), but
+            # making it match the taskhash has a few advantages:
+            #
+            # 1) All of the sstate code that assumes hashes can be the same
+            # 2) It provides maximal compatibility with builders that don't use
+            #    an equivalency server
+            # 3) The value is easy for multiple independent builders to derive the
+            #    same unique hash from the same input. This means that if the
+            #    independent builders find the same taskhash, but it isn't reported
+            #    to the server, there is a better chance that they will agree on
+            #    the unique hash.
+            taskhash = self.taskhash[tid]
+            if unihash:
                 # A unique hash equal to the taskhash is not very interesting,
                 # so it is reported it at debug level 2. If they differ, that
                 # is much more interesting, so it is reported at debug level 1
                 hashequiv_logger.bbdebug((1, 2)[unihash == taskhash], 'Found unihash %s in place of %s for %s from %s' % (unihash, taskhash, tid, self.server))
             else:
                 hashequiv_logger.debug2('No reported unihash for %s:%s from %s' % (tid, taskhash, self.server))
-        except ConnectionError as e:
-            bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))
+                unihash = taskhash
 
-        self.set_unihash(tid, unihash)
-        self.unihash[tid] = unihash
-        return unihash
+
+            self.set_unihash(tid, unihash)
+            self.unihash[tid] = unihash
+            result[tid] = unihash
+
+        return result
 
     def report_unihash(self, path, task, d):
         import importlib
@@ -680,7 +830,9 @@ class SignatureGeneratorUniHashMixIn(object):
                 if tid in self.extramethod:
                     method = method + self.extramethod[tid]
 
-                data = self.client().report_unihash(taskhash, method, outhash, unihash, extra_data)
+                with self.client() as client:
+                    data = client.report_unihash(taskhash, method, outhash, unihash, extra_data)
+
                 new_unihash = data['unihash']
 
                 if new_unihash != unihash:
@@ -711,7 +863,9 @@ class SignatureGeneratorUniHashMixIn(object):
             if tid in self.extramethod:
                 method = method + self.extramethod[tid]
 
-            data = self.client().report_unihash_equiv(taskhash, method, wanted_unihash, extra_data)
+            with self.client() as client:
+                data = client.report_unihash_equiv(taskhash, method, wanted_unihash, extra_data)
+
             hashequiv_logger.verbose('Reported task %s as unihash %s to %s (%s)' % (tid, wanted_unihash, self.server, str(data)))
 
             if data is None:
@@ -744,6 +898,7 @@ class SignatureGeneratorTestEquivHash(SignatureGeneratorUniHashMixIn, SignatureG
         super().init_rundepcheck(data)
         self.server = data.getVar('BB_HASHSERVE')
         self.method = "sstate_output_hash"
+        self.max_parallel = 1
 
 def clean_checksum_file_path(file_checksum_tuple):
     f, cs = file_checksum_tuple
@@ -839,10 +994,18 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
         formatparams.update(values)
         return formatstr.format(**formatparams)
 
-    with bb.compress.zstd.open(a, "rt", encoding="utf-8", num_threads=1) as f:
-        a_data = json.load(f, object_hook=SetDecoder)
-    with bb.compress.zstd.open(b, "rt", encoding="utf-8", num_threads=1) as f:
-        b_data = json.load(f, object_hook=SetDecoder)
+    try:
+        with bb.compress.zstd.open(a, "rt", encoding="utf-8", num_threads=1) as f:
+            a_data = json.load(f, object_hook=SetDecoder)
+    except (TypeError, OSError) as err:
+        bb.error("Failed to open sigdata file '%s': %s" % (a, str(err)))
+        raise err
+    try:
+        with bb.compress.zstd.open(b, "rt", encoding="utf-8", num_threads=1) as f:
+            b_data = json.load(f, object_hook=SetDecoder)
+    except (TypeError, OSError) as err:
+        bb.error("Failed to open sigdata file '%s': %s" % (b, str(err)))
+        raise err
 
     for data in [a_data, b_data]:
         handle_renames(data)
@@ -1080,8 +1243,12 @@ def calc_taskhash(sigdata):
 def dump_sigfile(a):
     output = []
 
-    with bb.compress.zstd.open(a, "rt", encoding="utf-8", num_threads=1) as f:
-        a_data = json.load(f, object_hook=SetDecoder)
+    try:
+        with bb.compress.zstd.open(a, "rt", encoding="utf-8", num_threads=1) as f:
+            a_data = json.load(f, object_hook=SetDecoder)
+    except (TypeError, OSError) as err:
+        bb.error("Failed to open sigdata file '%s': %s" % (a, str(err)))
+        raise err
 
     handle_renames(a_data)
 

bitbake/lib/bb/tests/codeparser.py

@@ -467,6 +467,6 @@ esac
     #    self.d.setVar("oe_libinstall", "echo test")
     #    self.d.setVar("FOO", "foo=oe_libinstall; eval $foo")
     #    self.d.setVarFlag("FOO", "vardeps", "oe_*")
-    #    self.assertEquals(deps, set(["oe_libinstall"]))
+    #    self.assertEqual(deps, set(["oe_libinstall"]))
 
 

bitbake/lib/bb/tests/data.py

@@ -395,6 +395,16 @@ class TestOverrides(unittest.TestCase):
         self.d.setVar("OVERRIDES", "foo:bar:some_val")
         self.assertEqual(self.d.getVar("TEST"), "testvalue3")
 
+    # Test an override with _<numeric> in it based on a real world OE issue
+    def test_underscore_override_2(self):
+        self.d.setVar("TARGET_ARCH", "x86_64")
+        self.d.setVar("PN", "test-${TARGET_ARCH}")
+        self.d.setVar("VERSION", "1")
+        self.d.setVar("VERSION:pn-test-${TARGET_ARCH}", "2")
+        self.d.setVar("OVERRIDES", "pn-${PN}")
+        bb.data.expandKeys(self.d)
+        self.assertEqual(self.d.getVar("VERSION"), "2")
+
     def test_remove_with_override(self):
         self.d.setVar("TEST:bar", "testvalue2")
         self.d.setVar("TEST:some_val", "testvalue3 testvalue5")
@@ -416,16 +426,6 @@ class TestOverrides(unittest.TestCase):
         self.d.setVar("TEST:bar:append", "testvalue2")
         self.assertEqual(self.d.getVar("TEST"), "testvalue2")
 
-    # Test an override with _<numeric> in it based on a real world OE issue
-    def test_underscore_override(self):
-        self.d.setVar("TARGET_ARCH", "x86_64")
-        self.d.setVar("PN", "test-${TARGET_ARCH}")
-        self.d.setVar("VERSION", "1")
-        self.d.setVar("VERSION:pn-test-${TARGET_ARCH}", "2")
-        self.d.setVar("OVERRIDES", "pn-${PN}")
-        bb.data.expandKeys(self.d)
-        self.assertEqual(self.d.getVar("VERSION"), "2")
-
     def test_append_and_unused_override(self):
         # Had a bug where an unused override append could return "" instead of None
         self.d.setVar("BAR:append:unusedoverride", "testvalue2")

bitbake/lib/bb/tests/event.py

@@ -13,6 +13,7 @@ import pickle
 import threading
 import time
 import unittest
+import tempfile
 from unittest.mock import Mock
 from unittest.mock import call
 
@@ -468,6 +469,8 @@ class EventClassesTest(unittest.TestCase):
 
     def setUp(self):
         bb.event.worker_pid = EventClassesTest._worker_pid
+        self.d = bb.data.init()
+        bb.parse.siggen = bb.siggen.init(self.d)
 
     def test_Event(self):
         """ Test the Event base class """
@@ -950,3 +953,24 @@ class EventClassesTest(unittest.TestCase):
         event = bb.event.FindSigInfoResult(result)
         self.assertEqual(event.result, result)
         self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_lineno_in_eventhandler(self):
+        # The error lineno is 5, not 4 since the first line is '\n'
+        error_line = """
+# Comment line1
+# Comment line2
+python test_lineno_in_eventhandler() {
+    This is an error line
+}
+addhandler test_lineno_in_eventhandler
+test_lineno_in_eventhandler[eventmask] = "bb.event.ConfigParsed"
+"""
+
+        with self.assertLogs() as logs:
+            f = tempfile.NamedTemporaryFile(suffix = '.bb')
+            f.write(bytes(error_line, "utf-8"))
+            f.flush()
+            d = bb.parse.handle(f.name, self.d)['']
+
+        output = "".join(logs.output)
+        self.assertTrue(" line 5\n" in output)

bitbake/lib/bb/tests/fetch.py

@@ -6,6 +6,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
+import contextlib
 import unittest
 import hashlib
 import tempfile
@@ -307,6 +308,21 @@ class URITest(unittest.TestCase):
             'params': {"someparam" : "1"},
             'query': {},
             'relative': True
+        },
+        "https://www.innodisk.com/Download_file?9BE0BF6657;downloadfilename=EGPL-T101.zip": {
+            'uri': 'https://www.innodisk.com/Download_file?9BE0BF6657;downloadfilename=EGPL-T101.zip',
+            'scheme': 'https',
+            'hostname': 'www.innodisk.com',
+            'port': None,
+            'hostport': 'www.innodisk.com',
+            'path': '/Download_file',
+            'userinfo': '',
+            'userinfo': '',
+            'username': '',
+            'password': '',
+            'params': {"downloadfilename" : "EGPL-T101.zip"},
+            'query': {"9BE0BF6657": None},
+            'relative': False
         }
 
     }
@@ -416,9 +432,9 @@ class FetcherTest(unittest.TestCase):
 
     def git(self, cmd, cwd=None):
         if isinstance(cmd, str):
-            cmd = 'git ' + cmd
+            cmd = 'git -c safe.bareRepository=all ' + cmd
         else:
-            cmd = ['git'] + cmd
+            cmd = ['git', '-c', 'safe.bareRepository=all'] + cmd
         if cwd is None:
             cwd = self.gitdir
         return bb.process.run(cmd, cwd=cwd)[0]
@@ -1107,6 +1123,25 @@ class FetcherNetworkTest(FetcherTest):
         if os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1')):
             self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1', 'bitbake')), msg='submodule of submodule missing')
 
+    @skipIfNoNetwork()
+    def test_git_submodule_restricted_network_premirrors(self):
+        # this test is to ensure that premirrors will be tried in restricted network
+        # that is, BB_ALLOWED_NETWORKS does not contain the domain the url uses
+        url = "gitsm://github.com/grpc/grpc.git;protocol=https;name=grpc;branch=v1.60.x;rev=0ef13a7555dbaadd4633399242524129eef5e231"
+        # create a download directory to be used as premirror later
+        tempdir = tempfile.mkdtemp(prefix="bitbake-fetch-")
+        dl_premirror = os.path.join(tempdir, "download-premirror")
+        os.mkdir(dl_premirror)
+        self.d.setVar("DL_DIR", dl_premirror)
+        fetcher = bb.fetch.Fetch([url], self.d)
+        fetcher.download()
+        # now use the premirror in restricted network
+        self.d.setVar("DL_DIR", self.dldir)
+        self.d.setVar("PREMIRRORS", "gitsm://.*/.* gitsm://%s/git2/MIRRORNAME;protocol=file" % dl_premirror)
+        self.d.setVar("BB_ALLOWED_NETWORKS", "*.some.domain")
+        fetcher = bb.fetch.Fetch([url], self.d)
+        fetcher.download()
+
     @skipIfNoNetwork()
     def test_git_submodule_dbus_broker(self):
         # The following external repositories have show failures in fetch and unpack operations
@@ -1247,8 +1282,9 @@ class SVNTest(FetcherTest):
                        cwd=repo_dir)
 
         bb.process.run("svn co %s svnfetch_co" % self.repo_url, cwd=self.tempdir)
-        # Github will emulate SVN.  Use this to check if we're downloding...
-        bb.process.run("svn propset svn:externals 'bitbake https://github.com/PhilipHazel/pcre2.git' .",
+        # Github won't emulate SVN anymore (see https://github.blog/2023-01-20-sunsetting-subversion-support/)
+        # Use still accessible svn repo (only trunk to avoid longer downloads)
+        bb.process.run("svn propset svn:externals 'bitbake https://svn.apache.org/repos/asf/serf/trunk' .",
                        cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk'))
         bb.process.run("svn commit --non-interactive -m 'Add external'",
                        cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk'))
@@ -1276,8 +1312,8 @@ class SVNTest(FetcherTest):
 
         self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk")
         self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents")
-        self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk')), msg="External dir should NOT exist")
-        self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk', 'README')), msg="External README should NOT exit")
+        self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols')), msg="External dir should NOT exist")
+        self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols', 'fcgi_buckets.h')), msg="External fcgi_buckets.h should NOT exit")
 
     @skipIfNoSvn()
     def test_external_svn(self):
@@ -1290,8 +1326,8 @@ class SVNTest(FetcherTest):
 
         self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk")
         self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents")
-        self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk')), msg="External dir should exist")
-        self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk', 'README')), msg="External README should exit")
+        self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols')), msg="External dir should exist")
+        self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols', 'fcgi_buckets.h')), msg="External fcgi_buckets.h should exit")
 
 class TrustedNetworksTest(FetcherTest):
     def test_trusted_network(self):
@@ -1369,37 +1405,39 @@ class FetchLatestVersionTest(FetcherTest):
 
     test_git_uris = {
         # version pattern "X.Y.Z"
-        ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4;protocol=https", "9b1db6b8060bd00b121a692f942404a24ae2960f", "")
+        ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4;protocol=https", "9b1db6b8060bd00b121a692f942404a24ae2960f", "", "")
             : "1.99.4",
         # version pattern "vX.Y"
         # mirror of git.infradead.org since network issues interfered with testing
-        ("mtd-utils", "git://git.yoctoproject.org/mtd-utils.git;branch=master;protocol=https", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "")
+        ("mtd-utils", "git://git.yoctoproject.org/mtd-utils.git;branch=master;protocol=https", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "", "")
             : "1.5.0",
         # version pattern "pkg_name-X.Y"
         # mirror of git://anongit.freedesktop.org/git/xorg/proto/presentproto since network issues interfered with testing
-        ("presentproto", "git://git.yoctoproject.org/bbfetchtests-presentproto;branch=master;protocol=https", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "")
+        ("presentproto", "git://git.yoctoproject.org/bbfetchtests-presentproto;branch=master;protocol=https", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "", "")
             : "1.0",
         # version pattern "pkg_name-vX.Y.Z"
-        ("dtc", "git://git.yoctoproject.org/bbfetchtests-dtc.git;branch=master;protocol=https", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "")
+        ("dtc", "git://git.yoctoproject.org/bbfetchtests-dtc.git;branch=master;protocol=https", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "", "")
             : "1.4.0",
         # combination version pattern
-        ("sysprof", "git://gitlab.gnome.org/GNOME/sysprof.git;protocol=https;branch=master", "cd44ee6644c3641507fb53b8a2a69137f2971219", "")
+        ("sysprof", "git://gitlab.gnome.org/GNOME/sysprof.git;protocol=https;branch=master", "cd44ee6644c3641507fb53b8a2a69137f2971219", "", "")
             : "1.2.0",
-        ("u-boot-mkimage", "git://git.denx.de/u-boot.git;branch=master;protocol=git", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "")
+        ("u-boot-mkimage", "git://source.denx.de/u-boot/u-boot.git;branch=master;protocol=https", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "", "")
             : "2014.01",
         # version pattern "yyyymmdd"
-        ("mobile-broadband-provider-info", "git://gitlab.gnome.org/GNOME/mobile-broadband-provider-info.git;protocol=https;branch=master", "4ed19e11c2975105b71b956440acdb25d46a347d", "")
+        ("mobile-broadband-provider-info", "git://gitlab.gnome.org/GNOME/mobile-broadband-provider-info.git;protocol=https;branch=master", "4ed19e11c2975105b71b956440acdb25d46a347d", "", "")
             : "20120614",
         # packages with a valid UPSTREAM_CHECK_GITTAGREGEX
                 # mirror of git://anongit.freedesktop.org/xorg/driver/xf86-video-omap since network issues interfered with testing
-        ("xf86-video-omap", "git://git.yoctoproject.org/bbfetchtests-xf86-video-omap;branch=master;protocol=https", "ae0394e687f1a77e966cf72f895da91840dffb8f", r"(?P<pver>(\d+\.(\d\.?)*))")
+        ("xf86-video-omap", "git://git.yoctoproject.org/bbfetchtests-xf86-video-omap;branch=master;protocol=https", "ae0394e687f1a77e966cf72f895da91840dffb8f", r"(?P<pver>(\d+\.(\d\.?)*))", "")
             : "0.4.3",
-        ("build-appliance-image", "git://git.yoctoproject.org/poky;branch=master;protocol=https", "b37dd451a52622d5b570183a81583cc34c2ff555", r"(?P<pver>(([0-9][\.|_]?)+[0-9]))")
+        ("build-appliance-image", "git://git.yoctoproject.org/poky;branch=master;protocol=https", "b37dd451a52622d5b570183a81583cc34c2ff555", r"(?P<pver>(([0-9][\.|_]?)+[0-9]))", "")
             : "11.0.0",
-        ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot;protocol=https", "cd437ecbd8986c894442f8fce1e0061e20f04dee", r"chkconfig\-(?P<pver>((\d+[\.\-_]*)+))")
+        ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot;protocol=https", "cd437ecbd8986c894442f8fce1e0061e20f04dee", r"chkconfig\-(?P<pver>((\d+[\.\-_]*)+))", "")
             : "1.3.59",
-        ("remake", "git://github.com/rocky/remake.git;protocol=https;branch=master", "f05508e521987c8494c92d9c2871aec46307d51d", r"(?P<pver>(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))")
+        ("remake", "git://github.com/rocky/remake.git;protocol=https;branch=master", "f05508e521987c8494c92d9c2871aec46307d51d", r"(?P<pver>(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))", "")
             : "3.82+dbg0.9",
+        ("sysdig", "git://github.com/draios/sysdig.git;branch=dev;protocol=https", "4fb6288275f567f63515df0ff0a6518043ecfa9b", r"^(?P<pver>\d+(\.\d+)+)", "10.0.0")
+            : "0.28.0",
     }
 
     test_wget_uris = {
@@ -1467,10 +1505,13 @@ class FetchLatestVersionTest(FetcherTest):
             self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0])
             r = bb.utils.vercmp_string(v, verstring)
             self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring))
+            if k[4]:
+                r = bb.utils.vercmp_string(verstring, k[4])
+                self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], verstring, k[4]))
 
     def test_wget_latest_versionstring(self):
         testdata = os.path.dirname(os.path.abspath(__file__)) + "/fetch-testdata"
-        server = HTTPService(testdata)
+        server = HTTPService(testdata, host="127.0.0.1")
         server.start()
         port = server.port
         try:
@@ -1478,10 +1519,10 @@ class FetchLatestVersionTest(FetcherTest):
                 self.d.setVar("PN", k[0])
                 checkuri = ""
                 if k[2]:
-                    checkuri = "http://localhost:%s/" % port + k[2]
+                    checkuri = "http://127.0.0.1:%s/" % port + k[2]
                 self.d.setVar("UPSTREAM_CHECK_URI", checkuri)
                 self.d.setVar("UPSTREAM_CHECK_REGEX", k[3])
-                url = "http://localhost:%s/" % port + k[1]
+                url = "http://127.0.0.1:%s/" % port + k[1]
                 ud = bb.fetch2.FetchData(url, self.d)
                 pupver = ud.method.latest_versionstring(ud, self.d)
                 verstring = pupver[0]
@@ -2236,10 +2277,14 @@ class GitLfsTest(FetcherTest):
 
         bb.utils.mkdirhier(self.srcdir)
         self.git_init(cwd=self.srcdir)
-        with open(os.path.join(self.srcdir, '.gitattributes'), 'wt') as attrs:
-            attrs.write('*.mp3 filter=lfs -text')
-        self.git(['add', '.gitattributes'], cwd=self.srcdir)
-        self.git(['commit', '-m', "attributes", '.gitattributes'], cwd=self.srcdir)
+        self.commit_file('.gitattributes', '*.mp3 filter=lfs -text')
+
+    def commit_file(self, filename, content):
+        with open(os.path.join(self.srcdir, filename), "w") as f:
+            f.write(content)
+        self.git(["add", filename], cwd=self.srcdir)
+        self.git(["commit", "-m", "Change"], cwd=self.srcdir)
+        return self.git(["rev-parse", "HEAD"], cwd=self.srcdir).strip()
 
     def fetch(self, uri=None, download=True):
         uris = self.d.getVar('SRC_URI').split()
@@ -2259,6 +2304,44 @@ class GitLfsTest(FetcherTest):
         unpacked_lfs_file = os.path.join(self.d.getVar('WORKDIR'), 'git', "Cat_poster_1.jpg")
         return unpacked_lfs_file
 
+    @skipIfNoGitLFS()
+    def test_fetch_lfs_on_srcrev_change(self):
+        """Test if fetch downloads missing LFS objects when a different revision within an existing repository is requested"""
+        self.git(["lfs", "install", "--local"], cwd=self.srcdir)
+
+        @contextlib.contextmanager
+        def hide_upstream_repository():
+            """Hide the upstream repository to make sure that git lfs cannot pull from it"""
+            temp_name = self.srcdir + ".bak"
+            os.rename(self.srcdir, temp_name)
+            try:
+                yield
+            finally:
+                os.rename(temp_name, self.srcdir)
+
+        def fetch_and_verify(revision, filename, content):
+            self.d.setVar('SRCREV', revision)
+            fetcher, ud = self.fetch()
+
+            with hide_upstream_repository():
+                workdir = self.d.getVar('WORKDIR')
+                fetcher.unpack(workdir)
+
+                with open(os.path.join(workdir, "git", filename)) as f:
+                    self.assertEqual(f.read(), content)
+
+        commit_1 = self.commit_file("a.mp3", "version 1")
+        commit_2 = self.commit_file("a.mp3", "version 2")
+
+        self.d.setVar('SRC_URI', "git://%s;protocol=file;lfs=1;branch=master" % self.srcdir)
+
+        # Seed the local download folder by fetching the latest commit and verifying that the LFS contents are
+        # available even when the upstream repository disappears.
+        fetch_and_verify(commit_2, "a.mp3", "version 2")
+        # Verify that even when an older revision is fetched, the needed LFS objects are fetched into the download
+        # folder.
+        fetch_and_verify(commit_1, "a.mp3", "version 1")
+
     @skipIfNoGitLFS()
     @skipIfNoNetwork()
     def test_real_git_lfs_repo_succeeds_without_lfs_param(self):
@@ -2277,7 +2360,7 @@ class GitLfsTest(FetcherTest):
 
     @skipIfNoGitLFS()
     @skipIfNoNetwork()
-    def test_real_git_lfs_repo_succeeds(self):
+    def test_real_git_lfs_repo_skips(self):
         self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master;lfs=0")
         f = self.get_real_git_lfs_file()
         # This is the actual non-smudged placeholder file on the repo if git-lfs does not run
@@ -2290,24 +2373,41 @@ class GitLfsTest(FetcherTest):
         with open(f) as fh:
             self.assertEqual(lfs_file, fh.read())
 
+    @skipIfNoGitLFS()
     def test_lfs_enabled(self):
         import shutil
 
         uri = 'git://%s;protocol=file;lfs=1;branch=master' % self.srcdir
         self.d.setVar('SRC_URI', uri)
 
-        # Careful: suppress initial attempt at downloading until
-        # we know whether git-lfs is installed.
+        # With git-lfs installed, test that we can fetch and unpack
+        fetcher, ud = self.fetch()
+        shutil.rmtree(self.gitdir, ignore_errors=True)
+        fetcher.unpack(self.d.getVar('WORKDIR'))
+
+    @skipIfNoGitLFS()
+    def test_lfs_disabled(self):
+        import shutil
+
+        uri = 'git://%s;protocol=file;lfs=0;branch=master' % self.srcdir
+        self.d.setVar('SRC_URI', uri)
+
+        # Verify that the fetcher can survive even if the source
+        # repository has Git LFS usage configured.
+        fetcher, ud = self.fetch()
+        fetcher.unpack(self.d.getVar('WORKDIR'))
+
+    def test_lfs_enabled_not_installed(self):
+        import shutil
+
+        uri = 'git://%s;protocol=file;lfs=1;branch=master' % self.srcdir
+        self.d.setVar('SRC_URI', uri)
+
+        # Careful: suppress initial attempt at downloading
         fetcher, ud = self.fetch(uri=None, download=False)
-        self.assertIsNotNone(ud.method._find_git_lfs)
-
-        # If git-lfs can be found, the unpack should be successful. Only
-        # attempt this with the real live copy of git-lfs installed.
-        if ud.method._find_git_lfs(self.d):
-            fetcher.download()
-            shutil.rmtree(self.gitdir, ignore_errors=True)
-            fetcher.unpack(self.d.getVar('WORKDIR'))
 
+        # Artificially assert that git-lfs is not installed, so
+        # we can verify a failure to unpack in it's absence.
         old_find_git_lfs = ud.method._find_git_lfs
         try:
             # If git-lfs cannot be found, the unpack should throw an error
@@ -2319,29 +2419,21 @@ class GitLfsTest(FetcherTest):
         finally:
             ud.method._find_git_lfs = old_find_git_lfs
 
-    def test_lfs_disabled(self):
+    def test_lfs_disabled_not_installed(self):
         import shutil
 
         uri = 'git://%s;protocol=file;lfs=0;branch=master' % self.srcdir
         self.d.setVar('SRC_URI', uri)
 
-        # In contrast to test_lfs_enabled(), allow the implicit download
-        # done by self.fetch() to occur here. The point of this test case
-        # is to verify that the fetcher can survive even if the source
-        # repository has Git LFS usage configured.
-        fetcher, ud = self.fetch()
-        self.assertIsNotNone(ud.method._find_git_lfs)
+        # Careful: suppress initial attempt at downloading
+        fetcher, ud = self.fetch(uri=None, download=False)
 
+        # Artificially assert that git-lfs is not installed, so
+        # we can verify a failure to unpack in it's absence.
         old_find_git_lfs = ud.method._find_git_lfs
         try:
-            # If git-lfs can be found, the unpack should be successful. A
-            # live copy of git-lfs is not required for this case, so
-            # unconditionally forge its presence.
-            ud.method._find_git_lfs = lambda d: True
-            shutil.rmtree(self.gitdir, ignore_errors=True)
-            fetcher.unpack(self.d.getVar('WORKDIR'))
-            # If git-lfs cannot be found, the unpack should be successful
-
+            # Even if git-lfs cannot be found, the unpack should be successful
+            fetcher.download()
             ud.method._find_git_lfs = lambda d: False
             shutil.rmtree(self.gitdir, ignore_errors=True)
             fetcher.unpack(self.d.getVar('WORKDIR'))
@@ -3042,9 +3134,11 @@ class FetchPremirroronlyLocalTest(FetcherTest):
         self.d.setVar("BB_FETCH_PREMIRRORONLY", "1")
         self.d.setVar("BB_NO_NETWORK", "1")
         self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n")
+        self.mirrorname = "git2_git.fake.repo.bitbake.tar.gz"
+        self.mirrorfile = os.path.join(self.mirrordir, self.mirrorname)
+        self.testfilename = "bitbake-fetch.test"
 
     def make_git_repo(self):
-        self.mirrorname = "git2_git.fake.repo.bitbake.tar.gz"
         recipeurl = "git:/git.fake.repo/bitbake"
         os.makedirs(self.gitdir)
         self.git_init(cwd=self.gitdir)
@@ -3054,15 +3148,41 @@ class FetchPremirroronlyLocalTest(FetcherTest):
 
     def git_new_commit(self):
         import random
-        testfilename = "bibake-fetch.test"
         os.unlink(os.path.join(self.mirrordir, self.mirrorname))
-        with open(os.path.join(self.gitdir, testfilename), "w") as testfile:
-            testfile.write("Useless random data {}".format(random.random()))
-        self.git("add {}".format(testfilename), self.gitdir)
-        self.git("commit -a -m \"This random commit {}. I'm useless.\"".format(random.random()), self.gitdir)
+        branch = self.git("branch --show-current", self.gitdir).split()
+        with open(os.path.join(self.gitdir, self.testfilename), "w") as testfile:
+            testfile.write("File {} from branch {}; Useless random data {}".format(self.testfilename, branch, random.random()))
+        self.git("add {}".format(self.testfilename), self.gitdir)
+        self.git("commit -a -m \"This random commit {} in branch {}. I'm useless.\"".format(random.random(), branch), self.gitdir)
         bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd =  self.gitdir)
         return self.git("rev-parse HEAD", self.gitdir).strip()
 
+    def git_new_branch(self, name):
+        self.git_new_commit()
+        head = self.git("rev-parse HEAD", self.gitdir).strip()
+        self.git("checkout -b {}".format(name), self.gitdir)
+        newrev = self.git_new_commit()
+        self.git("checkout {}".format(head), self.gitdir)
+        return newrev
+
+    def test_mirror_multiple_fetches(self):
+        self.make_git_repo()
+        self.d.setVar("SRCREV", self.git_new_commit())
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+        fetcher.download()
+        fetcher.unpack(self.unpackdir)
+        ## New commit in premirror. it's not in the download_dir
+        self.d.setVar("SRCREV", self.git_new_commit())
+        fetcher2 = bb.fetch.Fetch([self.recipe_url], self.d)
+        fetcher2.download()
+        fetcher2.unpack(self.unpackdir)
+        ## New commit in premirror. it's not in the download_dir
+        self.d.setVar("SRCREV", self.git_new_commit())
+        fetcher3 = bb.fetch.Fetch([self.recipe_url], self.d)
+        fetcher3.download()
+        fetcher3.unpack(self.unpackdir)
+
+
     def test_mirror_commit_nonexistent(self):
         self.make_git_repo()
         self.d.setVar("SRCREV", "0"*40)
@@ -3083,6 +3203,59 @@ class FetchPremirroronlyLocalTest(FetcherTest):
         with self.assertRaises(bb.fetch2.NetworkAccess):
             fetcher.download()
 
+    def test_mirror_tarball_multiple_branches(self):
+        """
+        test if PREMIRRORS can handle multiple name/branches correctly
+        both branches have required revisions
+        """
+        self.make_git_repo()
+        branch1rev = self.git_new_branch("testbranch1")
+        branch2rev = self.git_new_branch("testbranch2")
+        self.recipe_url = "git://git.fake.repo/bitbake;branch=testbranch1,testbranch2;protocol=https;name=branch1,branch2"
+        self.d.setVar("SRCREV_branch1", branch1rev)
+        self.d.setVar("SRCREV_branch2", branch2rev)
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+        self.assertTrue(os.path.exists(self.mirrorfile), "Mirror file doesn't exist")
+        fetcher.download()
+        fetcher.unpack(os.path.join(self.tempdir, "unpacked"))
+        unpacked = os.path.join(self.tempdir, "unpacked", "git", self.testfilename)
+        self.assertTrue(os.path.exists(unpacked), "Repo has not been unpackaged properly!")
+        with open(unpacked, 'r') as f:
+            content = f.read()
+            ## We expect to see testbranch1 in the file, not master, not testbranch2
+            self.assertTrue(content.find("testbranch1") != -1, "Wrong branch has been checked out!")
+
+    def test_mirror_tarball_multiple_branches_nobranch(self):
+        """
+        test if PREMIRRORS can handle multiple name/branches correctly
+        Unbalanced name/branches raises ParameterError
+        """
+        self.make_git_repo()
+        branch1rev = self.git_new_branch("testbranch1")
+        branch2rev = self.git_new_branch("testbranch2")
+        self.recipe_url = "git://git.fake.repo/bitbake;branch=testbranch1;protocol=https;name=branch1,branch2"
+        self.d.setVar("SRCREV_branch1", branch1rev)
+        self.d.setVar("SRCREV_branch2", branch2rev)
+        with self.assertRaises(bb.fetch2.ParameterError):
+            fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+
+    def test_mirror_tarball_multiple_branches_norev(self):
+        """
+        test if PREMIRRORS can handle multiple name/branches correctly
+        one of the branches specifies non existing SRCREV
+        """
+        self.make_git_repo()
+        branch1rev = self.git_new_branch("testbranch1")
+        branch2rev = self.git_new_branch("testbranch2")
+        self.recipe_url = "git://git.fake.repo/bitbake;branch=testbranch1,testbranch2;protocol=https;name=branch1,branch2"
+        self.d.setVar("SRCREV_branch1", branch1rev)
+        self.d.setVar("SRCREV_branch2", "0"*40)
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+        self.assertTrue(os.path.exists(self.mirrorfile), "Mirror file doesn't exist")
+        with self.assertRaises(bb.fetch2.NetworkAccess):
+            fetcher.download()
+
+
 class FetchPremirroronlyNetworkTest(FetcherTest):
 
     def setUp(self):

bitbake/lib/bb/tests/parse.py

@@ -243,3 +243,101 @@ unset A[flag@.service]
         with self.assertRaises(bb.parse.ParseError):
             d = bb.parse.handle(f.name, self.d)['']
 
+    export_function_recipe = """
+inherit someclass
+"""
+
+    export_function_recipe2 = """
+inherit someclass
+
+do_compile () {
+    false
+}
+
+python do_compilepython () {
+    bb.note("Something else")
+}
+
+"""
+    export_function_class = """
+someclass_do_compile() {
+    true
+}
+
+python someclass_do_compilepython () {
+    bb.note("Something")
+}
+
+EXPORT_FUNCTIONS do_compile do_compilepython
+"""
+
+    export_function_class2 = """
+secondclass_do_compile() {
+    true
+}
+
+python secondclass_do_compilepython () {
+    bb.note("Something")
+}
+
+EXPORT_FUNCTIONS do_compile do_compilepython
+"""
+
+    def test_parse_export_functions(self):
+        def check_function_flags(d):
+            self.assertEqual(d.getVarFlag("do_compile", "func"), 1)
+            self.assertEqual(d.getVarFlag("do_compilepython", "func"), 1)
+            self.assertEqual(d.getVarFlag("do_compile", "python"), None)
+            self.assertEqual(d.getVarFlag("do_compilepython", "python"), "1")
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            self.d.setVar("__bbclasstype", "recipe")
+            recipename = tempdir + "/recipe.bb"
+            os.makedirs(tempdir + "/classes")
+            with open(tempdir + "/classes/someclass.bbclass", "w") as f:
+                f.write(self.export_function_class)
+                f.flush()
+            with open(tempdir + "/classes/secondclass.bbclass", "w") as f:
+                f.write(self.export_function_class2)
+                f.flush()
+
+            with open(recipename, "w") as f:
+                f.write(self.export_function_recipe)
+                f.flush()
+            os.chdir(tempdir)
+            d = bb.parse.handle(recipename, bb.data.createCopy(self.d))['']
+            self.assertIn("someclass_do_compile", d.getVar("do_compile"))
+            self.assertIn("someclass_do_compilepython", d.getVar("do_compilepython"))
+            check_function_flags(d)
+
+            recipename2 = tempdir + "/recipe2.bb"
+            with open(recipename2, "w") as f:
+                f.write(self.export_function_recipe2)
+                f.flush()
+
+            d = bb.parse.handle(recipename2, bb.data.createCopy(self.d))['']
+            self.assertNotIn("someclass_do_compile", d.getVar("do_compile"))
+            self.assertNotIn("someclass_do_compilepython", d.getVar("do_compilepython"))
+            self.assertIn("false", d.getVar("do_compile"))
+            self.assertIn("else", d.getVar("do_compilepython"))
+            check_function_flags(d)
+
+            with open(recipename, "a+") as f:
+                f.write("\ninherit secondclass\n")
+                f.flush()
+            with open(recipename2, "a+") as f:
+                f.write("\ninherit secondclass\n")
+                f.flush()
+
+            d = bb.parse.handle(recipename, bb.data.createCopy(self.d))['']
+            self.assertIn("secondclass_do_compile", d.getVar("do_compile"))
+            self.assertIn("secondclass_do_compilepython", d.getVar("do_compilepython"))
+            check_function_flags(d)
+
+            d = bb.parse.handle(recipename2, bb.data.createCopy(self.d))['']
+            self.assertNotIn("someclass_do_compile", d.getVar("do_compile"))
+            self.assertNotIn("someclass_do_compilepython", d.getVar("do_compilepython"))
+            self.assertIn("false", d.getVar("do_compile"))
+            self.assertIn("else", d.getVar("do_compilepython"))
+            check_function_flags(d)
+

bitbake/lib/bb/ui/eventreplay.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file re-uses code spread throughout other Bitbake source files.
+# As such, all other copyrights belong to their own right holders.
+#
+
+
+import os
+import sys
+import json
+import pickle
+import codecs
+
+
+class EventPlayer:
+    """Emulate a connection to a bitbake server."""
+
+    def __init__(self, eventfile, variables):
+        self.eventfile = eventfile
+        self.variables = variables
+        self.eventmask = []
+
+    def waitEvent(self, _timeout):
+        """Read event from the file."""
+        line = self.eventfile.readline().strip()
+        if not line:
+            return
+        try:
+            decodedline = json.loads(line)
+            if 'allvariables' in decodedline:
+                self.variables = decodedline['allvariables']
+                return
+            if not 'vars' in decodedline:
+                raise ValueError
+            event_str = decodedline['vars'].encode('utf-8')
+            event = pickle.loads(codecs.decode(event_str, 'base64'))
+            event_name = "%s.%s" % (event.__module__, event.__class__.__name__)
+            if event_name not in self.eventmask:
+                return
+            return event
+        except ValueError as err:
+            print("Failed loading ", line)
+            raise err
+
+    def runCommand(self, command_line):
+        """Emulate running a command on the server."""
+        name = command_line[0]
+
+        if name == "getVariable":
+            var_name = command_line[1]
+            variable = self.variables.get(var_name)
+            if variable:
+                return variable['v'], None
+            return None, "Missing variable %s" % var_name
+
+        elif name == "getAllKeysWithFlags":
+            dump = {}
+            flaglist = command_line[1]
+            for key, val in self.variables.items():
+                try:
+                    if not key.startswith("__"):
+                        dump[key] = {
+                            'v': val['v'],
+                            'history' : val['history'],
+                        }
+                        for flag in flaglist:
+                            dump[key][flag] = val[flag]
+                except Exception as err:
+                    print(err)
+            return (dump, None)
+
+        elif name == 'setEventMask':
+            self.eventmask = command_line[-1]
+            return True, None
+
+        else:
+            raise Exception("Command %s not implemented" % command_line[0])
+
+    def getEventHandle(self):
+        """
+        This method is called by toasterui.
+        The return value is passed to self.runCommand but not used there.
+        """
+        pass

bitbake/lib/bb/ui/knotty.py

@@ -179,7 +179,7 @@ class TerminalFilter(object):
             new[3] = new[3] & ~termios.ECHO
             termios.tcsetattr(fd, termios.TCSADRAIN, new)
             curses.setupterm()
-            if curses.tigetnum("colors") > 2:
+            if curses.tigetnum("colors") > 2 and os.environ.get('NO_COLOR', '') == '':
                 for h in handlers:
                     try:
                         h.formatter.enable_color()
@@ -420,6 +420,11 @@ def main(server, eventHandler, params, tf = TerminalFilter):
     except bb.BBHandledException:
         drain_events_errorhandling(eventHandler)
         return 1
+    except Exception as e:
+        # bitbake-server comms failure
+        early_logger = bb.msg.logger_create('bitbake', sys.stdout)
+        early_logger.fatal("Attempting to set server environment: %s", e)
+        return 1
 
     if params.options.quiet == 0:
         console_loglevel = loglevel
@@ -585,7 +590,12 @@ def main(server, eventHandler, params, tf = TerminalFilter):
         return
 
     llevel, debug_domains = bb.msg.constructLogOptions()
-    server.runCommand(["setEventMask", server.getEventHandle(), llevel, debug_domains, _evt_list])
+    try:
+        server.runCommand(["setEventMask", server.getEventHandle(), llevel, debug_domains, _evt_list])
+    except (BrokenPipeError, EOFError) as e:
+        # bitbake-server comms failure
+        logger.fatal("Attempting to set event mask: %s", e)
+        return 1
 
     # The logging_tree module is *extremely* helpful in debugging logging
     # domains. Uncomment here to dump the logging tree when bitbake starts
@@ -594,7 +604,11 @@ def main(server, eventHandler, params, tf = TerminalFilter):
 
     universe = False
     if not params.observe_only:
-        params.updateFromServer(server)
+        try:
+            params.updateFromServer(server)
+        except Exception as e:
+            logger.fatal("Fetching command line: %s", e)
+            return 1
         cmdline = params.parseActions()
         if not cmdline:
             print("Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.")
@@ -605,7 +619,12 @@ def main(server, eventHandler, params, tf = TerminalFilter):
         if cmdline['action'][0] == "buildTargets" and "universe" in cmdline['action'][1]:
             universe = True
 
-        ret, error = server.runCommand(cmdline['action'])
+        try:
+            ret, error = server.runCommand(cmdline['action'])
+        except (BrokenPipeError, EOFError) as e:
+            # bitbake-server comms failure
+            logger.fatal("Command '{}' failed: %s".format(cmdline), e)
+            return 1
         if error:
             logger.error("Command '%s' failed: %s" % (cmdline, error))
             return 1
@@ -854,15 +873,26 @@ def main(server, eventHandler, params, tf = TerminalFilter):
 
             logger.error("Unknown event: %s", event)
 
+        except (BrokenPipeError, EOFError) as e:
+            # bitbake-server comms failure, don't attempt further comms and exit
+            logger.fatal("Executing event: %s", e)
+            return_value = 1
+            errors = errors + 1
+            main.shutdown = 3
         except EnvironmentError as ioerror:
             termfilter.clearFooter()
             # ignore interrupted io
             if ioerror.args[0] == 4:
                 continue
             sys.stderr.write(str(ioerror))
-            if not params.observe_only:
-                _, error = server.runCommand(["stateForceShutdown"])
             main.shutdown = 2
+            if not params.observe_only:
+                try:
+                    _, error = server.runCommand(["stateForceShutdown"])
+                except (BrokenPipeError, EOFError) as e:
+                    # bitbake-server comms failure, don't attempt further comms and exit
+                    logger.fatal("Unable to force shutdown: %s", e)
+                    main.shutdown = 3
         except KeyboardInterrupt:
             termfilter.clearFooter()
             if params.observe_only:
@@ -871,9 +901,13 @@ def main(server, eventHandler, params, tf = TerminalFilter):
 
             def state_force_shutdown():
                 print("\nSecond Keyboard Interrupt, stopping...\n")
-                _, error = server.runCommand(["stateForceShutdown"])
-                if error:
-                    logger.error("Unable to cleanly stop: %s" % error)
+                try:
+                    _, error = server.runCommand(["stateForceShutdown"])
+                    if error:
+                        logger.error("Unable to cleanly stop: %s" % error)
+                except (BrokenPipeError, EOFError) as e:
+                    # bitbake-server comms failure
+                    logger.fatal("Unable to cleanly stop: %s", e)
 
             if not params.observe_only and main.shutdown == 1:
                 state_force_shutdown()
@@ -886,6 +920,9 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                     _, error = server.runCommand(["stateShutdown"])
                     if error:
                         logger.error("Unable to cleanly shutdown: %s" % error)
+                except (BrokenPipeError, EOFError) as e:
+                    # bitbake-server comms failure
+                    logger.fatal("Unable to cleanly shutdown: %s", e)
                 except KeyboardInterrupt:
                     state_force_shutdown()
 
@@ -893,9 +930,14 @@ def main(server, eventHandler, params, tf = TerminalFilter):
         except Exception as e:
             import traceback
             sys.stderr.write(traceback.format_exc())
-            if not params.observe_only:
-                _, error = server.runCommand(["stateForceShutdown"])
             main.shutdown = 2
+            if not params.observe_only:
+                try:
+                    _, error = server.runCommand(["stateForceShutdown"])
+                except (BrokenPipeError, EOFError) as e:
+                    # bitbake-server comms failure, don't attempt further comms and exit
+                    logger.fatal("Unable to force shutdown: %s", e)
+                    main.shudown = 3
             return_value = 1
     try:
         termfilter.clearFooter()

bitbake/lib/bb/ui/ncurses.py

@@ -227,6 +227,9 @@ class NCursesUI:
         shutdown = 0
 
         try:
+            if not params.observe_only:
+                params.updateToServer(server, os.environ.copy())
+
             params.updateFromServer(server)
             cmdline = params.parseActions()
             if not cmdline:

bitbake/lib/bb/ui/toasterui.py

@@ -385,7 +385,7 @@ def main(server, eventHandler, params):
                     main.shutdown = 1
 
                 logger.info("ToasterUI build done, brbe: %s", brbe)
-                continue
+                break
 
             if isinstance(event, (bb.command.CommandCompleted,
                                   bb.command.CommandFailed,

bitbake/lib/bb/utils.py

@@ -50,7 +50,7 @@ def clean_context():
 
 def get_context():
     return _context
-    
+
 
 def set_context(ctx):
     _context = ctx
@@ -212,8 +212,8 @@ def explode_dep_versions2(s, *, sort=True):
             inversion = True
             # This list is based on behavior and supported comparisons from deb, opkg and rpm.
             #
-            # Even though =<, <<, ==, !=, =>, and >> may not be supported, 
-            # we list each possibly valid item. 
+            # Even though =<, <<, ==, !=, =>, and >> may not be supported,
+            # we list each possibly valid item.
             # The build system is responsible for validation of what it supports.
             if i.startswith(('<=', '=<', '<<', '==', '!=', '>=', '=>', '>>')):
                 lastcmp = i[0:2]
@@ -347,7 +347,7 @@ def _print_exception(t, value, tb, realfile, text, context):
         exception = traceback.format_exception_only(t, value)
         error.append('Error executing a python function in %s:\n' % realfile)
 
-        # Strip 'us' from the stack (better_exec call) unless that was where the 
+        # Strip 'us' from the stack (better_exec call) unless that was where the
         # error came from
         if tb.tb_next is not None:
             tb = tb.tb_next
@@ -604,7 +604,6 @@ def preserved_envvars():
     v = [
         'BBPATH',
         'BB_PRESERVE_ENV',
-        'BB_ENV_PASSTHROUGH',
         'BB_ENV_PASSTHROUGH_ADDITIONS',
     ]
     return v + preserved_envvars_exported()
@@ -746,9 +745,9 @@ def prunedir(topdir, ionice=False):
 # but thats possibly insane and suffixes is probably going to be small
 #
 def prune_suffix(var, suffixes, d):
-    """ 
+    """
     See if var ends with any of the suffixes listed and
-    remove it if found 
+    remove it if found
     """
     for suffix in suffixes:
         if suffix and var.endswith(suffix):
@@ -759,7 +758,8 @@ def mkdirhier(directory):
     """Create a directory like 'mkdir -p', but does not complain if
     directory already exists like os.makedirs
     """
-
+    if '${' in str(directory):
+        bb.fatal("Directory name {} contains unexpanded bitbake variable. This may cause build failures and WORKDIR polution.".format(directory))
     try:
         os.makedirs(directory)
     except OSError as e:
@@ -1001,9 +1001,9 @@ def umask(new_mask):
         os.umask(current_mask)
 
 def to_boolean(string, default=None):
-    """ 
+    """
     Check input string and return boolean value True/False/None
-    depending upon the checks 
+    depending upon the checks
     """
     if not string:
         return default
@@ -1142,7 +1142,10 @@ def get_referenced_vars(start_expr, d):
 
 
 def cpu_count():
-    return multiprocessing.cpu_count()
+    try:
+        return len(os.sched_getaffinity(0))
+    except OSError:
+        return multiprocessing.cpu_count()
 
 def nonblockingfd(fd):
     fcntl.fcntl(fd, fcntl.F_SETFL, fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK)

bitbake/lib/bs4/tests/test_tree.py

@@ -585,7 +585,7 @@ class SiblingTest(TreeTest):
                     </html>'''
         # All that whitespace looks good but makes the tests more
         # difficult. Get rid of it.
-        markup = re.compile("\n\s*").sub("", markup)
+        markup = re.compile(r"\n\s*").sub("", markup)
         self.tree = self.soup(markup)
 
 

bitbake/lib/codegen.py

@@ -392,19 +392,7 @@ class SourceGenerator(NodeVisitor):
     def visit_Name(self, node):
         self.write(node.id)
 
-    def visit_Str(self, node):
-        self.write(repr(node.s))
-
-    def visit_Bytes(self, node):
-        self.write(repr(node.s))
-
-    def visit_Num(self, node):
-        self.write(repr(node.n))
-
     def visit_Constant(self, node):
-        # Python 3.8 deprecated visit_Num(), visit_Str(), visit_Bytes(),
-        # visit_NameConstant() and visit_Ellipsis(). They can be removed once we
-        # require 3.8+.
         self.write(repr(node.value))
 
     def visit_Tuple(self, node):

bitbake/lib/hashserv/__init__.py

@@ -5,151 +5,102 @@
 
 import asyncio
 from contextlib import closing
-import re
-import sqlite3
 import itertools
 import json
+from collections import namedtuple
+from urllib.parse import urlparse
+from bb.asyncrpc.client import parse_address, ADDR_TYPE_UNIX, ADDR_TYPE_WS
 
-UNIX_PREFIX = "unix://"
+User = namedtuple("User", ("username", "permissions"))
 
-ADDR_TYPE_UNIX = 0
-ADDR_TYPE_TCP = 1
+def create_server(
+    addr,
+    dbname,
+    *,
+    sync=True,
+    upstream=None,
+    read_only=False,
+    db_username=None,
+    db_password=None,
+    anon_perms=None,
+    admin_username=None,
+    admin_password=None,
+):
+    def sqlite_engine():
+        from .sqlite import DatabaseEngine
 
-# The Python async server defaults to a 64K receive buffer, so we hardcode our
-# maximum chunk size. It would be better if the client and server reported to
-# each other what the maximum chunk sizes were, but that will slow down the
-# connection setup with a round trip delay so I'd rather not do that unless it
-# is necessary
-DEFAULT_MAX_CHUNK = 32 * 1024
+        return DatabaseEngine(dbname, sync)
 
-UNIHASH_TABLE_DEFINITION = (
-    ("method", "TEXT NOT NULL", "UNIQUE"),
-    ("taskhash", "TEXT NOT NULL", "UNIQUE"),
-    ("unihash", "TEXT NOT NULL", ""),
-)
+    def sqlalchemy_engine():
+        from .sqlalchemy import DatabaseEngine
 
-UNIHASH_TABLE_COLUMNS = tuple(name for name, _, _ in UNIHASH_TABLE_DEFINITION)
+        return DatabaseEngine(dbname, db_username, db_password)
 
-OUTHASH_TABLE_DEFINITION = (
-    ("method", "TEXT NOT NULL", "UNIQUE"),
-    ("taskhash", "TEXT NOT NULL", "UNIQUE"),
-    ("outhash", "TEXT NOT NULL", "UNIQUE"),
-    ("created", "DATETIME", ""),
-
-    # Optional fields
-    ("owner", "TEXT", ""),
-    ("PN", "TEXT", ""),
-    ("PV", "TEXT", ""),
-    ("PR", "TEXT", ""),
-    ("task", "TEXT", ""),
-    ("outhash_siginfo", "TEXT", ""),
-)
-
-OUTHASH_TABLE_COLUMNS = tuple(name for name, _, _ in OUTHASH_TABLE_DEFINITION)
-
-def _make_table(cursor, name, definition):
-    cursor.execute('''
-        CREATE TABLE IF NOT EXISTS {name} (
-            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            {fields}
-            UNIQUE({unique})
-            )
-        '''.format(
-            name=name,
-            fields=" ".join("%s %s," % (name, typ) for name, typ, _ in definition),
-            unique=", ".join(name for name, _, flags in definition if "UNIQUE" in flags)
-    ))
-
-
-def setup_database(database, sync=True):
-    db = sqlite3.connect(database)
-    db.row_factory = sqlite3.Row
-
-    with closing(db.cursor()) as cursor:
-        _make_table(cursor, "unihashes_v2", UNIHASH_TABLE_DEFINITION)
-        _make_table(cursor, "outhashes_v2", OUTHASH_TABLE_DEFINITION)
-
-        cursor.execute('PRAGMA journal_mode = WAL')
-        cursor.execute('PRAGMA synchronous = %s' % ('NORMAL' if sync else 'OFF'))
-
-        # Drop old indexes
-        cursor.execute('DROP INDEX IF EXISTS taskhash_lookup')
-        cursor.execute('DROP INDEX IF EXISTS outhash_lookup')
-        cursor.execute('DROP INDEX IF EXISTS taskhash_lookup_v2')
-        cursor.execute('DROP INDEX IF EXISTS outhash_lookup_v2')
-
-        # TODO: Upgrade from tasks_v2?
-        cursor.execute('DROP TABLE IF EXISTS tasks_v2')
-
-        # Create new indexes
-        cursor.execute('CREATE INDEX IF NOT EXISTS taskhash_lookup_v3 ON unihashes_v2 (method, taskhash)')
-        cursor.execute('CREATE INDEX IF NOT EXISTS outhash_lookup_v3 ON outhashes_v2 (method, outhash)')
-
-    return db
-
-
-def parse_address(addr):
-    if addr.startswith(UNIX_PREFIX):
-        return (ADDR_TYPE_UNIX, (addr[len(UNIX_PREFIX):],))
-    else:
-        m = re.match(r'\[(?P<host>[^\]]*)\]:(?P<port>\d+)$', addr)
-        if m is not None:
-            host = m.group('host')
-            port = m.group('port')
-        else:
-            host, port = addr.split(':')
-
-        return (ADDR_TYPE_TCP, (host, int(port)))
-
-
-def chunkify(msg, max_chunk):
-    if len(msg) < max_chunk - 1:
-        yield ''.join((msg, "\n"))
-    else:
-        yield ''.join((json.dumps({
-                'chunk-stream': None
-            }), "\n"))
-
-        args = [iter(msg)] * (max_chunk - 1)
-        for m in map(''.join, itertools.zip_longest(*args, fillvalue='')):
-            yield ''.join(itertools.chain(m, "\n"))
-        yield "\n"
-
-
-def create_server(addr, dbname, *, sync=True, upstream=None, read_only=False):
     from . import server
-    db = setup_database(dbname, sync=sync)
-    s = server.Server(db, upstream=upstream, read_only=read_only)
+
+    if "://" in dbname:
+        db_engine = sqlalchemy_engine()
+    else:
+        db_engine = sqlite_engine()
+
+    if anon_perms is None:
+        anon_perms = server.DEFAULT_ANON_PERMS
+
+    s = server.Server(
+        db_engine,
+        upstream=upstream,
+        read_only=read_only,
+        anon_perms=anon_perms,
+        admin_username=admin_username,
+        admin_password=admin_password,
+    )
 
     (typ, a) = parse_address(addr)
     if typ == ADDR_TYPE_UNIX:
         s.start_unix_server(*a)
+    elif typ == ADDR_TYPE_WS:
+        url = urlparse(a[0])
+        s.start_websocket_server(url.hostname, url.port)
     else:
         s.start_tcp_server(*a)
 
     return s
 
 
-def create_client(addr):
+def create_client(addr, username=None, password=None):
     from . import client
-    c = client.Client()
 
-    (typ, a) = parse_address(addr)
-    if typ == ADDR_TYPE_UNIX:
-        c.connect_unix(*a)
-    else:
-        c.connect_tcp(*a)
+    c = client.Client(username, password)
 
-    return c
+    try:
+        (typ, a) = parse_address(addr)
+        if typ == ADDR_TYPE_UNIX:
+            c.connect_unix(*a)
+        elif typ == ADDR_TYPE_WS:
+            c.connect_websocket(*a)
+        else:
+            c.connect_tcp(*a)
+        return c
+    except Exception as e:
+        c.close()
+        raise e
 
-async def create_async_client(addr):
+
+async def create_async_client(addr, username=None, password=None):
     from . import client
-    c = client.AsyncClient()
 
-    (typ, a) = parse_address(addr)
-    if typ == ADDR_TYPE_UNIX:
-        await c.connect_unix(*a)
-    else:
-        await c.connect_tcp(*a)
+    c = client.AsyncClient(username, password)
 
-    return c
+    try:
+        (typ, a) = parse_address(addr)
+        if typ == ADDR_TYPE_UNIX:
+            await c.connect_unix(*a)
+        elif typ == ADDR_TYPE_WS:
+            await c.connect_websocket(*a)
+        else:
+            await c.connect_tcp(*a)
+
+        return c
+    except Exception as e:
+        await c.close()
+        raise e

bitbake/lib/hashserv/client.py

@@ -5,127 +5,430 @@
 
 import logging
 import socket
+import asyncio
 import bb.asyncrpc
+import json
 from . import create_async_client
 
 
 logger = logging.getLogger("hashserv.client")
 
 
+class Batch(object):
+    def __init__(self):
+        self.done = False
+        self.cond = asyncio.Condition()
+        self.pending = []
+        self.results = []
+        self.sent_count = 0
+
+    async def recv(self, socket):
+        while True:
+            async with self.cond:
+                await self.cond.wait_for(lambda: self.pending or self.done)
+
+                if not self.pending:
+                    if self.done:
+                        return
+                    continue
+
+            r = await socket.recv()
+            self.results.append(r)
+
+            async with self.cond:
+                self.pending.pop(0)
+
+    async def send(self, socket, msgs):
+        try:
+            # In the event of a restart due to a reconnect, all in-flight
+            # messages need to be resent first to keep to result count in sync
+            for m in self.pending:
+                await socket.send(m)
+
+            for m in msgs:
+                # Add the message to the pending list before attempting to send
+                # it so that if the send fails it will be retried
+                async with self.cond:
+                    self.pending.append(m)
+                    self.cond.notify()
+                    self.sent_count += 1
+
+                await socket.send(m)
+
+        finally:
+            async with self.cond:
+                self.done = True
+                self.cond.notify()
+
+    async def process(self, socket, msgs):
+        await asyncio.gather(
+            self.recv(socket),
+            self.send(socket, msgs),
+        )
+
+        if len(self.results) != self.sent_count:
+            raise ValueError(
+                f"Expected result count {len(self.results)}. Expected {self.sent_count}"
+            )
+
+        return self.results
+
+
 class AsyncClient(bb.asyncrpc.AsyncClient):
     MODE_NORMAL = 0
     MODE_GET_STREAM = 1
+    MODE_EXIST_STREAM = 2
 
-    def __init__(self):
-        super().__init__('OEHASHEQUIV', '1.1', logger)
+    def __init__(self, username=None, password=None):
+        super().__init__("OEHASHEQUIV", "1.1", logger)
         self.mode = self.MODE_NORMAL
+        self.username = username
+        self.password = password
+        self.saved_become_user = None
 
     async def setup_connection(self):
         await super().setup_connection()
-        cur_mode = self.mode
         self.mode = self.MODE_NORMAL
-        await self._set_mode(cur_mode)
+        if self.username:
+            # Save off become user temporarily because auth() resets it
+            become = self.saved_become_user
+            await self.auth(self.username, self.password)
+
+            if become:
+                await self.become_user(become)
+
+    async def send_stream_batch(self, mode, msgs):
+        """
+        Does a "batch" process of stream messages. This sends the query
+        messages as fast as possible, and simultaneously attempts to read the
+        messages back. This helps to mitigate the effects of latency to the
+        hash equivalence server be allowing multiple queries to be "in-flight"
+        at once
+
+        The implementation does more complicated tracking using a count of sent
+        messages so that `msgs` can be a generator function (i.e. its length is
+        unknown)
+
+        """
+
+        b = Batch()
 
-    async def send_stream(self, msg):
         async def proc():
-            self.writer.write(("%s\n" % msg).encode("utf-8"))
-            await self.writer.drain()
-            l = await self.reader.readline()
-            if not l:
-                raise ConnectionError("Connection closed")
-            return l.decode("utf-8").rstrip()
+            nonlocal b
+
+            await self._set_mode(mode)
+            return await b.process(self.socket, msgs)
 
         return await self._send_wrapper(proc)
 
+    async def invoke(self, *args, **kwargs):
+        # It's OK if connection errors cause a failure here, because the mode
+        # is also reset to normal on a new connection
+        await self._set_mode(self.MODE_NORMAL)
+        return await super().invoke(*args, **kwargs)
+
     async def _set_mode(self, new_mode):
-        if new_mode == self.MODE_NORMAL and self.mode == self.MODE_GET_STREAM:
-            r = await self.send_stream("END")
+        async def stream_to_normal():
+            await self.socket.send("END")
+            return await self.socket.recv()
+
+        async def normal_to_stream(command):
+            r = await self.invoke({command: None})
             if r != "ok":
-                raise ConnectionError("Bad response from server %r" % r)
-        elif new_mode == self.MODE_GET_STREAM and self.mode == self.MODE_NORMAL:
-            r = await self.send_message({"get-stream": None})
+                raise ConnectionError(
+                    f"Unable to transition to stream mode: Bad response from server {r!r}"
+                )
+
+            self.logger.debug("Mode is now %s", command)
+
+        if new_mode == self.mode:
+            return
+
+        self.logger.debug("Transitioning mode %s -> %s", self.mode, new_mode)
+
+        # Always transition to normal mode before switching to any other mode
+        if self.mode != self.MODE_NORMAL:
+            r = await self._send_wrapper(stream_to_normal)
             if r != "ok":
-                raise ConnectionError("Bad response from server %r" % r)
-        elif new_mode != self.mode:
-            raise Exception(
-                "Undefined mode transition %r -> %r" % (self.mode, new_mode)
-            )
+                self.check_invoke_error(r)
+                raise ConnectionError(
+                    f"Unable to transition to normal mode: Bad response from server {r!r}"
+                )
+            self.logger.debug("Mode is now normal")
+
+        if new_mode == self.MODE_GET_STREAM:
+            await normal_to_stream("get-stream")
+        elif new_mode == self.MODE_EXIST_STREAM:
+            await normal_to_stream("exists-stream")
+        elif new_mode != self.MODE_NORMAL:
+            raise Exception("Undefined mode transition {self.mode!r} -> {new_mode!r}")
 
         self.mode = new_mode
 
     async def get_unihash(self, method, taskhash):
-        await self._set_mode(self.MODE_GET_STREAM)
-        r = await self.send_stream("%s %s" % (method, taskhash))
-        if not r:
-            return None
-        return r
+        r = await self.get_unihash_batch([(method, taskhash)])
+        return r[0]
+
+    async def get_unihash_batch(self, args):
+        result = await self.send_stream_batch(
+            self.MODE_GET_STREAM,
+            (f"{method} {taskhash}" for method, taskhash in args),
+        )
+        return [r if r else None for r in result]
 
     async def report_unihash(self, taskhash, method, outhash, unihash, extra={}):
-        await self._set_mode(self.MODE_NORMAL)
         m = extra.copy()
         m["taskhash"] = taskhash
         m["method"] = method
         m["outhash"] = outhash
         m["unihash"] = unihash
-        return await self.send_message({"report": m})
+        return await self.invoke({"report": m})
 
     async def report_unihash_equiv(self, taskhash, method, unihash, extra={}):
-        await self._set_mode(self.MODE_NORMAL)
         m = extra.copy()
         m["taskhash"] = taskhash
         m["method"] = method
         m["unihash"] = unihash
-        return await self.send_message({"report-equiv": m})
+        return await self.invoke({"report-equiv": m})
 
     async def get_taskhash(self, method, taskhash, all_properties=False):
-        await self._set_mode(self.MODE_NORMAL)
-        return await self.send_message(
+        return await self.invoke(
             {"get": {"taskhash": taskhash, "method": method, "all": all_properties}}
         )
 
+    async def unihash_exists(self, unihash):
+        r = await self.unihash_exists_batch([unihash])
+        return r[0]
+
+    async def unihash_exists_batch(self, unihashes):
+        result = await self.send_stream_batch(self.MODE_EXIST_STREAM, unihashes)
+        return [r == "true" for r in result]
+
     async def get_outhash(self, method, outhash, taskhash, with_unihash=True):
-        await self._set_mode(self.MODE_NORMAL)
-        return await self.send_message(
-            {"get-outhash": {"outhash": outhash, "taskhash": taskhash, "method": method, "with_unihash": with_unihash}}
+        return await self.invoke(
+            {
+                "get-outhash": {
+                    "outhash": outhash,
+                    "taskhash": taskhash,
+                    "method": method,
+                    "with_unihash": with_unihash,
+                }
+            }
         )
 
     async def get_stats(self):
-        await self._set_mode(self.MODE_NORMAL)
-        return await self.send_message({"get-stats": None})
+        return await self.invoke({"get-stats": None})
 
     async def reset_stats(self):
-        await self._set_mode(self.MODE_NORMAL)
-        return await self.send_message({"reset-stats": None})
+        return await self.invoke({"reset-stats": None})
 
     async def backfill_wait(self):
-        await self._set_mode(self.MODE_NORMAL)
-        return (await self.send_message({"backfill-wait": None}))["tasks"]
+        return (await self.invoke({"backfill-wait": None}))["tasks"]
 
     async def remove(self, where):
-        await self._set_mode(self.MODE_NORMAL)
-        return await self.send_message({"remove": {"where": where}})
+        return await self.invoke({"remove": {"where": where}})
 
     async def clean_unused(self, max_age):
-        await self._set_mode(self.MODE_NORMAL)
-        return await self.send_message({"clean-unused": {"max_age_seconds": max_age}})
+        return await self.invoke({"clean-unused": {"max_age_seconds": max_age}})
+
+    async def auth(self, username, token):
+        result = await self.invoke({"auth": {"username": username, "token": token}})
+        self.username = username
+        self.password = token
+        self.saved_become_user = None
+        return result
+
+    async def refresh_token(self, username=None):
+        m = {}
+        if username:
+            m["username"] = username
+        result = await self.invoke({"refresh-token": m})
+        if (
+            self.username
+            and not self.saved_become_user
+            and result["username"] == self.username
+        ):
+            self.password = result["token"]
+        return result
+
+    async def set_user_perms(self, username, permissions):
+        return await self.invoke(
+            {"set-user-perms": {"username": username, "permissions": permissions}}
+        )
+
+    async def get_user(self, username=None):
+        m = {}
+        if username:
+            m["username"] = username
+        return await self.invoke({"get-user": m})
+
+    async def get_all_users(self):
+        return (await self.invoke({"get-all-users": {}}))["users"]
+
+    async def new_user(self, username, permissions):
+        return await self.invoke(
+            {"new-user": {"username": username, "permissions": permissions}}
+        )
+
+    async def delete_user(self, username):
+        return await self.invoke({"delete-user": {"username": username}})
+
+    async def become_user(self, username):
+        result = await self.invoke({"become-user": {"username": username}})
+        if username == self.username:
+            self.saved_become_user = None
+        else:
+            self.saved_become_user = username
+        return result
+
+    async def get_db_usage(self):
+        return (await self.invoke({"get-db-usage": {}}))["usage"]
+
+    async def get_db_query_columns(self):
+        return (await self.invoke({"get-db-query-columns": {}}))["columns"]
+
+    async def gc_status(self):
+        return await self.invoke({"gc-status": {}})
+
+    async def gc_mark(self, mark, where):
+        """
+        Starts a new garbage collection operation identified by "mark". If
+        garbage collection is already in progress with "mark", the collection
+        is continued.
+
+        All unihash entries that match the "where" clause are marked to be
+        kept. In addition, any new entries added to the database after this
+        command will be automatically marked with "mark"
+        """
+        return await self.invoke({"gc-mark": {"mark": mark, "where": where}})
+
+    async def gc_sweep(self, mark):
+        """
+        Finishes garbage collection for "mark". All unihash entries that have
+        not been marked will be deleted.
+
+        It is recommended to clean unused outhash entries after running this to
+        cleanup any dangling outhashes
+        """
+        return await self.invoke({"gc-sweep": {"mark": mark}})
 
 
 class Client(bb.asyncrpc.Client):
-    def __init__(self):
+    def __init__(self, username=None, password=None):
+        self.username = username
+        self.password = password
+
         super().__init__()
         self._add_methods(
             "connect_tcp",
+            "connect_websocket",
             "get_unihash",
+            "get_unihash_batch",
             "report_unihash",
             "report_unihash_equiv",
             "get_taskhash",
+            "unihash_exists",
+            "unihash_exists_batch",
             "get_outhash",
             "get_stats",
             "reset_stats",
             "backfill_wait",
             "remove",
             "clean_unused",
+            "auth",
+            "refresh_token",
+            "set_user_perms",
+            "get_user",
+            "get_all_users",
+            "new_user",
+            "delete_user",
+            "become_user",
+            "get_db_usage",
+            "get_db_query_columns",
+            "gc_status",
+            "gc_mark",
+            "gc_sweep",
         )
 
     def _get_async_client(self):
-        return AsyncClient()
+        return AsyncClient(self.username, self.password)
+
+
+class ClientPool(bb.asyncrpc.ClientPool):
+    def __init__(
+        self,
+        address,
+        max_clients,
+        *,
+        username=None,
+        password=None,
+        become=None,
+    ):
+        super().__init__(max_clients)
+        self.address = address
+        self.username = username
+        self.password = password
+        self.become = become
+
+    async def _new_client(self):
+        client = await create_async_client(
+            self.address,
+            username=self.username,
+            password=self.password,
+        )
+        if self.become:
+            await client.become_user(self.become)
+        return client
+
+    def _run_key_tasks(self, queries, call):
+        results = {key: None for key in queries.keys()}
+
+        def make_task(key, args):
+            async def task(client):
+                nonlocal results
+                unihash = await call(client, args)
+                results[key] = unihash
+
+            return task
+
+        def gen_tasks():
+            for key, args in queries.items():
+                yield make_task(key, args)
+
+        self.run_tasks(gen_tasks())
+        return results
+
+    def get_unihashes(self, queries):
+        """
+        Query multiple unihashes in parallel.
+
+        The queries argument is a dictionary with arbitrary key. The values
+        must be a tuple of (method, taskhash).
+
+        Returns a dictionary with a corresponding key for each input key, and
+        the value is the queried unihash (which might be none if the query
+        failed)
+        """
+
+        async def call(client, args):
+            method, taskhash = args
+            return await client.get_unihash(method, taskhash)
+
+        return self._run_key_tasks(queries, call)
+
+    def unihashes_exist(self, queries):
+        """
+        Query multiple unihash existence checks in parallel.
+
+        The queries argument is a dictionary with arbitrary key. The values
+        must be a unihash.
+
+        Returns a dictionary with a corresponding key for each input key, and
+        the value is True or False if the unihash is known by the server (or
+        None if there was a failure)
+        """
+
+        async def call(client, unihash):
+            return await client.unihash_exists(unihash)
+
+        return self._run_key_tasks(queries, call)

bitbake/lib/hashserv/sqlalchemy.py

@@ -0,0 +1,598 @@
+#! /usr/bin/env python3
+#
+# Copyright (C) 2023 Garmin Ltd.
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import logging
+from datetime import datetime
+from . import User
+
+from sqlalchemy.ext.asyncio import create_async_engine
+from sqlalchemy.pool import NullPool
+from sqlalchemy import (
+    MetaData,
+    Column,
+    Table,
+    Text,
+    Integer,
+    UniqueConstraint,
+    DateTime,
+    Index,
+    select,
+    insert,
+    exists,
+    literal,
+    and_,
+    delete,
+    update,
+    func,
+    inspect,
+)
+import sqlalchemy.engine
+from sqlalchemy.orm import declarative_base
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.dialects.postgresql import insert as postgres_insert
+
+Base = declarative_base()
+
+
+class UnihashesV3(Base):
+    __tablename__ = "unihashes_v3"
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    method = Column(Text, nullable=False)
+    taskhash = Column(Text, nullable=False)
+    unihash = Column(Text, nullable=False)
+    gc_mark = Column(Text, nullable=False)
+
+    __table_args__ = (
+        UniqueConstraint("method", "taskhash"),
+        Index("taskhash_lookup_v4", "method", "taskhash"),
+        Index("unihash_lookup_v1", "unihash"),
+    )
+
+
+class OuthashesV2(Base):
+    __tablename__ = "outhashes_v2"
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    method = Column(Text, nullable=False)
+    taskhash = Column(Text, nullable=False)
+    outhash = Column(Text, nullable=False)
+    created = Column(DateTime)
+    owner = Column(Text)
+    PN = Column(Text)
+    PV = Column(Text)
+    PR = Column(Text)
+    task = Column(Text)
+    outhash_siginfo = Column(Text)
+
+    __table_args__ = (
+        UniqueConstraint("method", "taskhash", "outhash"),
+        Index("outhash_lookup_v3", "method", "outhash"),
+    )
+
+
+class Users(Base):
+    __tablename__ = "users"
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    username = Column(Text, nullable=False)
+    token = Column(Text, nullable=False)
+    permissions = Column(Text)
+
+    __table_args__ = (UniqueConstraint("username"),)
+
+
+class Config(Base):
+    __tablename__ = "config"
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    name = Column(Text, nullable=False)
+    value = Column(Text)
+    __table_args__ = (
+        UniqueConstraint("name"),
+        Index("config_lookup", "name"),
+    )
+
+
+#
+# Old table versions
+#
+DeprecatedBase = declarative_base()
+
+
+class UnihashesV2(DeprecatedBase):
+    __tablename__ = "unihashes_v2"
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    method = Column(Text, nullable=False)
+    taskhash = Column(Text, nullable=False)
+    unihash = Column(Text, nullable=False)
+
+    __table_args__ = (
+        UniqueConstraint("method", "taskhash"),
+        Index("taskhash_lookup_v3", "method", "taskhash"),
+    )
+
+
+class DatabaseEngine(object):
+    def __init__(self, url, username=None, password=None):
+        self.logger = logging.getLogger("hashserv.sqlalchemy")
+        self.url = sqlalchemy.engine.make_url(url)
+
+        if username is not None:
+            self.url = self.url.set(username=username)
+
+        if password is not None:
+            self.url = self.url.set(password=password)
+
+    async def create(self):
+        def check_table_exists(conn, name):
+            return inspect(conn).has_table(name)
+
+        self.logger.info("Using database %s", self.url)
+        if self.url.drivername == 'postgresql+psycopg':
+            # Psygopg 3 (psygopg) driver can handle async connection pooling
+            self.engine = create_async_engine(self.url, max_overflow=-1)
+        else:
+            self.engine = create_async_engine(self.url, poolclass=NullPool)
+
+        async with self.engine.begin() as conn:
+            # Create tables
+            self.logger.info("Creating tables...")
+            await conn.run_sync(Base.metadata.create_all)
+
+            if await conn.run_sync(check_table_exists, UnihashesV2.__tablename__):
+                self.logger.info("Upgrading Unihashes V2 -> V3...")
+                statement = insert(UnihashesV3).from_select(
+                    ["id", "method", "unihash", "taskhash", "gc_mark"],
+                    select(
+                        UnihashesV2.id,
+                        UnihashesV2.method,
+                        UnihashesV2.unihash,
+                        UnihashesV2.taskhash,
+                        literal("").label("gc_mark"),
+                    ),
+                )
+                self.logger.debug("%s", statement)
+                await conn.execute(statement)
+
+                await conn.run_sync(Base.metadata.drop_all, [UnihashesV2.__table__])
+                self.logger.info("Upgrade complete")
+
+    def connect(self, logger):
+        return Database(self.engine, logger)
+
+
+def map_row(row):
+    if row is None:
+        return None
+    return dict(**row._mapping)
+
+
+def map_user(row):
+    if row is None:
+        return None
+    return User(
+        username=row.username,
+        permissions=set(row.permissions.split()),
+    )
+
+
+def _make_condition_statement(table, condition):
+    where = {}
+    for c in table.__table__.columns:
+        if c.key in condition and condition[c.key] is not None:
+            where[c] = condition[c.key]
+
+    return [(k == v) for k, v in where.items()]
+
+
+class Database(object):
+    def __init__(self, engine, logger):
+        self.engine = engine
+        self.db = None
+        self.logger = logger
+
+    async def __aenter__(self):
+        self.db = await self.engine.connect()
+        return self
+
+    async def __aexit__(self, exc_type, exc_value, traceback):
+        await self.close()
+
+    async def close(self):
+        await self.db.close()
+        self.db = None
+
+    async def _execute(self, statement):
+        self.logger.debug("%s", statement)
+        return await self.db.execute(statement)
+
+    async def _set_config(self, name, value):
+        while True:
+            result = await self._execute(
+                update(Config).where(Config.name == name).values(value=value)
+            )
+
+            if result.rowcount == 0:
+                self.logger.debug("Config '%s' not found. Adding it", name)
+                try:
+                    await self._execute(insert(Config).values(name=name, value=value))
+                except IntegrityError:
+                    # Race. Try again
+                    continue
+
+            break
+
+    def _get_config_subquery(self, name, default=None):
+        if default is not None:
+            return func.coalesce(
+                select(Config.value).where(Config.name == name).scalar_subquery(),
+                default,
+            )
+        return select(Config.value).where(Config.name == name).scalar_subquery()
+
+    async def _get_config(self, name):
+        result = await self._execute(select(Config.value).where(Config.name == name))
+        row = result.first()
+        if row is None:
+            return None
+        return row.value
+
+    async def get_unihash_by_taskhash_full(self, method, taskhash):
+        async with self.db.begin():
+            result = await self._execute(
+                select(
+                    OuthashesV2,
+                    UnihashesV3.unihash.label("unihash"),
+                )
+                .join(
+                    UnihashesV3,
+                    and_(
+                        UnihashesV3.method == OuthashesV2.method,
+                        UnihashesV3.taskhash == OuthashesV2.taskhash,
+                    ),
+                )
+                .where(
+                    OuthashesV2.method == method,
+                    OuthashesV2.taskhash == taskhash,
+                )
+                .order_by(
+                    OuthashesV2.created.asc(),
+                )
+                .limit(1)
+            )
+            return map_row(result.first())
+
+    async def get_unihash_by_outhash(self, method, outhash):
+        async with self.db.begin():
+            result = await self._execute(
+                select(OuthashesV2, UnihashesV3.unihash.label("unihash"))
+                .join(
+                    UnihashesV3,
+                    and_(
+                        UnihashesV3.method == OuthashesV2.method,
+                        UnihashesV3.taskhash == OuthashesV2.taskhash,
+                    ),
+                )
+                .where(
+                    OuthashesV2.method == method,
+                    OuthashesV2.outhash == outhash,
+                )
+                .order_by(
+                    OuthashesV2.created.asc(),
+                )
+                .limit(1)
+            )
+            return map_row(result.first())
+
+    async def unihash_exists(self, unihash):
+        async with self.db.begin():
+            result = await self._execute(
+                select(UnihashesV3).where(UnihashesV3.unihash == unihash).limit(1)
+            )
+
+            return result.first() is not None
+
+    async def get_outhash(self, method, outhash):
+        async with self.db.begin():
+            result = await self._execute(
+                select(OuthashesV2)
+                .where(
+                    OuthashesV2.method == method,
+                    OuthashesV2.outhash == outhash,
+                )
+                .order_by(
+                    OuthashesV2.created.asc(),
+                )
+                .limit(1)
+            )
+            return map_row(result.first())
+
+    async def get_equivalent_for_outhash(self, method, outhash, taskhash):
+        async with self.db.begin():
+            result = await self._execute(
+                select(
+                    OuthashesV2.taskhash.label("taskhash"),
+                    UnihashesV3.unihash.label("unihash"),
+                )
+                .join(
+                    UnihashesV3,
+                    and_(
+                        UnihashesV3.method == OuthashesV2.method,
+                        UnihashesV3.taskhash == OuthashesV2.taskhash,
+                    ),
+                )
+                .where(
+                    OuthashesV2.method == method,
+                    OuthashesV2.outhash == outhash,
+                    OuthashesV2.taskhash != taskhash,
+                )
+                .order_by(
+                    OuthashesV2.created.asc(),
+                )
+                .limit(1)
+            )
+            return map_row(result.first())
+
+    async def get_equivalent(self, method, taskhash):
+        async with self.db.begin():
+            result = await self._execute(
+                select(
+                    UnihashesV3.unihash,
+                    UnihashesV3.method,
+                    UnihashesV3.taskhash,
+                ).where(
+                    UnihashesV3.method == method,
+                    UnihashesV3.taskhash == taskhash,
+                )
+            )
+            return map_row(result.first())
+
+    async def remove(self, condition):
+        async def do_remove(table):
+            where = _make_condition_statement(table, condition)
+            if where:
+                async with self.db.begin():
+                    result = await self._execute(delete(table).where(*where))
+                return result.rowcount
+
+            return 0
+
+        count = 0
+        count += await do_remove(UnihashesV3)
+        count += await do_remove(OuthashesV2)
+
+        return count
+
+    async def get_current_gc_mark(self):
+        async with self.db.begin():
+            return await self._get_config("gc-mark")
+
+    async def gc_status(self):
+        async with self.db.begin():
+            gc_mark_subquery = self._get_config_subquery("gc-mark", "")
+
+            result = await self._execute(
+                select(func.count())
+                .select_from(UnihashesV3)
+                .where(UnihashesV3.gc_mark == gc_mark_subquery)
+            )
+            keep_rows = result.scalar()
+
+            result = await self._execute(
+                select(func.count())
+                .select_from(UnihashesV3)
+                .where(UnihashesV3.gc_mark != gc_mark_subquery)
+            )
+            remove_rows = result.scalar()
+
+            return (keep_rows, remove_rows, await self._get_config("gc-mark"))
+
+    async def gc_mark(self, mark, condition):
+        async with self.db.begin():
+            await self._set_config("gc-mark", mark)
+
+            where = _make_condition_statement(UnihashesV3, condition)
+            if not where:
+                return 0
+
+            result = await self._execute(
+                update(UnihashesV3)
+                .values(gc_mark=self._get_config_subquery("gc-mark", ""))
+                .where(*where)
+            )
+            return result.rowcount
+
+    async def gc_sweep(self):
+        async with self.db.begin():
+            result = await self._execute(
+                delete(UnihashesV3).where(
+                    # A sneaky conditional that provides some errant use
+                    # protection: If the config mark is NULL, this will not
+                    # match any rows because No default is specified in the
+                    # select statement
+                    UnihashesV3.gc_mark
+                    != self._get_config_subquery("gc-mark")
+                )
+            )
+            await self._set_config("gc-mark", None)
+
+            return result.rowcount
+
+    async def clean_unused(self, oldest):
+        async with self.db.begin():
+            result = await self._execute(
+                delete(OuthashesV2).where(
+                    OuthashesV2.created < oldest,
+                    ~(
+                        select(UnihashesV3.id)
+                        .where(
+                            UnihashesV3.method == OuthashesV2.method,
+                            UnihashesV3.taskhash == OuthashesV2.taskhash,
+                        )
+                        .limit(1)
+                        .exists()
+                    ),
+                )
+            )
+            return result.rowcount
+
+    async def insert_unihash(self, method, taskhash, unihash):
+        # Postgres specific ignore on insert duplicate
+        if self.engine.name == "postgresql":
+            statement = (
+                postgres_insert(UnihashesV3)
+                .values(
+                    method=method,
+                    taskhash=taskhash,
+                    unihash=unihash,
+                    gc_mark=self._get_config_subquery("gc-mark", ""),
+                )
+                .on_conflict_do_nothing(index_elements=("method", "taskhash"))
+            )
+        else:
+            statement = insert(UnihashesV3).values(
+                method=method,
+                taskhash=taskhash,
+                unihash=unihash,
+                gc_mark=self._get_config_subquery("gc-mark", ""),
+            )
+
+        try:
+            async with self.db.begin():
+                result = await self._execute(statement)
+                return result.rowcount != 0
+        except IntegrityError:
+            self.logger.debug(
+                "%s, %s, %s already in unihash database", method, taskhash, unihash
+            )
+            return False
+
+    async def insert_outhash(self, data):
+        outhash_columns = set(c.key for c in OuthashesV2.__table__.columns)
+
+        data = {k: v for k, v in data.items() if k in outhash_columns}
+
+        if "created" in data and not isinstance(data["created"], datetime):
+            data["created"] = datetime.fromisoformat(data["created"])
+
+        # Postgres specific ignore on insert duplicate
+        if self.engine.name == "postgresql":
+            statement = (
+                postgres_insert(OuthashesV2)
+                .values(**data)
+                .on_conflict_do_nothing(
+                    index_elements=("method", "taskhash", "outhash")
+                )
+            )
+        else:
+            statement = insert(OuthashesV2).values(**data)
+
+        try:
+            async with self.db.begin():
+                result = await self._execute(statement)
+                return result.rowcount != 0
+        except IntegrityError:
+            self.logger.debug(
+                "%s, %s already in outhash database", data["method"], data["outhash"]
+            )
+            return False
+
+    async def _get_user(self, username):
+        async with self.db.begin():
+            result = await self._execute(
+                select(
+                    Users.username,
+                    Users.permissions,
+                    Users.token,
+                ).where(
+                    Users.username == username,
+                )
+            )
+            return result.first()
+
+    async def lookup_user_token(self, username):
+        row = await self._get_user(username)
+        if not row:
+            return None, None
+        return map_user(row), row.token
+
+    async def lookup_user(self, username):
+        return map_user(await self._get_user(username))
+
+    async def set_user_token(self, username, token):
+        async with self.db.begin():
+            result = await self._execute(
+                update(Users)
+                .where(
+                    Users.username == username,
+                )
+                .values(
+                    token=token,
+                )
+            )
+            return result.rowcount != 0
+
+    async def set_user_perms(self, username, permissions):
+        async with self.db.begin():
+            result = await self._execute(
+                update(Users)
+                .where(Users.username == username)
+                .values(permissions=" ".join(permissions))
+            )
+            return result.rowcount != 0
+
+    async def get_all_users(self):
+        async with self.db.begin():
+            result = await self._execute(
+                select(
+                    Users.username,
+                    Users.permissions,
+                )
+            )
+            return [map_user(row) for row in result]
+
+    async def new_user(self, username, permissions, token):
+        try:
+            async with self.db.begin():
+                await self._execute(
+                    insert(Users).values(
+                        username=username,
+                        permissions=" ".join(permissions),
+                        token=token,
+                    )
+                )
+            return True
+        except IntegrityError as e:
+            self.logger.debug("Cannot create new user %s: %s", username, e)
+            return False
+
+    async def delete_user(self, username):
+        async with self.db.begin():
+            result = await self._execute(
+                delete(Users).where(Users.username == username)
+            )
+            return result.rowcount != 0
+
+    async def get_usage(self):
+        usage = {}
+        async with self.db.begin() as session:
+            for name, table in Base.metadata.tables.items():
+                result = await self._execute(
+                    statement=select(func.count()).select_from(table)
+                )
+                usage[name] = {
+                    "rows": result.scalar(),
+                }
+
+        return usage
+
+    async def get_query_columns(self):
+        columns = set()
+        for table in (UnihashesV3, OuthashesV2):
+            for c in table.__table__.columns:
+                if not isinstance(c.type, Text):
+                    continue
+                columns.add(c.key)
+
+        return list(columns)

bitbake/lib/hashserv/sqlite.py

@@ -0,0 +1,562 @@
+#! /usr/bin/env python3
+#
+# Copyright (C) 2023 Garmin Ltd.
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+import sqlite3
+import logging
+from contextlib import closing
+from . import User
+
+logger = logging.getLogger("hashserv.sqlite")
+
+UNIHASH_TABLE_DEFINITION = (
+    ("method", "TEXT NOT NULL", "UNIQUE"),
+    ("taskhash", "TEXT NOT NULL", "UNIQUE"),
+    ("unihash", "TEXT NOT NULL", ""),
+    ("gc_mark", "TEXT NOT NULL", ""),
+)
+
+UNIHASH_TABLE_COLUMNS = tuple(name for name, _, _ in UNIHASH_TABLE_DEFINITION)
+
+OUTHASH_TABLE_DEFINITION = (
+    ("method", "TEXT NOT NULL", "UNIQUE"),
+    ("taskhash", "TEXT NOT NULL", "UNIQUE"),
+    ("outhash", "TEXT NOT NULL", "UNIQUE"),
+    ("created", "DATETIME", ""),
+    # Optional fields
+    ("owner", "TEXT", ""),
+    ("PN", "TEXT", ""),
+    ("PV", "TEXT", ""),
+    ("PR", "TEXT", ""),
+    ("task", "TEXT", ""),
+    ("outhash_siginfo", "TEXT", ""),
+)
+
+OUTHASH_TABLE_COLUMNS = tuple(name for name, _, _ in OUTHASH_TABLE_DEFINITION)
+
+USERS_TABLE_DEFINITION = (
+    ("username", "TEXT NOT NULL", "UNIQUE"),
+    ("token", "TEXT NOT NULL", ""),
+    ("permissions", "TEXT NOT NULL", ""),
+)
+
+USERS_TABLE_COLUMNS = tuple(name for name, _, _ in USERS_TABLE_DEFINITION)
+
+
+CONFIG_TABLE_DEFINITION = (
+    ("name", "TEXT NOT NULL", "UNIQUE"),
+    ("value", "TEXT", ""),
+)
+
+CONFIG_TABLE_COLUMNS = tuple(name for name, _, _ in CONFIG_TABLE_DEFINITION)
+
+
+def _make_table(cursor, name, definition):
+    cursor.execute(
+        """
+        CREATE TABLE IF NOT EXISTS {name} (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            {fields}
+            UNIQUE({unique})
+            )
+        """.format(
+            name=name,
+            fields=" ".join("%s %s," % (name, typ) for name, typ, _ in definition),
+            unique=", ".join(
+                name for name, _, flags in definition if "UNIQUE" in flags
+            ),
+        )
+    )
+
+
+def map_user(row):
+    if row is None:
+        return None
+    return User(
+        username=row["username"],
+        permissions=set(row["permissions"].split()),
+    )
+
+
+def _make_condition_statement(columns, condition):
+    where = {}
+    for c in columns:
+        if c in condition and condition[c] is not None:
+            where[c] = condition[c]
+
+    return where, " AND ".join("%s=:%s" % (k, k) for k in where.keys())
+
+
+def _get_sqlite_version(cursor):
+    cursor.execute("SELECT sqlite_version()")
+
+    version = []
+    for v in cursor.fetchone()[0].split("."):
+        try:
+            version.append(int(v))
+        except ValueError:
+            version.append(v)
+
+    return tuple(version)
+
+
+def _schema_table_name(version):
+    if version >= (3, 33):
+        return "sqlite_schema"
+
+    return "sqlite_master"
+
+
+class DatabaseEngine(object):
+    def __init__(self, dbname, sync):
+        self.dbname = dbname
+        self.logger = logger
+        self.sync = sync
+
+    async def create(self):
+        db = sqlite3.connect(self.dbname)
+        db.row_factory = sqlite3.Row
+
+        with closing(db.cursor()) as cursor:
+            _make_table(cursor, "unihashes_v3", UNIHASH_TABLE_DEFINITION)
+            _make_table(cursor, "outhashes_v2", OUTHASH_TABLE_DEFINITION)
+            _make_table(cursor, "users", USERS_TABLE_DEFINITION)
+            _make_table(cursor, "config", CONFIG_TABLE_DEFINITION)
+
+            cursor.execute("PRAGMA journal_mode = WAL")
+            cursor.execute(
+                "PRAGMA synchronous = %s" % ("NORMAL" if self.sync else "OFF")
+            )
+
+            # Drop old indexes
+            cursor.execute("DROP INDEX IF EXISTS taskhash_lookup")
+            cursor.execute("DROP INDEX IF EXISTS outhash_lookup")
+            cursor.execute("DROP INDEX IF EXISTS taskhash_lookup_v2")
+            cursor.execute("DROP INDEX IF EXISTS outhash_lookup_v2")
+            cursor.execute("DROP INDEX IF EXISTS taskhash_lookup_v3")
+
+            # TODO: Upgrade from tasks_v2?
+            cursor.execute("DROP TABLE IF EXISTS tasks_v2")
+
+            # Create new indexes
+            cursor.execute(
+                "CREATE INDEX IF NOT EXISTS taskhash_lookup_v4 ON unihashes_v3 (method, taskhash)"
+            )
+            cursor.execute(
+                "CREATE INDEX IF NOT EXISTS unihash_lookup_v1 ON unihashes_v3 (unihash)"
+            )
+            cursor.execute(
+                "CREATE INDEX IF NOT EXISTS outhash_lookup_v3 ON outhashes_v2 (method, outhash)"
+            )
+            cursor.execute("CREATE INDEX IF NOT EXISTS config_lookup ON config (name)")
+
+            sqlite_version = _get_sqlite_version(cursor)
+
+            cursor.execute(
+                f"""
+                SELECT name FROM {_schema_table_name(sqlite_version)} WHERE type = 'table' AND name = 'unihashes_v2'
+                """
+            )
+            if cursor.fetchone():
+                self.logger.info("Upgrading Unihashes V2 -> V3...")
+                cursor.execute(
+                    """
+                    INSERT INTO unihashes_v3 (id, method, unihash, taskhash, gc_mark)
+                    SELECT id, method, unihash, taskhash, '' FROM unihashes_v2
+                    """
+                )
+                cursor.execute("DROP TABLE unihashes_v2")
+                db.commit()
+                self.logger.info("Upgrade complete")
+
+    def connect(self, logger):
+        return Database(logger, self.dbname, self.sync)
+
+
+class Database(object):
+    def __init__(self, logger, dbname, sync):
+        self.dbname = dbname
+        self.logger = logger
+
+        self.db = sqlite3.connect(self.dbname)
+        self.db.row_factory = sqlite3.Row
+
+        with closing(self.db.cursor()) as cursor:
+            cursor.execute("PRAGMA journal_mode = WAL")
+            cursor.execute(
+                "PRAGMA synchronous = %s" % ("NORMAL" if sync else "OFF")
+            )
+
+            self.sqlite_version = _get_sqlite_version(cursor)
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, exc_type, exc_value, traceback):
+        await self.close()
+
+    async def _set_config(self, cursor, name, value):
+        cursor.execute(
+            """
+            INSERT OR REPLACE INTO config (id, name, value) VALUES
+            ((SELECT id FROM config WHERE name=:name), :name, :value)
+            """,
+            {
+                "name": name,
+                "value": value,
+            },
+        )
+
+    async def _get_config(self, cursor, name):
+        cursor.execute(
+            "SELECT value FROM config WHERE name=:name",
+            {
+                "name": name,
+            },
+        )
+        row = cursor.fetchone()
+        if row is None:
+            return None
+        return row["value"]
+
+    async def close(self):
+        self.db.close()
+
+    async def get_unihash_by_taskhash_full(self, method, taskhash):
+        with closing(self.db.cursor()) as cursor:
+            cursor.execute(
+                """
+                SELECT *, unihashes_v3.unihash AS unihash FROM outhashes_v2
+                INNER JOIN unihashes_v3 ON unihashes_v3.method=outhashes_v2.method AND unihashes_v3.taskhash=outhashes_v2.taskhash
+                WHERE outhashes_v2.method=:method AND outhashes_v2.taskhash=:taskhash
+                ORDER BY outhashes_v2.created ASC
+                LIMIT 1
+                """,
+                {
+                    "method": method,
+                    "taskhash": taskhash,
+                },
+            )
+            return cursor.fetchone()
+
+    async def get_unihash_by_outhash(self, method, outhash):
+        with closing(self.db.cursor()) as cursor:
+            cursor.execute(
+                """
+                SELECT *, unihashes_v3.unihash AS unihash FROM outhashes_v2
+                INNER JOIN unihashes_v3 ON unihashes_v3.method=outhashes_v2.method AND unihashes_v3.taskhash=outhashes_v2.taskhash
+                WHERE outhashes_v2.method=:method AND outhashes_v2.outhash=:outhash
+                ORDER BY outhashes_v2.created ASC
+                LIMIT 1
+                """,
+                {
+                    "method": method,
+                    "outhash": outhash,
+                },
+            )
+            return cursor.fetchone()
+
+    async def unihash_exists(self, unihash):
+        with closing(self.db.cursor()) as cursor:
+            cursor.execute(
+                """
+                SELECT * FROM unihashes_v3 WHERE unihash=:unihash
+                LIMIT 1
+                """,
+                {
+                    "unihash": unihash,
+                },
+            )
+            return cursor.fetchone() is not None
+
+    async def get_outhash(self, method, outhash):
+        with closing(self.db.cursor()) as cursor:
+            cursor.execute(
+                """
+                SELECT * FROM outhashes_v2
+                WHERE outhashes_v2.method=:method AND outhashes_v2.outhash=:outhash
+                ORDER BY outhashes_v2.created ASC
+                LIMIT 1
+                """,
+                {
+                    "method": method,
+                    "outhash": outhash,
+                },
+            )
+            return cursor.fetchone()
+
+    async def get_equivalent_for_outhash(self, method, outhash, taskhash):
+        with closing(self.db.cursor()) as cursor:
+            cursor.execute(
+                """
+                SELECT outhashes_v2.taskhash AS taskhash, unihashes_v3.unihash AS unihash FROM outhashes_v2
+                INNER JOIN unihashes_v3 ON unihashes_v3.method=outhashes_v2.method AND unihashes_v3.taskhash=outhashes_v2.taskhash
+                -- Select any matching output hash except the one we just inserted
+                WHERE outhashes_v2.method=:method AND outhashes_v2.outhash=:outhash AND outhashes_v2.taskhash!=:taskhash
+                -- Pick the oldest hash
+                ORDER BY outhashes_v2.created ASC
+                LIMIT 1
+                """,
+                {
+                    "method": method,
+                    "outhash": outhash,
+                    "taskhash": taskhash,
+                },
+            )
+            return cursor.fetchone()
+
+    async def get_equivalent(self, method, taskhash):
+        with closing(self.db.cursor()) as cursor:
+            cursor.execute(
+                "SELECT taskhash, method, unihash FROM unihashes_v3 WHERE method=:method AND taskhash=:taskhash",
+                {
+                    "method": method,
+                    "taskhash": taskhash,
+                },
+            )
+            return cursor.fetchone()
+
+    async def remove(self, condition):
+        def do_remove(columns, table_name, cursor):
+            where, clause = _make_condition_statement(columns, condition)
+            if where:
+                query = f"DELETE FROM {table_name} WHERE {clause}"
+                cursor.execute(query, where)
+                return cursor.rowcount
+
+            return 0
+
+        count = 0
+        with closing(self.db.cursor()) as cursor:
+            count += do_remove(OUTHASH_TABLE_COLUMNS, "outhashes_v2", cursor)
+            count += do_remove(UNIHASH_TABLE_COLUMNS, "unihashes_v3", cursor)
+            self.db.commit()
+
+        return count
+
+    async def get_current_gc_mark(self):
+        with closing(self.db.cursor()) as cursor:
+            return await self._get_config(cursor, "gc-mark")
+
+    async def gc_status(self):
+        with closing(self.db.cursor()) as cursor:
+            cursor.execute(
+                """
+                SELECT COUNT() FROM unihashes_v3 WHERE
+                    gc_mark=COALESCE((SELECT value FROM config WHERE name='gc-mark'), '')
+                """
+            )
+            keep_rows = cursor.fetchone()[0]
+
+            cursor.execute(
+                """
+                SELECT COUNT() FROM unihashes_v3 WHERE
+                    gc_mark!=COALESCE((SELECT value FROM config WHERE name='gc-mark'), '')
+                """
+            )
+            remove_rows = cursor.fetchone()[0]
+
+            current_mark = await self._get_config(cursor, "gc-mark")
+
+            return (keep_rows, remove_rows, current_mark)
+
+    async def gc_mark(self, mark, condition):
+        with closing(self.db.cursor()) as cursor:
+            await self._set_config(cursor, "gc-mark", mark)
+
+            where, clause = _make_condition_statement(UNIHASH_TABLE_COLUMNS, condition)
+
+            new_rows = 0
+            if where:
+                cursor.execute(
+                    f"""
+                    UPDATE unihashes_v3 SET
+                        gc_mark=COALESCE((SELECT value FROM config WHERE name='gc-mark'), '')
+                    WHERE {clause}
+                    """,
+                    where,
+                )
+                new_rows = cursor.rowcount
+
+            self.db.commit()
+            return new_rows
+
+    async def gc_sweep(self):
+        with closing(self.db.cursor()) as cursor:
+            # NOTE: COALESCE is not used in this query so that if the current
+            # mark is NULL, nothing will happen
+            cursor.execute(
+                """
+                DELETE FROM unihashes_v3 WHERE
+                    gc_mark!=(SELECT value FROM config WHERE name='gc-mark')
+                """
+            )
+            count = cursor.rowcount
+            await self._set_config(cursor, "gc-mark", None)
+
+            self.db.commit()
+            return count
+
+    async def clean_unused(self, oldest):
+        with closing(self.db.cursor()) as cursor:
+            cursor.execute(
+                """
+                DELETE FROM outhashes_v2 WHERE created<:oldest AND NOT EXISTS (
+                    SELECT unihashes_v3.id FROM unihashes_v3 WHERE unihashes_v3.method=outhashes_v2.method AND unihashes_v3.taskhash=outhashes_v2.taskhash LIMIT 1
+                )
+                """,
+                {
+                    "oldest": oldest,
+                },
+            )
+            self.db.commit()
+            return cursor.rowcount
+
+    async def insert_unihash(self, method, taskhash, unihash):
+        with closing(self.db.cursor()) as cursor:
+            prevrowid = cursor.lastrowid
+            cursor.execute(
+                """
+                INSERT OR IGNORE INTO unihashes_v3 (method, taskhash, unihash, gc_mark) VALUES
+                    (
+                    :method,
+                    :taskhash,
+                    :unihash,
+                    COALESCE((SELECT value FROM config WHERE name='gc-mark'), '')
+                    )
+                """,
+                {
+                    "method": method,
+                    "taskhash": taskhash,
+                    "unihash": unihash,
+                },
+            )
+            self.db.commit()
+            return cursor.lastrowid != prevrowid
+
+    async def insert_outhash(self, data):
+        data = {k: v for k, v in data.items() if k in OUTHASH_TABLE_COLUMNS}
+        keys = sorted(data.keys())
+        query = "INSERT OR IGNORE INTO outhashes_v2 ({fields}) VALUES({values})".format(
+            fields=", ".join(keys),
+            values=", ".join(":" + k for k in keys),
+        )
+        with closing(self.db.cursor()) as cursor:
+            prevrowid = cursor.lastrowid
+            cursor.execute(query, data)
+            self.db.commit()
+            return cursor.lastrowid != prevrowid
+
+    def _get_user(self, username):
+        with closing(self.db.cursor()) as cursor:
+            cursor.execute(
+                """
+                SELECT username, permissions, token FROM users WHERE username=:username
+                """,
+                {
+                    "username": username,
+                },
+            )
+            return cursor.fetchone()
+
+    async def lookup_user_token(self, username):
+        row = self._get_user(username)
+        if row is None:
+            return None, None
+        return map_user(row), row["token"]
+
+    async def lookup_user(self, username):
+        return map_user(self._get_user(username))
+
+    async def set_user_token(self, username, token):
+        with closing(self.db.cursor()) as cursor:
+            cursor.execute(
+                """
+                UPDATE users SET token=:token WHERE username=:username
+                """,
+                {
+                    "username": username,
+                    "token": token,
+                },
+            )
+            self.db.commit()
+            return cursor.rowcount != 0
+
+    async def set_user_perms(self, username, permissions):
+        with closing(self.db.cursor()) as cursor:
+            cursor.execute(
+                """
+                UPDATE users SET permissions=:permissions WHERE username=:username
+                """,
+                {
+                    "username": username,
+                    "permissions": " ".join(permissions),
+                },
+            )
+            self.db.commit()
+            return cursor.rowcount != 0
+
+    async def get_all_users(self):
+        with closing(self.db.cursor()) as cursor:
+            cursor.execute("SELECT username, permissions FROM users")
+            return [map_user(r) for r in cursor.fetchall()]
+
+    async def new_user(self, username, permissions, token):
+        with closing(self.db.cursor()) as cursor:
+            try:
+                cursor.execute(
+                    """
+                    INSERT INTO users (username, token, permissions) VALUES (:username, :token, :permissions)
+                    """,
+                    {
+                        "username": username,
+                        "token": token,
+                        "permissions": " ".join(permissions),
+                    },
+                )
+                self.db.commit()
+                return True
+            except sqlite3.IntegrityError:
+                return False
+
+    async def delete_user(self, username):
+        with closing(self.db.cursor()) as cursor:
+            cursor.execute(
+                """
+                DELETE FROM users WHERE username=:username
+                """,
+                {
+                    "username": username,
+                },
+            )
+            self.db.commit()
+            return cursor.rowcount != 0
+
+    async def get_usage(self):
+        usage = {}
+        with closing(self.db.cursor()) as cursor:
+            cursor.execute(
+                f"""
+                SELECT name FROM {_schema_table_name(self.sqlite_version)} WHERE type = 'table' AND name NOT LIKE 'sqlite_%'
+                """
+            )
+            for row in cursor.fetchall():
+                cursor.execute(
+                    """
+                    SELECT COUNT() FROM %s
+                    """
+                    % row["name"],
+                )
+                usage[row["name"]] = {
+                    "rows": cursor.fetchone()[0],
+                }
+        return usage
+
+    async def get_query_columns(self):
+        columns = set()
+        for name, typ, _ in UNIHASH_TABLE_DEFINITION + OUTHASH_TABLE_DEFINITION:
+            if typ.startswith("TEXT"):
+                columns.add(name)
+        return list(columns)

bitbake/lib/layerindexlib/__init__.py

@@ -178,9 +178,9 @@ class LayerIndex():
         '''Load the layerindex.
 
            indexURI - An index to load.  (Use multiple calls to load multiple indexes)
-           
+
            reload - If reload is True, then any previously loaded indexes will be forgotten.
-           
+
            load - List of elements to load.  Default loads all items.
                   Note: plugs may ignore this.
 
@@ -383,7 +383,14 @@ layerBranches set.  If not, they are effectively blank.'''
 
                 # Get a list of dependencies and then recursively process them
                 for layerdependency in layerbranch.index.layerDependencies_layerBranchId[layerbranch.id]:
-                    deplayerbranch = layerdependency.dependency_layerBranch
+                    try:
+                        deplayerbranch = layerdependency.dependency_layerBranch
+                    except AttributeError as e:
+                            logger.error('LayerBranch does not exist for dependent layer {}:{}\n' \
+                                '       Cannot continue successfully.\n' \
+                                '       You might be able to resolve this by checking out the layer locally.\n' \
+                                '       Consider reaching out the to the layer maintainers or the layerindex admins' \
+                                .format(layerdependency.dependency.name, layerbranch.branch.name))
 
                     if ignores and deplayerbranch.layer.name in ignores:
                         continue
@@ -846,7 +853,7 @@ class LayerIndexObj():
                     continue
 
                 for layerdependency in layerbranch.index.layerDependencies_layerBranchId[layerbranch.id]:
-                    deplayerbranch = layerdependency.dependency_layerBranch
+                    deplayerbranch = layerdependency.dependency_layerBranch or None
 
                     if ignores and deplayerbranch.layer.name in ignores:
                         continue

bitbake/lib/progressbar/progressbar.py

@@ -253,7 +253,7 @@ class ProgressBar(object):
             if (self.maxval is not UnknownLength
                 and not 0 <= value <= self.maxval):
 
-                raise ValueError('Value out of range')
+                self.maxval = value
 
             self.currval = value
 

bitbake/lib/prserv/__init__.py

@@ -7,13 +7,13 @@
 __version__ = "1.0.0"
 
 import os, time
-import sys,logging
+import sys, logging
 
 def init_logger(logfile, loglevel):
     numeric_level = getattr(logging, loglevel.upper(), None)
     if not isinstance(numeric_level, int):
-        raise ValueError('Invalid log level: %s' % loglevel)
-    FORMAT = '%(asctime)-15s %(message)s'
+        raise ValueError("Invalid log level: %s" % loglevel)
+    FORMAT = "%(asctime)-15s %(message)s"
     logging.basicConfig(level=numeric_level, filename=logfile, format=FORMAT)
 
 class NotFoundError(Exception):

bitbake/lib/prserv/client.py

@@ -11,40 +11,61 @@ logger = logging.getLogger("BitBake.PRserv")
 
 class PRAsyncClient(bb.asyncrpc.AsyncClient):
     def __init__(self):
-        super().__init__('PRSERVICE', '1.0', logger)
+        super().__init__("PRSERVICE", "1.0", logger)
 
     async def getPR(self, version, pkgarch, checksum):
-        response = await self.send_message(
-            {'get-pr': {'version': version, 'pkgarch': pkgarch, 'checksum': checksum}}
+        response = await self.invoke(
+            {"get-pr": {"version": version, "pkgarch": pkgarch, "checksum": checksum}}
         )
         if response:
-            return response['value']
+            return response["value"]
+
+    async def test_pr(self, version, pkgarch, checksum):
+        response = await self.invoke(
+            {"test-pr": {"version": version, "pkgarch": pkgarch, "checksum": checksum}}
+        )
+        if response:
+            return response["value"]
+
+    async def test_package(self, version, pkgarch):
+        response = await self.invoke(
+            {"test-package": {"version": version, "pkgarch": pkgarch}}
+        )
+        if response:
+            return response["value"]
+
+    async def max_package_pr(self, version, pkgarch):
+        response = await self.invoke(
+            {"max-package-pr": {"version": version, "pkgarch": pkgarch}}
+        )
+        if response:
+            return response["value"]
 
     async def importone(self, version, pkgarch, checksum, value):
-        response = await self.send_message(
-            {'import-one': {'version': version, 'pkgarch': pkgarch, 'checksum': checksum, 'value': value}}
+        response = await self.invoke(
+            {"import-one": {"version": version, "pkgarch": pkgarch, "checksum": checksum, "value": value}}
         )
         if response:
-            return response['value']
+            return response["value"]
 
     async def export(self, version, pkgarch, checksum, colinfo):
-        response = await self.send_message(
-            {'export': {'version': version, 'pkgarch': pkgarch, 'checksum': checksum, 'colinfo': colinfo}}
+        response = await self.invoke(
+            {"export": {"version": version, "pkgarch": pkgarch, "checksum": checksum, "colinfo": colinfo}}
         )
         if response:
-            return (response['metainfo'], response['datainfo'])
+            return (response["metainfo"], response["datainfo"])
 
     async def is_readonly(self):
-        response = await self.send_message(
-            {'is-readonly': {}}
+        response = await self.invoke(
+            {"is-readonly": {}}
         )
         if response:
-            return response['readonly']
+            return response["readonly"]
 
 class PRClient(bb.asyncrpc.Client):
     def __init__(self):
         super().__init__()
-        self._add_methods('getPR', 'importone', 'export', 'is_readonly')
+        self._add_methods("getPR", "test_pr", "test_package", "importone", "export", "is_readonly")
 
     def _get_async_client(self):
         return PRAsyncClient()

bitbake/lib/prserv/db.py

@@ -38,9 +38,9 @@ class PRTable(object):
         self.read_only = read_only
         self.dirty = False
         if nohist:
-            self.table = "%s_nohist" % table 
+            self.table = "%s_nohist" % table
         else:
-            self.table = "%s_hist" % table 
+            self.table = "%s_hist" % table
 
         if self.read_only:
             table_exists = self._execute(
@@ -64,7 +64,7 @@ class PRTable(object):
             try:
                 return self.conn.execute(*query)
             except sqlite3.OperationalError as exc:
-                if 'is locked' in str(exc) and end > time.time():
+                if "is locked" in str(exc) and end > time.time():
                     continue
                 raise exc
 
@@ -78,7 +78,53 @@ class PRTable(object):
             self.sync()
             self.dirty = False
 
-    def _getValueHist(self, version, pkgarch, checksum):
+    def test_package(self, version, pkgarch):
+        """Returns whether the specified package version is found in the database for the specified architecture"""
+
+        # Just returns the value if found or None otherwise
+        data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=?;" % self.table,
+                           (version, pkgarch))
+        row=data.fetchone()
+        if row is not None:
+            return True
+        else:
+            return False
+
+    def test_value(self, version, pkgarch, value):
+        """Returns whether the specified value is found in the database for the specified package and architecture"""
+
+        # Just returns the value if found or None otherwise
+        data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? and value=?;" % self.table,
+                           (version, pkgarch, value))
+        row=data.fetchone()
+        if row is not None:
+            return True
+        else:
+            return False
+
+    def find_value(self, version, pkgarch, checksum):
+        """Returns the value for the specified checksum if found or None otherwise."""
+
+        data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
+                           (version, pkgarch, checksum))
+        row=data.fetchone()
+        if row is not None:
+            return row[0]
+        else:
+            return None
+
+    def find_max_value(self, version, pkgarch):
+        """Returns the greatest value for (version, pkgarch), or None if not found. Doesn't create a new value"""
+
+        data = self._execute("SELECT max(value) FROM %s where version=? AND pkgarch=?;" % (self.table),
+                             (version, pkgarch))
+        row = data.fetchone()
+        if row is not None:
+            return row[0]
+        else:
+            return None
+
+    def _get_value_hist(self, version, pkgarch, checksum):
         data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
                            (version, pkgarch, checksum))
         row=data.fetchone()
@@ -87,7 +133,7 @@ class PRTable(object):
         else:
             #no value found, try to insert
             if self.read_only:
-                data = self._execute("SELECT ifnull(max(value)+1,0) FROM %s where version=? AND pkgarch=?;" % (self.table),
+                data = self._execute("SELECT ifnull(max(value)+1, 0) FROM %s where version=? AND pkgarch=?;" % (self.table),
                                    (version, pkgarch))
                 row = data.fetchone()
                 if row is not None:
@@ -96,9 +142,9 @@ class PRTable(object):
                     return 0
 
             try:
-                self._execute("INSERT INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1,0) from %s where version=? AND pkgarch=?));"
-                           % (self.table,self.table),
-                           (version,pkgarch, checksum,version, pkgarch))
+                self._execute("INSERT INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1, 0) from %s where version=? AND pkgarch=?));"
+                           % (self.table, self.table),
+                           (version, pkgarch, checksum, version, pkgarch))
             except sqlite3.IntegrityError as exc:
                 logger.error(str(exc))
 
@@ -112,10 +158,10 @@ class PRTable(object):
             else:
                 raise prserv.NotFoundError
 
-    def _getValueNohist(self, version, pkgarch, checksum):
+    def _get_value_no_hist(self, version, pkgarch, checksum):
         data=self._execute("SELECT value FROM %s \
                             WHERE version=? AND pkgarch=? AND checksum=? AND \
-                            value >= (select max(value) from %s where version=? AND pkgarch=?);" 
+                            value >= (select max(value) from %s where version=? AND pkgarch=?);"
                             % (self.table, self.table),
                             (version, pkgarch, checksum, version, pkgarch))
         row=data.fetchone()
@@ -124,17 +170,13 @@ class PRTable(object):
         else:
             #no value found, try to insert
             if self.read_only:
-                data = self._execute("SELECT ifnull(max(value)+1,0) FROM %s where version=? AND pkgarch=?;" % (self.table),
+                data = self._execute("SELECT ifnull(max(value)+1, 0) FROM %s where version=? AND pkgarch=?;" % (self.table),
                                    (version, pkgarch))
-                row = data.fetchone()
-                if row is not None:
-                    return row[0]
-                else:
-                    return 0
+                return data.fetchone()[0]
 
             try:
-                self._execute("INSERT OR REPLACE INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1,0) from %s where version=? AND pkgarch=?));"
-                               % (self.table,self.table),
+                self._execute("INSERT OR REPLACE INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1, 0) from %s where version=? AND pkgarch=?));"
+                               % (self.table, self.table),
                                (version, pkgarch, checksum, version, pkgarch))
             except sqlite3.IntegrityError as exc:
                 logger.error(str(exc))
@@ -150,17 +192,17 @@ class PRTable(object):
             else:
                 raise prserv.NotFoundError
 
-    def getValue(self, version, pkgarch, checksum):
+    def get_value(self, version, pkgarch, checksum):
         if self.nohist:
-            return self._getValueNohist(version, pkgarch, checksum)
+            return self._get_value_no_hist(version, pkgarch, checksum)
         else:
-            return self._getValueHist(version, pkgarch, checksum)
+            return self._get_value_hist(version, pkgarch, checksum)
 
-    def _importHist(self, version, pkgarch, checksum, value):
+    def _import_hist(self, version, pkgarch, checksum, value):
         if self.read_only:
             return None
 
-        val = None 
+        val = None
         data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
                            (version, pkgarch, checksum))
         row = data.fetchone()
@@ -183,27 +225,27 @@ class PRTable(object):
                 val = row[0]
         return val
 
-    def _importNohist(self, version, pkgarch, checksum, value):
+    def _import_no_hist(self, version, pkgarch, checksum, value):
         if self.read_only:
             return None
 
         try:
             #try to insert
             self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);"  % (self.table),
-                           (version, pkgarch, checksum,value))
+                           (version, pkgarch, checksum, value))
         except sqlite3.IntegrityError as exc:
             #already have the record, try to update
             try:
-                self._execute("UPDATE %s SET value=? WHERE version=? AND pkgarch=? AND checksum=? AND value<?"  
+                self._execute("UPDATE %s SET value=? WHERE version=? AND pkgarch=? AND checksum=? AND value<?"
                               % (self.table),
-                               (value,version,pkgarch,checksum,value))
+                               (value, version, pkgarch, checksum, value))
             except sqlite3.IntegrityError as exc:
                 logger.error(str(exc))
 
         self.dirty = True
 
         data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=? AND value>=?;" % self.table,
-                            (version,pkgarch,checksum,value))
+                            (version, pkgarch, checksum, value))
         row=data.fetchone()
         if row is not None:
             return row[0]
@@ -212,33 +254,33 @@ class PRTable(object):
 
     def importone(self, version, pkgarch, checksum, value):
         if self.nohist:
-            return self._importNohist(version, pkgarch, checksum, value)
+            return self._import_no_hist(version, pkgarch, checksum, value)
         else:
-            return self._importHist(version, pkgarch, checksum, value)
+            return self._import_hist(version, pkgarch, checksum, value)
 
     def export(self, version, pkgarch, checksum, colinfo):
         metainfo = {}
-        #column info 
+        #column info
         if colinfo:
-            metainfo['tbl_name'] = self.table
-            metainfo['core_ver'] = prserv.__version__
-            metainfo['col_info'] = []
+            metainfo["tbl_name"] = self.table
+            metainfo["core_ver"] = prserv.__version__
+            metainfo["col_info"] = []
             data = self._execute("PRAGMA table_info(%s);" % self.table)
             for row in data:
                 col = {}
-                col['name'] = row['name']
-                col['type'] = row['type']
-                col['notnull'] = row['notnull']
-                col['dflt_value'] = row['dflt_value']
-                col['pk'] = row['pk']
-                metainfo['col_info'].append(col)
+                col["name"] = row["name"]
+                col["type"] = row["type"]
+                col["notnull"] = row["notnull"]
+                col["dflt_value"] = row["dflt_value"]
+                col["pk"] = row["pk"]
+                metainfo["col_info"].append(col)
 
         #data info
         datainfo = []
 
         if self.nohist:
             sqlstmt = "SELECT T1.version, T1.pkgarch, T1.checksum, T1.value FROM %s as T1, \
-                    (SELECT version,pkgarch,max(value) as maxvalue FROM %s GROUP BY version,pkgarch) as T2 \
+                    (SELECT version, pkgarch, max(value) as maxvalue FROM %s GROUP BY version, pkgarch) as T2 \
                     WHERE T1.version=T2.version AND T1.pkgarch=T2.pkgarch AND T1.value=T2.maxvalue " % (self.table, self.table)
         else:
             sqlstmt = "SELECT * FROM %s as T1 WHERE 1=1 " % self.table
@@ -261,12 +303,12 @@ class PRTable(object):
         else:
             data = self._execute(sqlstmt)
         for row in data:
-            if row['version']:
+            if row["version"]:
                 col = {}
-                col['version'] = row['version']
-                col['pkgarch'] = row['pkgarch']
-                col['checksum'] = row['checksum']
-                col['value'] = row['value']
+                col["version"] = row["version"]
+                col["pkgarch"] = row["pkgarch"]
+                col["checksum"] = row["checksum"]
+                col["value"] = row["value"]
                 datainfo.append(col)
         return (metainfo, datainfo)
 
@@ -275,7 +317,7 @@ class PRTable(object):
         for line in self.conn.iterdump():
             writeCount = writeCount + len(line) + 1
             fd.write(line)
-            fd.write('\n')
+            fd.write("\n")
         return writeCount
 
 class PRData(object):
@@ -302,7 +344,7 @@ class PRData(object):
     def disconnect(self):
         self.connection.close()
 
-    def __getitem__(self,tblname):
+    def __getitem__(self, tblname):
         if not isinstance(tblname, str):
             raise TypeError("tblname argument must be a string, not '%s'" %
                             type(tblname))
@@ -316,4 +358,4 @@ class PRData(object):
         if tblname in self._tables:
             del self._tables[tblname]
         logger.info("drop table %s" % (tblname))
-        self.connection.execute("DROP TABLE IF EXISTS %s;" % tblname) 
+        self.connection.execute("DROP TABLE IF EXISTS %s;" % tblname)

bitbake/lib/prserv/serv.py

@@ -20,77 +20,101 @@ PIDPREFIX = "/tmp/PRServer_%s_%s.pid"
 singleton = None
 
 class PRServerClient(bb.asyncrpc.AsyncServerConnection):
-    def __init__(self, reader, writer, table, read_only):
-        super().__init__(reader, writer, 'PRSERVICE', logger)
+    def __init__(self, socket, server):
+        super().__init__(socket, "PRSERVICE", server.logger)
+        self.server = server
+
         self.handlers.update({
-            'get-pr': self.handle_get_pr,
-            'import-one': self.handle_import_one,
-            'export': self.handle_export,
-            'is-readonly': self.handle_is_readonly,
+            "get-pr": self.handle_get_pr,
+            "test-pr": self.handle_test_pr,
+            "test-package": self.handle_test_package,
+            "max-package-pr": self.handle_max_package_pr,
+            "import-one": self.handle_import_one,
+            "export": self.handle_export,
+            "is-readonly": self.handle_is_readonly,
         })
-        self.table = table
-        self.read_only = read_only
 
     def validate_proto_version(self):
         return (self.proto_version == (1, 0))
 
     async def dispatch_message(self, msg):
         try:
-            await super().dispatch_message(msg)
+            return await super().dispatch_message(msg)
         except:
-            self.table.sync()
+            self.server.table.sync()
             raise
+        else:
+            self.server.table.sync_if_dirty()
 
-        self.table.sync_if_dirty()
+    async def handle_test_pr(self, request):
+        '''Finds the PR value corresponding to the request. If not found, returns None and doesn't insert a new value'''
+        version = request["version"]
+        pkgarch = request["pkgarch"]
+        checksum = request["checksum"]
+
+        value = self.server.table.find_value(version, pkgarch, checksum)
+        return {"value": value}
+
+    async def handle_test_package(self, request):
+        '''Tells whether there are entries for (version, pkgarch) in the db. Returns True or False'''
+        version = request["version"]
+        pkgarch = request["pkgarch"]
+
+        value = self.server.table.test_package(version, pkgarch)
+        return {"value": value}
+
+    async def handle_max_package_pr(self, request):
+        '''Finds the greatest PR value for (version, pkgarch) in the db. Returns None if no entry was found'''
+        version = request["version"]
+        pkgarch = request["pkgarch"]
+
+        value = self.server.table.find_max_value(version, pkgarch)
+        return {"value": value}
 
     async def handle_get_pr(self, request):
-        version = request['version']
-        pkgarch = request['pkgarch']
-        checksum = request['checksum']
+        version = request["version"]
+        pkgarch = request["pkgarch"]
+        checksum = request["checksum"]
 
         response = None
         try:
-            value = self.table.getValue(version, pkgarch, checksum)
-            response = {'value': value}
+            value = self.server.table.get_value(version, pkgarch, checksum)
+            response = {"value": value}
         except prserv.NotFoundError:
-            logger.error("can not find value for (%s, %s)",version, checksum)
-        except sqlite3.Error as exc:
-            logger.error(str(exc))
+            self.logger.error("failure storing value in database for (%s, %s)",version, checksum)
 
-        self.write_message(response)
+        return response
 
     async def handle_import_one(self, request):
         response = None
-        if not self.read_only:
-            version = request['version']
-            pkgarch = request['pkgarch']
-            checksum = request['checksum']
-            value = request['value']
+        if not self.server.read_only:
+            version = request["version"]
+            pkgarch = request["pkgarch"]
+            checksum = request["checksum"]
+            value = request["value"]
 
-            value = self.table.importone(version, pkgarch, checksum, value)
+            value = self.server.table.importone(version, pkgarch, checksum, value)
             if value is not None:
-                response = {'value': value}
+                response = {"value": value}
 
-        self.write_message(response)
+        return response
 
     async def handle_export(self, request):
-        version = request['version']
-        pkgarch = request['pkgarch']
-        checksum = request['checksum']
-        colinfo = request['colinfo']
+        version = request["version"]
+        pkgarch = request["pkgarch"]
+        checksum = request["checksum"]
+        colinfo = request["colinfo"]
 
         try:
-            (metainfo, datainfo) = self.table.export(version, pkgarch, checksum, colinfo)
+            (metainfo, datainfo) = self.server.table.export(version, pkgarch, checksum, colinfo)
         except sqlite3.Error as exc:
-            logger.error(str(exc))
+            self.logger.error(str(exc))
             metainfo = datainfo = None
 
-        response = {'metainfo': metainfo, 'datainfo': datainfo}
-        self.write_message(response)
+        return {"metainfo": metainfo, "datainfo": datainfo}
 
     async def handle_is_readonly(self, request):
-        response = {'readonly': self.read_only}
-        self.write_message(response)
+        return {"readonly": self.server.read_only}
 
 class PRServer(bb.asyncrpc.AsyncServer):
     def __init__(self, dbfile, read_only=False):
@@ -99,20 +123,23 @@ class PRServer(bb.asyncrpc.AsyncServer):
         self.table = None
         self.read_only = read_only
 
-    def accept_client(self, reader, writer):
-        return PRServerClient(reader, writer, self.table, self.read_only)
+    def accept_client(self, socket):
+        return PRServerClient(socket, self)
 
-    def _serve_forever(self):
+    def start(self):
+        tasks = super().start()
         self.db = prserv.db.PRData(self.dbfile, read_only=self.read_only)
         self.table = self.db["PRMAIN"]
 
-        logger.info("Started PRServer with DBfile: %s, Address: %s, PID: %s" %
+        self.logger.info("Started PRServer with DBfile: %s, Address: %s, PID: %s" %
                      (self.dbfile, self.address, str(os.getpid())))
 
-        super()._serve_forever()
+        return tasks
 
+    async def stop(self):
         self.table.sync_if_dirty()
         self.db.disconnect()
+        await super().stop()
 
     def signal_handler(self):
         super().signal_handler()
@@ -129,12 +156,12 @@ class PRServSingleton(object):
     def start(self):
         self.prserv = PRServer(self.dbfile)
         self.prserv.start_tcp_server(socket.gethostbyname(self.host), self.port)
-        self.process = self.prserv.serve_as_process()
+        self.process = self.prserv.serve_as_process(log_level=logging.WARNING)
 
         if not self.prserv.address:
             raise PRServiceConfigError
         if not self.port:
-            self.port = int(self.prserv.address.rsplit(':', 1)[1])
+            self.port = int(self.prserv.address.rsplit(":", 1)[1])
 
 def run_as_daemon(func, pidfile, logfile):
     """
@@ -170,18 +197,18 @@ def run_as_daemon(func, pidfile, logfile):
     # stdout/stderr or it could be 'real' unix fd forking where we need
     # to physically close the fds to prevent the program launching us from
     # potentially hanging on a pipe. Handle both cases.
-    si = open('/dev/null', 'r')
+    si = open("/dev/null", "r")
     try:
-        os.dup2(si.fileno(),sys.stdin.fileno())
+        os.dup2(si.fileno(), sys.stdin.fileno())
     except (AttributeError, io.UnsupportedOperation):
         sys.stdin = si
-    so = open(logfile, 'a+')
+    so = open(logfile, "a+")
     try:
-        os.dup2(so.fileno(),sys.stdout.fileno())
+        os.dup2(so.fileno(), sys.stdout.fileno())
     except (AttributeError, io.UnsupportedOperation):
         sys.stdout = so
     try:
-        os.dup2(so.fileno(),sys.stderr.fileno())
+        os.dup2(so.fileno(), sys.stderr.fileno())
     except (AttributeError, io.UnsupportedOperation):
         sys.stderr = so
 
@@ -199,7 +226,7 @@ def run_as_daemon(func, pidfile, logfile):
 
     # write pidfile
     pid = str(os.getpid())
-    with open(pidfile, 'w') as pf:
+    with open(pidfile, "w") as pf:
         pf.write("%s\n" % pid)
 
     func()
@@ -244,15 +271,15 @@ def stop_daemon(host, port):
         # so at least advise the user which ports the corresponding server is listening
         ports = []
         portstr = ""
-        for pf in glob.glob(PIDPREFIX % (ip,'*')):
+        for pf in glob.glob(PIDPREFIX % (ip, "*")):
             bn = os.path.basename(pf)
             root, _ = os.path.splitext(bn)
-            ports.append(root.split('_')[-1])
+            ports.append(root.split("_")[-1])
         if len(ports):
-            portstr = "Wrong port? Other ports listening at %s: %s" % (host, ' '.join(ports))
+            portstr = "Wrong port? Other ports listening at %s: %s" % (host, " ".join(ports))
 
         sys.stderr.write("pidfile %s does not exist. Daemon not running? %s\n"
-                         % (pidfile,portstr))
+                         % (pidfile, portstr))
         return 1
 
     try:
@@ -261,8 +288,11 @@ def stop_daemon(host, port):
             os.kill(pid, signal.SIGTERM)
             time.sleep(0.1)
 
-        if os.path.exists(pidfile):
+        try:
             os.remove(pidfile)
+        except FileNotFoundError:
+            # The PID file might have been removed by the exiting process
+            pass
 
     except OSError as e:
         err = str(e)
@@ -280,7 +310,7 @@ def is_running(pid):
     return True
 
 def is_local_special(host, port):
-    if (host == 'localhost' or host == '127.0.0.1') and not port:
+    if (host == "localhost" or host == "127.0.0.1") and not port:
         return True
     else:
         return False
@@ -291,7 +321,7 @@ class PRServiceConfigError(Exception):
 def auto_start(d):
     global singleton
 
-    host_params = list(filter(None, (d.getVar('PRSERV_HOST') or '').split(':')))
+    host_params = list(filter(None, (d.getVar("PRSERV_HOST") or "").split(":")))
     if not host_params:
         # Shutdown any existing PR Server
         auto_shutdown()
@@ -300,7 +330,7 @@ def auto_start(d):
     if len(host_params) != 2:
         # Shutdown any existing PR Server
         auto_shutdown()
-        logger.critical('\n'.join(['PRSERV_HOST: incorrect format',
+        logger.critical("\n".join(["PRSERV_HOST: incorrect format",
                 'Usage: PRSERV_HOST = "<hostname>:<port>"']))
         raise PRServiceConfigError
 
@@ -344,17 +374,17 @@ def auto_shutdown():
 def ping(host, port):
     from . import client
 
-    conn = client.PRClient()
-    conn.connect_tcp(host, port)
-    return conn.ping()
+    with client.PRClient() as conn:
+        conn.connect_tcp(host, port)
+        return conn.ping()
 
 def connect(host, port):
     from . import client
 
     global singleton
 
-    if host.strip().lower() == 'localhost' and not port:
-        host = 'localhost'
+    if host.strip().lower() == "localhost" and not port:
+        host = "localhost"
         port = singleton.port
 
     conn = client.PRClient()

bitbake/lib/toaster/orm/fixtures/settings.xml

@@ -12,7 +12,7 @@
   </object>
   <object model="orm.toastersetting" pk="4">
     <field type="CharField" name="name">DEFCONF_MACHINE</field>
-    <field type="CharField" name="value">qemux86</field>
+    <field type="CharField" name="value">qemux86-64</field>
   </object>
   <object model="orm.toastersetting" pk="5">
     <field type="CharField" name="name">DEFCONF_SSTATE_DIR</field>

bitbake/lib/toaster/orm/migrations/0021_eventlogsimports.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.2.5 on 2023-11-23 18:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('orm', '0020_models_bigautofield'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='EventLogsImports',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=255)),
+                ('imported', models.BooleanField(default=False)),
+                ('build_id', models.IntegerField(blank=True, null=True)),
+            ],
+        ),
+    ]

bitbake/lib/toaster/orm/models.py

@@ -1389,9 +1389,6 @@ class Machine(models.Model):
         return "Machine " + self.name + "(" + self.description + ")"
 
 
-
-
-
 class BitbakeVersion(models.Model):
 
     name = models.CharField(max_length=32, unique = True)
@@ -1853,6 +1850,8 @@ def signal_runbuilds():
             os.kill(int(pidf.read()), SIGUSR1)
     except FileNotFoundError:
         logger.info("Stopping existing runbuilds: no current process found")
+    except ProcessLookupError:
+        logger.warning("Stopping existing runbuilds: process lookup not found")
 
 class Distro(models.Model):
     search_allowed_fields = ["name", "description", "layer_version__layer__name"]
@@ -1869,6 +1868,15 @@ class Distro(models.Model):
     def __unicode__(self):
         return "Distro " + self.name + "(" + self.description + ")"
 
+class EventLogsImports(models.Model):
+    name = models.CharField(max_length=255)
+    imported = models.BooleanField(default=False)
+    build_id = models.IntegerField(blank=True, null=True)
+
+    def __str__(self):
+        return self.name
+
+
 django.db.models.signals.post_save.connect(invalidate_cache)
 django.db.models.signals.post_delete.connect(invalidate_cache)
 django.db.models.signals.m2m_changed.connect(invalidate_cache)

bitbake/lib/toaster/pytest.ini

@@ -0,0 +1,16 @@
+# -- FILE: pytest.ini (or tox.ini)
+[pytest]
+# --create-db - force re creation of the test database
+# https://pytest-django.readthedocs.io/en/latest/database.html#create-db-force-re-creation-of-the-test-database
+
+# --html=report.html --self-contained-html
+# https://docs.pytest.org/en/latest/usage.html#creating-html-reports
+# https://pytest-html.readthedocs.io/en/latest/user_guide.html#creating-a-self-contained-report
+addopts = --create-db --html="Toaster Tests Report.html" --self-contained-html 
+
+# Define environment variables using pytest-env
+# A pytest plugin that enables you to set environment variables in the pytest.ini file.
+# https://pypi.org/project/pytest-env/
+env =
+    TOASTER_BUILDSERVER=1
+    DJANGO_SETTINGS_MODULE=toastermain.settings_test

bitbake/lib/toaster/tests/browser/selenium_helpers_base.py

@@ -19,12 +19,15 @@ import os
 import time
 import unittest
 
+import pytest
 from selenium import webdriver
+from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
 from selenium.common.exceptions import NoSuchElementException, \
-        StaleElementReferenceException, TimeoutException
+        StaleElementReferenceException, TimeoutException, \
+        SessionNotCreatedException
 
 def create_selenium_driver(cls,browser='chrome'):
     # set default browser string based on env (if available)
@@ -33,7 +36,32 @@ def create_selenium_driver(cls,browser='chrome'):
         browser = env_browser
 
     if browser == 'chrome':
-        return webdriver.Chrome()
+        options = webdriver.ChromeOptions()
+        options.add_argument('--headless')
+        options.add_argument('--disable-infobars')
+        options.add_argument('--disable-dev-shm-usage')
+        options.add_argument('--no-sandbox')
+        options.add_argument('--remote-debugging-port=9222')
+        try:
+            return webdriver.Chrome(options=options)
+        except SessionNotCreatedException as e:
+            exit_message = "Halting tests prematurely to avoid cascading errors."
+            # check if chrome / chromedriver exists
+            chrome_path = os.popen("find ~/.cache/selenium/chrome/ -name 'chrome' -type f -print -quit").read().strip()
+            if not chrome_path:
+                pytest.exit(f"Failed to install/find chrome.\n{exit_message}")
+            chromedriver_path = os.popen("find ~/.cache/selenium/chromedriver/ -name 'chromedriver' -type f -print -quit").read().strip()
+            if not chromedriver_path:
+                pytest.exit(f"Failed to install/find chromedriver.\n{exit_message}")
+            # check if depends on each are fulfilled
+            depends_chrome = os.popen(f"ldd {chrome_path} | grep 'not found'").read().strip()
+            if depends_chrome:
+                pytest.exit(f"Missing chrome dependencies.\n{depends_chrome}\n{exit_message}")
+            depends_chromedriver = os.popen(f"ldd {chromedriver_path} | grep 'not found'").read().strip()
+            if depends_chromedriver:
+                pytest.exit(f"Missing chromedriver dependencies.\n{depends_chromedriver}\n{exit_message}")
+            # print original error otherwise
+            pytest.exit(f"Failed to start chromedriver.\n{e}\n{exit_message}")
     elif browser == 'firefox':
         return webdriver.Firefox()
     elif browser == 'marionette':
@@ -65,7 +93,9 @@ class Wait(WebDriverWait):
     _TIMEOUT = 10
     _POLL_FREQUENCY = 0.5
 
-    def __init__(self, driver):
+    def __init__(self, driver, timeout=_TIMEOUT, poll=_POLL_FREQUENCY):
+        self._TIMEOUT = timeout
+        self._POLL_FREQUENCY = poll
         super(Wait, self).__init__(driver, self._TIMEOUT, self._POLL_FREQUENCY)
 
     def until(self, method, message=''):
@@ -137,6 +167,8 @@ class SeleniumTestCaseBase(unittest.TestCase):
         """ Clean up webdriver driver """
 
         cls.driver.quit()
+        # Allow driver resources to be properly freed before proceeding with further tests
+        time.sleep(5)
         super(SeleniumTestCaseBase, cls).tearDownClass()
 
     def get(self, url):
@@ -150,6 +182,13 @@ class SeleniumTestCaseBase(unittest.TestCase):
         abs_url = '%s%s' % (self.live_server_url, url)
         self.driver.get(abs_url)
 
+        try:  # Ensure page is loaded before proceeding
+            self.wait_until_visible("#global-nav", poll=3)
+        except NoSuchElementException:
+            self.driver.implicitly_wait(3)
+        except TimeoutException:
+            self.driver.implicitly_wait(3)
+
     def find(self, selector):
         """ Find single element by CSS selector """
         return self.driver.find_element(By.CSS_SELECTOR, selector)
@@ -169,18 +208,34 @@ class SeleniumTestCaseBase(unittest.TestCase):
         """ Return the element which currently has focus on the page """
         return self.driver.switch_to.active_element
 
-    def wait_until_present(self, selector):
+    def wait_until_present(self, selector, poll=0.5):
         """ Wait until element matching CSS selector is on the page """
         is_present = lambda driver: self.find(selector)
         msg = 'An element matching "%s" should be on the page' % selector
-        element = Wait(self.driver).until(is_present, msg)
+        element = Wait(self.driver, poll=poll).until(is_present, msg)
+        if poll > 2:
+            time.sleep(poll)  # element need more delay to be present
         return element
 
-    def wait_until_visible(self, selector):
+    def wait_until_visible(self, selector, poll=1):
         """ Wait until element matching CSS selector is visible on the page """
         is_visible = lambda driver: self.find(selector).is_displayed()
         msg = 'An element matching "%s" should be visible' % selector
-        Wait(self.driver).until(is_visible, msg)
+        Wait(self.driver, poll=poll).until(is_visible, msg)
+        time.sleep(poll)  # wait for visibility to settle
+        return self.find(selector)
+
+    def wait_until_clickable(self, selector, poll=1):
+        """ Wait until element matching CSS selector is visible on the page """
+        WebDriverWait(
+            self.driver,
+            Wait._TIMEOUT,
+            poll_frequency=poll
+        ).until(
+            EC.element_to_be_clickable((By.ID, selector.removeprefix('#')
+                                        )
+                                       )
+        )
         return self.find(selector)
 
     def wait_until_focused(self, selector):

bitbake/lib/toaster/tests/browser/test_all_builds_page.py

@@ -7,13 +7,16 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-import re, time
+import os
+import re
 
 from django.urls import reverse
+from selenium.webdriver.support.select import Select
 from django.utils import timezone
+from bldcontrol.models import BuildRequest
 from tests.browser.selenium_helpers import SeleniumTestCase
 
-from orm.models import BitbakeVersion, Release, Project, Build, Target
+from orm.models import BitbakeVersion, Layer, Layer_Version, Recipe, Release, Project, Build, Target, Task
 
 from selenium.webdriver.common.by import By
 
@@ -25,7 +28,8 @@ class TestAllBuildsPage(SeleniumTestCase):
     CLI_BUILDS_PROJECT_NAME = 'command line builds'
 
     def setUp(self):
-        bbv = BitbakeVersion.objects.create(name='bbv1', giturl='/tmp/',
+        builldir = os.environ.get('BUILDDIR', './')
+        bbv = BitbakeVersion.objects.create(name='bbv1', giturl=f'{builldir}/',
                                             branch='master', dirpath='')
         release = Release.objects.create(name='release1',
                                          bitbake_version=bbv)
@@ -71,7 +75,7 @@ class TestAllBuildsPage(SeleniumTestCase):
             '[data-role="data-recent-build-buildtime-field"]' % build.id
 
         # because this loads via Ajax, wait for it to be visible
-        self.wait_until_present(selector)
+        self.wait_until_visible(selector)
 
         build_time_spans = self.find_all(selector)
 
@@ -81,7 +85,7 @@ class TestAllBuildsPage(SeleniumTestCase):
 
     def _get_row_for_build(self, build):
         """ Get the table row for the build from the all builds table """
-        self.wait_until_present('#allbuildstable')
+        self.wait_until_visible('#allbuildstable')
 
         rows = self.find_all('#allbuildstable tr')
 
@@ -102,6 +106,66 @@ class TestAllBuildsPage(SeleniumTestCase):
 
         return found_row
 
+    def _get_create_builds(self, **kwargs):
+        """ Create a build and return the build object """
+        build1 = Build.objects.create(**self.project1_build_success)
+        build2 = Build.objects.create(**self.project1_build_failure)
+
+        # add some targets to these builds so they have recipe links
+        # (and so we can find the row in the ToasterTable corresponding to
+        # a particular build)
+        Target.objects.create(build=build1, target='foo')
+        Target.objects.create(build=build2, target='bar')
+
+        if kwargs:
+            # Create kwargs.get('success') builds with success status with target
+            # and kwargs.get('failure') builds with failure status with target
+            for i in range(kwargs.get('success', 0)):
+                now = timezone.now()
+                self.project1_build_success['started_on'] = now
+                self.project1_build_success[
+                    'completed_on'] = now - timezone.timedelta(days=i)
+                build = Build.objects.create(**self.project1_build_success)
+                Target.objects.create(build=build,
+                                      target=f'{i}_success_recipe',
+                                      task=f'{i}_success_task')
+
+                self._set_buildRequest_and_task_on_build(build)
+            for i in range(kwargs.get('failure', 0)):
+                now = timezone.now()
+                self.project1_build_failure['started_on'] = now
+                self.project1_build_failure[
+                    'completed_on'] = now - timezone.timedelta(days=i)
+                build = Build.objects.create(**self.project1_build_failure)
+                Target.objects.create(build=build,
+                                      target=f'{i}_fail_recipe',
+                                      task=f'{i}_fail_task')
+                self._set_buildRequest_and_task_on_build(build)
+        return build1, build2
+
+    def _create_recipe(self):
+        """ Add a recipe to the database and return it """
+        layer = Layer.objects.create()
+        layer_version = Layer_Version.objects.create(layer=layer)
+        return Recipe.objects.create(name='recipe_foo', layer_version=layer_version)
+
+    def _set_buildRequest_and_task_on_build(self, build):
+        """ Set buildRequest and task on build """
+        build.recipes_parsed = 1
+        build.save()
+        buildRequest = BuildRequest.objects.create(
+            build=build,
+            project=self.project1,
+            state=BuildRequest.REQ_COMPLETED)
+        build.build_request = buildRequest
+        recipe = self._create_recipe()
+        task = Task.objects.create(build=build,
+                                   recipe=recipe,
+                                   task_name='task',
+                                   outcome=Task.OUTCOME_SUCCESS)
+        task.save()
+        build.save()
+
     def test_show_tasks_with_suffix(self):
         """ Task should be shown as suffix on build name """
         build = Build.objects.create(**self.project1_build_success)
@@ -111,7 +175,7 @@ class TestAllBuildsPage(SeleniumTestCase):
 
         url = reverse('all-builds')
         self.get(url)
-        self.wait_until_present('td[class="target"]')
+        self.wait_until_visible('td[class="target"]')
 
         cell = self.find('td[class="target"]')
         content = cell.get_attribute('innerHTML')
@@ -128,14 +192,15 @@ class TestAllBuildsPage(SeleniumTestCase):
         but should be shown for other builds
         """
         build1 = Build.objects.create(**self.project1_build_success)
-        default_build = Build.objects.create(**self.default_project_build_success)
+        default_build = Build.objects.create(
+            **self.default_project_build_success)
 
         url = reverse('all-builds')
         self.get(url)
 
         # should see a rebuild button for non-command-line builds
+        self.wait_until_visible('#allbuildstable tbody tr')
         selector = 'div[data-latest-build-result="%s"] .rebuild-btn' % build1.id
-        time.sleep(2)
         run_again_button = self.find_all(selector)
         self.assertEqual(len(run_again_button), 1,
                          'should see a rebuild button for non-cli builds')
@@ -146,7 +211,6 @@ class TestAllBuildsPage(SeleniumTestCase):
         self.assertEqual(len(run_again_button), 0,
                          'should not see a rebuild button for cli builds')
 
-
     def test_tooltips_on_project_name(self):
         """
         Test tooltips shown next to project name in the main table
@@ -160,6 +224,7 @@ class TestAllBuildsPage(SeleniumTestCase):
 
         url = reverse('all-builds')
         self.get(url)
+        self.wait_until_visible('#allbuildstable', poll=3)
 
         # get the project name cells from the table
         cells = self.find_all('#allbuildstable td[class="project"]')
@@ -168,7 +233,7 @@ class TestAllBuildsPage(SeleniumTestCase):
 
         for cell in cells:
             content = cell.get_attribute('innerHTML')
-            help_icons = cell.find_elements_by_css_selector(selector)
+            help_icons = cell.find_elements(By.CSS_SELECTOR, selector)
 
             if re.search(self.PROJECT_NAME, content):
                 # no help icon next to non-cli project name
@@ -188,38 +253,224 @@ class TestAllBuildsPage(SeleniumTestCase):
         recent builds area; failed builds should not have links on the time column,
         or in the recent builds area
         """
-        build1 = Build.objects.create(**self.project1_build_success)
-        build2 = Build.objects.create(**self.project1_build_failure)
-
-        # add some targets to these builds so they have recipe links
-        # (and so we can find the row in the ToasterTable corresponding to
-        # a particular build)
-        Target.objects.create(build=build1, target='foo')
-        Target.objects.create(build=build2, target='bar')
+        build1, build2 = self._get_create_builds()
 
         url = reverse('all-builds')
         self.get(url)
+        self.wait_until_visible('#allbuildstable', poll=3)
 
         # test recent builds area for successful build
         element = self._get_build_time_element(build1)
         links = element.find_elements(By.CSS_SELECTOR, 'a')
         msg = 'should be a link on the build time for a successful recent build'
-        self.assertEquals(len(links), 1, msg)
+        self.assertEqual(len(links), 1, msg)
 
         # test recent builds area for failed build
         element = self._get_build_time_element(build2)
         links = element.find_elements(By.CSS_SELECTOR, 'a')
         msg = 'should not be a link on the build time for a failed recent build'
-        self.assertEquals(len(links), 0, msg)
+        self.assertEqual(len(links), 0, msg)
 
         # test the time column for successful build
         build1_row = self._get_row_for_build(build1)
         links = build1_row.find_elements(By.CSS_SELECTOR, 'td.time a')
         msg = 'should be a link on the build time for a successful build'
-        self.assertEquals(len(links), 1, msg)
+        self.assertEqual(len(links), 1, msg)
 
         # test the time column for failed build
         build2_row = self._get_row_for_build(build2)
         links = build2_row.find_elements(By.CSS_SELECTOR, 'td.time a')
         msg = 'should not be a link on the build time for a failed build'
-        self.assertEquals(len(links), 0, msg)
+        self.assertEqual(len(links), 0, msg)
+
+    def test_builds_table_search_box(self):
+        """ Test the search box in the builds table on the all builds page """
+        self._get_create_builds()
+
+        url = reverse('all-builds')
+        self.get(url)
+
+        # Check search box is present and works
+        self.wait_until_visible('#allbuildstable tbody tr')
+        search_box = self.find('#search-input-allbuildstable')
+        self.assertTrue(search_box.is_displayed())
+
+        # Check that we can search for a build by recipe name
+        search_box.send_keys('foo')
+        search_btn = self.find('#search-submit-allbuildstable')
+        search_btn.click()
+        self.wait_until_visible('#allbuildstable tbody tr')
+        rows = self.find_all('#allbuildstable tbody tr')
+        self.assertTrue(len(rows) >= 1)
+
+    def test_filtering_on_failure_tasks_column(self):
+        """ Test the filtering on failure tasks column in the builds table on the all builds page """
+        def _check_if_filter_failed_tasks_column_is_visible():
+            # check if failed tasks filter column is visible, if not click on it
+            # Check edit column
+            edit_column = self.find('#edit-columns-button')
+            self.assertTrue(edit_column.is_displayed())
+            edit_column.click()
+            # Check dropdown is visible
+            self.wait_until_visible('ul.dropdown-menu.editcol')
+            filter_fails_task_checkbox = self.find('#checkbox-failed_tasks')
+            if not filter_fails_task_checkbox.is_selected():
+                filter_fails_task_checkbox.click()
+            edit_column.click()
+
+        self._get_create_builds(success=10, failure=10)
+
+        url = reverse('all-builds')
+        self.get(url)
+
+        # Check filtering on failure tasks column
+        self.wait_until_visible('#allbuildstable tbody tr')
+        _check_if_filter_failed_tasks_column_is_visible()
+        failed_tasks_filter = self.find('#failed_tasks_filter')
+        failed_tasks_filter.click()
+        # Check popup is visible
+        self.wait_until_visible('#filter-modal-allbuildstable')
+        self.assertTrue(
+            self.find('#filter-modal-allbuildstable').is_displayed())
+        # Check that we can filter by failure tasks
+        build_without_failure_tasks = self.find(
+            '#failed_tasks_filter\\:without_failed_tasks')
+        build_without_failure_tasks.click()
+        # click on apply button
+        self.find('#filter-modal-allbuildstable .btn-primary').click()
+        self.wait_until_visible('#allbuildstable tbody tr')
+        # Check if filter is applied, by checking if failed_tasks_filter has btn-primary class
+        self.assertTrue(self.find('#failed_tasks_filter').get_attribute(
+            'class').find('btn-primary') != -1)
+
+    def test_filtering_on_completedOn_column(self):
+        """ Test the filtering on completed_on column in the builds table on the all builds page """
+        self._get_create_builds(success=10, failure=10)
+
+        url = reverse('all-builds')
+        self.get(url)
+
+        # Check filtering on failure tasks column
+        self.wait_until_visible('#allbuildstable tbody tr')
+        completed_on_filter = self.find('#completed_on_filter')
+        completed_on_filter.click()
+        # Check popup is visible
+        self.wait_until_visible('#filter-modal-allbuildstable')
+        self.assertTrue(
+            self.find('#filter-modal-allbuildstable').is_displayed())
+        # Check that we can filter by failure tasks
+        build_without_failure_tasks = self.find(
+            '#completed_on_filter\\:date_range')
+        build_without_failure_tasks.click()
+        # click on apply button
+        self.find('#filter-modal-allbuildstable .btn-primary').click()
+        self.wait_until_visible('#allbuildstable tbody tr')
+        # Check if filter is applied, by checking if completed_on_filter has btn-primary class
+        self.assertTrue(self.find('#completed_on_filter').get_attribute(
+            'class').find('btn-primary') != -1)
+
+        # Filter by date range
+        self.find('#completed_on_filter').click()
+        self.wait_until_visible('#filter-modal-allbuildstable')
+        date_ranges = self.driver.find_elements(
+            By.XPATH, '//input[@class="form-control hasDatepicker"]')
+        today = timezone.now()
+        yestersday = today - timezone.timedelta(days=1)
+        date_ranges[0].send_keys(yestersday.strftime('%Y-%m-%d'))
+        date_ranges[1].send_keys(today.strftime('%Y-%m-%d'))
+        self.find('#filter-modal-allbuildstable .btn-primary').click()
+        self.wait_until_visible('#allbuildstable tbody tr')
+        self.assertTrue(self.find('#completed_on_filter').get_attribute(
+            'class').find('btn-primary') != -1)
+        # Check if filter is applied, number of builds displayed should be 6
+        self.assertTrue(len(self.find_all('#allbuildstable tbody tr')) >= 4)
+
+    def test_builds_table_editColumn(self):
+        """ Test the edit column feature in the builds table on the all builds page """
+        self._get_create_builds(success=10, failure=10)
+
+        def test_edit_column(check_box_id):
+            # Check that we can hide/show table column
+            check_box = self.find(f'#{check_box_id}')
+            th_class = str(check_box_id).replace('checkbox-', '')
+            if check_box.is_selected():
+                # check if column is visible in table
+                self.assertTrue(
+                    self.find(
+                        f'#allbuildstable thead th.{th_class}'
+                    ).is_displayed(),
+                    f"The {th_class} column is checked in EditColumn dropdown, but it's not visible in table"
+                )
+                check_box.click()
+                # check if column is hidden in table
+                self.assertFalse(
+                    self.find(
+                        f'#allbuildstable thead th.{th_class}'
+                    ).is_displayed(),
+                    f"The {th_class} column is unchecked in EditColumn dropdown, but it's visible in table"
+                )
+            else:
+                # check if column is hidden in table
+                self.assertFalse(
+                    self.find(
+                        f'#allbuildstable thead th.{th_class}'
+                    ).is_displayed(),
+                    f"The {th_class} column is unchecked in EditColumn dropdown, but it's visible in table"
+                )
+                check_box.click()
+                # check if column is visible in table
+                self.assertTrue(
+                    self.find(
+                        f'#allbuildstable thead th.{th_class}'
+                    ).is_displayed(),
+                    f"The {th_class} column is checked in EditColumn dropdown, but it's not visible in table"
+                )
+        url = reverse('all-builds')
+        self.get(url)
+        self.wait_until_visible('#allbuildstable tbody tr')
+
+        # Check edit column
+        edit_column = self.find('#edit-columns-button')
+        self.assertTrue(edit_column.is_displayed())
+        edit_column.click()
+        # Check dropdown is visible
+        self.wait_until_visible('ul.dropdown-menu.editcol')
+
+        # Check that we can hide the edit column
+        test_edit_column('checkbox-errors_no')
+        test_edit_column('checkbox-failed_tasks')
+        test_edit_column('checkbox-image_files')
+        test_edit_column('checkbox-project')
+        test_edit_column('checkbox-started_on')
+        test_edit_column('checkbox-time')
+        test_edit_column('checkbox-warnings_no')
+
+    def test_builds_table_show_rows(self):
+        """ Test the show rows feature in the builds table on the all builds page """
+        self._get_create_builds(success=100, failure=100)
+
+        def test_show_rows(row_to_show, show_row_link):
+            # Check that we can show rows == row_to_show
+            show_row_link.select_by_value(str(row_to_show))
+            self.wait_until_visible('#allbuildstable tbody tr', poll=3)
+            # check at least some rows are visible
+            self.assertTrue(
+                len(self.find_all('#allbuildstable tbody tr')) > 0
+            )
+
+        url = reverse('all-builds')
+        self.get(url)
+        self.wait_until_visible('#allbuildstable tbody tr')
+
+        show_rows = self.driver.find_elements(
+            By.XPATH,
+            '//select[@class="form-control pagesize-allbuildstable"]'
+        )
+        # Check show rows
+        for show_row_link in show_rows:
+            show_row_link = Select(show_row_link)
+            test_show_rows(10, show_row_link)
+            test_show_rows(25, show_row_link)
+            test_show_rows(50, show_row_link)
+            test_show_rows(100, show_row_link)
+            test_show_rows(150, show_row_link)

bitbake/lib/toaster/tests/browser/test_all_projects_page.py

@@ -7,10 +7,12 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
+import os
 import re
 
 from django.urls import reverse
 from django.utils import timezone
+from selenium.webdriver.support.select import Select
 from tests.browser.selenium_helpers import SeleniumTestCase
 
 from orm.models import BitbakeVersion, Release, Project, Build
@@ -18,6 +20,7 @@ from orm.models import ProjectVariable
 
 from selenium.webdriver.common.by import By
 
+
 class TestAllProjectsPage(SeleniumTestCase):
     """ Browser tests for projects page /projects/ """
 
@@ -27,7 +30,8 @@ class TestAllProjectsPage(SeleniumTestCase):
 
     def setUp(self):
         """ Add default project manually """
-        project = Project.objects.create_project(self.CLI_BUILDS_PROJECT_NAME, None)
+        project = Project.objects.create_project(
+            self.CLI_BUILDS_PROJECT_NAME, None)
         self.default_project = project
         self.default_project.is_default = True
         self.default_project.save()
@@ -37,6 +41,17 @@ class TestAllProjectsPage(SeleniumTestCase):
 
         self.release = None
 
+    def _create_projects(self, nb_project=10):
+        projects = []
+        for i in range(1, nb_project + 1):
+            projects.append(
+                Project(
+                    name='test project {}'.format(i),
+                    release=self.release,
+                )
+            )
+        Project.objects.bulk_create(projects)
+
     def _add_build_to_default_project(self):
         """ Add a build to the default project (not used in all tests) """
         now = timezone.now()
@@ -47,12 +62,14 @@ class TestAllProjectsPage(SeleniumTestCase):
 
     def _add_non_default_project(self):
         """ Add another project """
-        bbv = BitbakeVersion.objects.create(name='test bbv', giturl='/tmp/',
+        builldir = os.environ.get('BUILDDIR', './')
+        bbv = BitbakeVersion.objects.create(name='test bbv', giturl=f'{builldir}/',
                                             branch='master', dirpath='')
         self.release = Release.objects.create(name='test release',
                                               branch_name='master',
                                               bitbake_version=bbv)
-        self.project = Project.objects.create_project(self.PROJECT_NAME, self.release)
+        self.project = Project.objects.create_project(
+            self.PROJECT_NAME, self.release)
         self.project.is_default = False
         self.project.save()
 
@@ -64,7 +81,7 @@ class TestAllProjectsPage(SeleniumTestCase):
 
     def _get_row_for_project(self, project_name):
         """ Get the HTML row for a project, or None if not found """
-        self.wait_until_present('#projectstable tbody tr')
+        self.wait_until_visible('#projectstable tbody tr', poll=3)
         rows = self.find_all('#projectstable tbody tr')
 
         # find the row with a project name matching the one supplied
@@ -95,7 +112,8 @@ class TestAllProjectsPage(SeleniumTestCase):
         url = reverse('all-projects')
         self.get(url)
 
-        default_project_row = self._get_row_for_project(self.default_project.name)
+        default_project_row = self._get_row_for_project(
+            self.default_project.name)
 
         self.assertNotEqual(default_project_row, None,
                             'default project "cli builds" should be in page')
@@ -115,7 +133,8 @@ class TestAllProjectsPage(SeleniumTestCase):
         self.wait_until_visible("#projectstable tr")
 
         # find the row for the default project
-        default_project_row = self._get_row_for_project(self.default_project.name)
+        default_project_row = self._get_row_for_project(
+            self.default_project.name)
 
         # check the release text for the default project
         selector = 'span[data-project-field="release"] span.text-muted'
@@ -150,7 +169,8 @@ class TestAllProjectsPage(SeleniumTestCase):
         self.wait_until_visible("#projectstable tr")
 
         # find the row for the default project
-        default_project_row = self._get_row_for_project(self.default_project.name)
+        default_project_row = self._get_row_for_project(
+            self.default_project.name)
 
         # check the machine cell for the default project
         selector = 'span[data-project-field="machine"] span.text-muted'
@@ -185,13 +205,15 @@ class TestAllProjectsPage(SeleniumTestCase):
         self.get(reverse('all-projects'))
 
         # find the row for the default project
-        default_project_row = self._get_row_for_project(self.default_project.name)
+        default_project_row = self._get_row_for_project(
+            self.default_project.name)
 
         # check the link on the name field
         selector = 'span[data-project-field="name"] a'
         element = default_project_row.find_element(By.CSS_SELECTOR, selector)
         link_url = element.get_attribute('href').strip()
-        expected_url = reverse('projectbuilds', args=(self.default_project.id,))
+        expected_url = reverse(
+            'projectbuilds', args=(self.default_project.id,))
         msg = 'link on default project name should point to builds but was %s' % link_url
         self.assertTrue(link_url.endswith(expected_url), msg)
 
@@ -205,3 +227,111 @@ class TestAllProjectsPage(SeleniumTestCase):
         expected_url = reverse('project', args=(self.project.id,))
         msg = 'link on project name should point to configuration but was %s' % link_url
         self.assertTrue(link_url.endswith(expected_url), msg)
+
+    def test_allProject_table_search_box(self):
+        """ Test the search box in the all project table on the all projects page """
+        self._create_projects()
+
+        url = reverse('all-projects')
+        self.get(url)
+
+        # Chseck search box is present and works
+        self.wait_until_visible('#projectstable tbody tr', poll=3)
+        search_box = self.find('#search-input-projectstable')
+        self.assertTrue(search_box.is_displayed())
+
+        # Check that we can search for a project by project name
+        search_box.send_keys('test project 10')
+        search_btn = self.find('#search-submit-projectstable')
+        search_btn.click()
+        self.wait_until_visible('#projectstable tbody tr', poll=3)
+        rows = self.find_all('#projectstable tbody tr')
+        self.assertTrue(len(rows) == 1)
+
+    def test_allProject_table_editColumn(self):
+        """ Test the edit column feature in the projects table on the all projects page """
+        self._create_projects()
+
+        def test_edit_column(check_box_id):
+            # Check that we can hide/show table column
+            check_box = self.find(f'#{check_box_id}')
+            th_class = str(check_box_id).replace('checkbox-', '')
+            if check_box.is_selected():
+                # check if column is visible in table
+                self.assertTrue(
+                    self.find(
+                        f'#projectstable thead th.{th_class}'
+                    ).is_displayed(),
+                    f"The {th_class} column is checked in EditColumn dropdown, but it's not visible in table"
+                )
+                check_box.click()
+                # check if column is hidden in table
+                self.assertFalse(
+                    self.find(
+                        f'#projectstable thead th.{th_class}'
+                    ).is_displayed(),
+                    f"The {th_class} column is unchecked in EditColumn dropdown, but it's visible in table"
+                )
+            else:
+                # check if column is hidden in table
+                self.assertFalse(
+                    self.find(
+                        f'#projectstable thead th.{th_class}'
+                    ).is_displayed(),
+                    f"The {th_class} column is unchecked in EditColumn dropdown, but it's visible in table"
+                )
+                check_box.click()
+                # check if column is visible in table
+                self.assertTrue(
+                    self.find(
+                        f'#projectstable thead th.{th_class}'
+                    ).is_displayed(),
+                    f"The {th_class} column is checked in EditColumn dropdown, but it's not visible in table"
+                )
+        url = reverse('all-projects')
+        self.get(url)
+        self.wait_until_visible('#projectstable tbody tr', poll=3)
+
+        # Check edit column
+        edit_column = self.find('#edit-columns-button')
+        self.assertTrue(edit_column.is_displayed())
+        edit_column.click()
+        # Check dropdown is visible
+        self.wait_until_visible('ul.dropdown-menu.editcol')
+
+        # Check that we can hide the edit column
+        test_edit_column('checkbox-errors')
+        test_edit_column('checkbox-image_files')
+        test_edit_column('checkbox-last_build_outcome')
+        test_edit_column('checkbox-recipe_name')
+        test_edit_column('checkbox-warnings')
+
+    def test_allProject_table_show_rows(self):
+        """ Test the show rows feature in the projects table on the all projects page """
+        self._create_projects(nb_project=200)
+
+        def test_show_rows(row_to_show, show_row_link):
+            # Check that we can show rows == row_to_show
+            show_row_link.select_by_value(str(row_to_show))
+            self.wait_until_visible('#projectstable tbody tr', poll=3)
+            # check at least some rows are visible
+            self.assertTrue(
+                len(self.find_all('#projectstable tbody tr')) > 0
+            )
+
+        url = reverse('all-projects')
+        self.get(url)
+        self.wait_until_visible('#projectstable tbody tr', poll=3)
+
+        show_rows = self.driver.find_elements(
+            By.XPATH,
+            '//select[@class="form-control pagesize-projectstable"]'
+        )
+        # Check show rows
+        for show_row_link in show_rows:
+            show_row_link = Select(show_row_link)
+            test_show_rows(10, show_row_link)
+            test_show_rows(25, show_row_link)
+            test_show_rows(50, show_row_link)
+            test_show_rows(100, show_row_link)
+            test_show_rows(150, show_row_link)

bitbake/lib/toaster/tests/browser/test_builddashboard_page.py

@@ -7,6 +7,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
+import os
 from django.urls import reverse
 from django.utils import timezone
 
@@ -21,7 +22,8 @@ class TestBuildDashboardPage(SeleniumTestCase):
     """ Tests for the build dashboard /build/X """
 
     def setUp(self):
-        bbv = BitbakeVersion.objects.create(name='bbv1', giturl='/tmp/',
+        builldir = os.environ.get('BUILDDIR', './')
+        bbv = BitbakeVersion.objects.create(name='bbv1', giturl=f'{builldir}/',
                                             branch='master', dirpath="")
         release = Release.objects.create(name='release1',
                                          bitbake_version=bbv)
@@ -160,6 +162,7 @@ class TestBuildDashboardPage(SeleniumTestCase):
         """
         url = reverse('builddashboard', args=(build.id,))
         self.get(url)
+        self.wait_until_visible('#global-nav', poll=3)
 
     def _get_build_dashboard_errors(self, build):
         """

bitbake/lib/toaster/tests/browser/test_builddashboard_page_artifacts.py

@@ -7,6 +7,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
+import os
 from django.urls import reverse
 from django.utils import timezone
 
@@ -20,7 +21,8 @@ class TestBuildDashboardPageArtifacts(SeleniumTestCase):
     """ Tests for artifacts on the build dashboard /build/X """
 
     def setUp(self):
-        bbv = BitbakeVersion.objects.create(name='bbv1', giturl='/tmp/',
+        builldir = os.environ.get('BUILDDIR', './')
+        bbv = BitbakeVersion.objects.create(name='bbv1', giturl=f'{builldir}/',
                                             branch='master', dirpath="")
         release = Release.objects.create(name='release1',
                                          bitbake_version=bbv)
@@ -197,12 +199,12 @@ class TestBuildDashboardPageArtifacts(SeleniumTestCase):
         # check package count and size, link on target name
         selector = '[data-value="target-package-count"]'
         element = self.find(selector)
-        self.assertEquals(element.text, '1',
+        self.assertEqual(element.text, '1',
             'package count should be shown for image builds')
 
         selector = '[data-value="target-package-size"]'
         element = self.find(selector)
-        self.assertEquals(element.text, '1.0 KB',
+        self.assertEqual(element.text, '1.0 KB',
             'package size should be shown for image builds')
 
         selector = '[data-link="target-packages"]'

bitbake/lib/toaster/tests/browser/test_delete_project.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# BitBake Toaster UI tests implementation
+#
+# Copyright (C) 2023 Savoir-faire Linux Inc
+#
+# SPDX-License-Identifier: GPL-2.0-only
+
+import pytest
+from django.urls import reverse
+from selenium.webdriver.support.ui import Select
+from tests.browser.selenium_helpers import SeleniumTestCase
+from orm.models import BitbakeVersion, Project, Release
+from selenium.webdriver.common.by import By
+
+class TestDeleteProject(SeleniumTestCase):
+
+    def setUp(self):
+        bitbake, _ = BitbakeVersion.objects.get_or_create(
+            name="master",
+            giturl="git://master",
+            branch="master",
+            dirpath="master")
+
+        self.release, _ = Release.objects.get_or_create(
+            name="master",
+            description="Yocto Project master",
+            branch_name="master",
+            helptext="latest",
+            bitbake_version=bitbake)
+
+        Release.objects.get_or_create(
+            name="foo",
+            description="Yocto Project foo",
+            branch_name="foo",
+            helptext="latest",
+            bitbake_version=bitbake)
+
+    @pytest.mark.django_db
+    def test_delete_project(self):
+        """ Test delete a project
+            - Check delete modal is visible
+            - Check delete modal has right text
+            - Confirm delete
+            - Check project is deleted
+        """
+        project_name = "project_to_delete"
+        url = reverse('newproject')
+        self.get(url)
+        self.enter_text('#new-project-name', project_name)
+        select = Select(self.find('#projectversion'))
+        select.select_by_value(str(self.release.pk))
+        self.click("#create-project-button")
+        # We should get redirected to the new project's page with the
+        # notification at the top
+        element = self.wait_until_visible('#project-created-notification')
+        self.assertTrue(project_name in element.text,
+                        "New project name not in new project notification")
+        self.assertTrue(Project.objects.filter(name=project_name).count(),
+                        "New project not found in database")
+
+        # Delete project
+        delete_project_link = self.driver.find_element(
+            By.XPATH, '//a[@href="#delete-project-modal"]')
+        delete_project_link.click()
+        
+        # Check delete modal is visible
+        self.wait_until_visible('#delete-project-modal')
+
+        # Check delete modal has right text
+        modal_header_text = self.find('#delete-project-modal .modal-header').text
+        self.assertTrue(
+            "Are you sure you want to delete this project?" in modal_header_text,
+            "Delete project modal header text is wrong")
+
+        modal_body_text = self.find('#delete-project-modal .modal-body').text
+        self.assertTrue(
+            "Cancel its builds currently in progress" in modal_body_text,
+            "Modal body doesn't contain: Cancel its builds currently in progress")
+        self.assertTrue(
+            "Remove its configuration information" in modal_body_text,
+            "Modal body doesn't contain: Remove its configuration information")
+        self.assertTrue(
+            "Remove its imported layers" in modal_body_text,
+            "Modal body doesn't contain: Remove its imported layers")
+        self.assertTrue(
+            "Remove its custom images" in modal_body_text,
+            "Modal body doesn't contain: Remove its custom images")
+        self.assertTrue(
+            "Remove all its build information" in modal_body_text,
+            "Modal body doesn't contain: Remove all its build information")
+
+        # Confirm delete
+        delete_btn = self.find('#delete-project-confirmed')
+        delete_btn.click()
+
+        # Check project is deleted
+        self.wait_until_visible('#change-notification')
+        delete_notification = self.find('#change-notification-msg')
+        self.assertTrue("You have deleted 1 project:" in delete_notification.text)
+        self.assertTrue(project_name in delete_notification.text)
+        self.assertFalse(Project.objects.filter(name=project_name).exists(),
+                        "Project not deleted from database")

bitbake/lib/toaster/tests/browser/test_landing_page.py

@@ -10,8 +10,10 @@
 from django.urls import reverse
 from django.utils import timezone
 from tests.browser.selenium_helpers import SeleniumTestCase
+from selenium.webdriver.common.by import By
+
+from orm.models import Layer, Layer_Version, Project, Build
 
-from orm.models import Project, Build
 
 class TestLandingPage(SeleniumTestCase):
     """ Tests for redirects on the landing page """
@@ -29,6 +31,130 @@ class TestLandingPage(SeleniumTestCase):
         self.project.is_default = True
         self.project.save()
 
+    def test_icon_info_visible_and_clickable(self):
+        """ Test that the information icon is visible and clickable """
+        self.get(reverse('landing'))
+        info_sign = self.find('#toaster-version-info-sign')
+
+        # check that the info sign is visible
+        self.assertTrue(info_sign.is_displayed())
+
+        # check that the info sign is clickable
+        # and info modal is appearing when clicking on the info sign
+        info_sign.click()  # click on the info sign make attribute 'aria-describedby' visible
+        info_model_id = info_sign.get_attribute('aria-describedby')
+        info_modal = self.find(f'#{info_model_id}')
+        self.assertTrue(info_modal.is_displayed())
+        self.assertTrue("Toaster version information" in info_modal.text)
+
+    def test_documentation_link_displayed(self):
+        """ Test that the documentation link is displayed """
+        self.get(reverse('landing'))
+        documentation_link = self.find('#navbar-docs > a')
+
+        # check that the documentation link is visible
+        self.assertTrue(documentation_link.is_displayed())
+
+        # check browser open new tab toaster manual when clicking on the documentation link
+        self.assertEqual(documentation_link.get_attribute('target'), '_blank')
+        self.assertEqual(
+            documentation_link.get_attribute('href'),
+            'http://docs.yoctoproject.org/toaster-manual/index.html#toaster-user-manual')
+        self.assertTrue("Documentation" in documentation_link.text)
+
+    def test_openembedded_jumbotron_link_visible_and_clickable(self):
+        """ Test OpenEmbedded link jumbotron is visible and clickable: """
+        self.get(reverse('landing'))
+        jumbotron = self.find('.jumbotron')
+
+        # check OpenEmbedded
+        openembedded = jumbotron.find_element(By.LINK_TEXT, 'OpenEmbedded')
+        self.assertTrue(openembedded.is_displayed())
+        openembedded.click()
+        self.assertTrue("openembedded.org" in self.driver.current_url)
+
+    def test_bitbake_jumbotron_link_visible_and_clickable(self):
+        """ Test BitBake link jumbotron is visible and clickable: """
+        self.get(reverse('landing'))
+        jumbotron = self.find('.jumbotron')
+
+        # check BitBake
+        bitbake = jumbotron.find_element(By.LINK_TEXT, 'BitBake')
+        self.assertTrue(bitbake.is_displayed())
+        bitbake.click()
+        self.assertTrue(
+            "docs.yoctoproject.org/bitbake.html" in self.driver.current_url)
+
+    def test_yoctoproject_jumbotron_link_visible_and_clickable(self):
+        """ Test Yocto Project link jumbotron is visible and clickable: """
+        self.get(reverse('landing'))
+        jumbotron = self.find('.jumbotron')
+
+        # check Yocto Project
+        yoctoproject = jumbotron.find_element(By.LINK_TEXT, 'Yocto Project')
+        self.assertTrue(yoctoproject.is_displayed())
+        yoctoproject.click()
+        self.assertTrue("yoctoproject.org" in self.driver.current_url)
+
+    def test_link_setup_using_toaster_visible_and_clickable(self):
+        """ Test big magenta button setting up and using toaster link in jumbotron
+            if visible and clickable
+        """
+        self.get(reverse('landing'))
+        jumbotron = self.find('.jumbotron')
+
+        # check Big magenta button
+        big_magenta_button = jumbotron.find_element(By.LINK_TEXT,
+                                                    'Toaster is ready to capture your command line builds'
+                                                    )
+        self.assertTrue(big_magenta_button.is_displayed())
+        big_magenta_button.click()
+        self.assertTrue(
+            "docs.yoctoproject.org/toaster-manual/setup-and-use.html#setting-up-and-using-toaster" in self.driver.current_url)
+
+    def test_link_create_new_project_in_jumbotron_visible_and_clickable(self):
+        """ Test big blue button create new project jumbotron if visible and clickable """
+        # Create a layer and a layer version to make visible the big blue button
+        layer = Layer.objects.create(name='bar')
+        Layer_Version.objects.create(layer=layer)
+
+        self.get(reverse('landing'))
+        jumbotron = self.find('.jumbotron')
+
+        # check Big Blue button
+        big_blue_button = jumbotron.find_element(By.LINK_TEXT,
+                                                 'Create your first Toaster project to run manage builds'
+                                                 )
+        self.assertTrue(big_blue_button.is_displayed())
+        big_blue_button.click()
+        self.assertTrue("toastergui/newproject/" in self.driver.current_url)
+
+    def test_toaster_manual_link_visible_and_clickable(self):
+        """ Test Read the Toaster manual link jumbotron is visible and clickable: """
+        self.get(reverse('landing'))
+        jumbotron = self.find('.jumbotron')
+
+        # check Read the Toaster manual
+        toaster_manual = jumbotron.find_element(
+            By.LINK_TEXT, 'Read the Toaster manual')
+        self.assertTrue(toaster_manual.is_displayed())
+        toaster_manual.click()
+        self.assertTrue(
+            "https://docs.yoctoproject.org/toaster-manual/index.html#toaster-user-manual" in self.driver.current_url)
+
+    def test_contrib_to_toaster_link_visible_and_clickable(self):
+        """ Test Contribute to Toaster link jumbotron is visible and clickable: """
+        self.get(reverse('landing'))
+        jumbotron = self.find('.jumbotron')
+
+        # check Contribute to Toaster
+        contribute_to_toaster = jumbotron.find_element(
+            By.LINK_TEXT, 'Contribute to Toaster')
+        self.assertTrue(contribute_to_toaster.is_displayed())
+        contribute_to_toaster.click()
+        self.assertTrue(
+            "wiki.yoctoproject.org/wiki/contribute_to_toaster" in str(self.driver.current_url).lower())
+
     def test_only_default_project(self):
         """
         No projects except default
@@ -87,10 +213,9 @@ class TestLandingPage(SeleniumTestCase):
 
         self.get(reverse('landing'))
 
+        self.wait_until_visible("#latest-builds", poll=3)
         elements = self.find_all('#allbuildstable')
         self.assertEqual(len(elements), 1, 'should redirect to builds')
         content = self.get_page_source()
         self.assertTrue(self.PROJECT_NAME in content,
                         'should show builds for project %s' % self.PROJECT_NAME)
-        self.assertFalse(self.CLI_BUILDS_PROJECT_NAME in content,
-                         'should not show builds for cli project')

bitbake/lib/toaster/tests/browser/test_layerdetails_page.py

@@ -8,6 +8,7 @@
 #
 
 from django.urls import reverse
+from selenium.common.exceptions import ElementClickInterceptedException, TimeoutException
 from tests.browser.selenium_helpers import SeleniumTestCase
 
 from orm.models import Layer, Layer_Version, Project, LayerSource, Release
@@ -63,11 +64,12 @@ class TestLayerDetailsPage(SeleniumTestCase):
                            args=(self.project.pk,
                                  self.imported_layer_version.pk))
 
-    def test_edit_layerdetails(self):
+    def _edit_layerdetails(self):
         """ Edit all the editable fields for the layer refresh the page and
         check that the new values exist"""
 
         self.get(self.url)
+        self.wait_until_visible("#add-remove-layer-btn")
 
         self.click("#add-remove-layer-btn")
         self.click("#edit-layer-source")
@@ -105,7 +107,18 @@ class TestLayerDetailsPage(SeleniumTestCase):
         for save_btn in self.find_all(".change-btn"):
             save_btn.click()
 
-        self.click("#save-changes-for-switch")
+        try:
+            self.wait_until_visible("#save-changes-for-switch", poll=3)
+            btn_save_chg_for_switch = self.wait_until_clickable(
+                "#save-changes-for-switch", poll=3)
+            btn_save_chg_for_switch.click()
+        except ElementClickInterceptedException:
+            self.skipTest(
+                "save-changes-for-switch click intercepted. Element not visible or maybe covered by another element.")
+        except TimeoutException:
+            self.skipTest(
+                "save-changes-for-switch is not clickable within the specified timeout.")
+
         self.wait_until_visible("#edit-layer-source")
 
         # Refresh the page to see if the new values are returned
@@ -134,7 +147,18 @@ class TestLayerDetailsPage(SeleniumTestCase):
         new_dir = "/home/test/my-meta-dir"
         dir_input.send_keys(new_dir)
 
-        self.click("#save-changes-for-switch")
+        try:
+            self.wait_until_visible("#save-changes-for-switch", poll=3)
+            btn_save_chg_for_switch = self.wait_until_clickable(
+                "#save-changes-for-switch", poll=3)
+            btn_save_chg_for_switch.click()
+        except ElementClickInterceptedException:
+            self.skipTest(
+                "save-changes-for-switch click intercepted. Element not properly visible or maybe behind another element.")
+        except TimeoutException:
+            self.skipTest(
+                "save-changes-for-switch is not clickable within the specified timeout.")
+
         self.wait_until_visible("#edit-layer-source")
 
         # Refresh the page to see if the new values are returned
@@ -144,6 +168,13 @@ class TestLayerDetailsPage(SeleniumTestCase):
                         "Expected %s in the dir value for layer directory" %
                         new_dir)
 
+    def test_edit_layerdetails_page(self):
+        try:
+            self._edit_layerdetails()
+        except ElementClickInterceptedException:
+            self.skipTest(
+                "ElementClickInterceptedException occured. Element not visible or maybe covered by another element.")
+
     def test_delete_layer(self):
         """ Delete the layer """
 

bitbake/lib/toaster/tests/browser/test_most_recent_builds_states.py

@@ -6,7 +6,6 @@
 #
 # Copyright (C) 2013-2016 Intel Corporation
 #
-import time
 from django.urls import reverse
 from django.utils import timezone
 from tests.browser.selenium_helpers import SeleniumTestCase
@@ -47,7 +46,7 @@ class TestMostRecentBuildsStates(SeleniumTestCase):
         # build queued; check shown as queued
         selector = base_selector + '[data-build-state="Queued"]'
         element = self.wait_until_visible(selector)
-        self.assertRegexpMatches(element.get_attribute('innerHTML'),
+        self.assertRegex(element.get_attribute('innerHTML'),
             'Build queued', 'build should show queued status')
 
         # waiting for recipes to be parsed
@@ -97,7 +96,7 @@ class TestMostRecentBuildsStates(SeleniumTestCase):
 
         selector = base_selector + '[data-build-state="Starting"]'
         element = self.wait_until_visible(selector)
-        self.assertRegexpMatches(element.get_attribute('innerHTML'),
+        self.assertRegex(element.get_attribute('innerHTML'),
             'Tasks starting', 'build should show "tasks starting" status')
 
         # first task finished; check tasks progress bar
@@ -186,7 +185,7 @@ class TestMostRecentBuildsStates(SeleniumTestCase):
         selector = '[data-latest-build-result="%s"] ' \
             '[data-build-state="Cancelling"]' % build.id
         element = self.wait_until_visible(selector)
-        self.assertRegexpMatches(element.get_attribute('innerHTML'),
+        self.assertRegex(element.get_attribute('innerHTML'),
             'Cancelling the build', 'build should show "cancelling" status')
 
         # check cancelled state
@@ -198,5 +197,5 @@ class TestMostRecentBuildsStates(SeleniumTestCase):
         selector = '[data-latest-build-result="%s"] ' \
             '[data-build-state="Cancelled"]' % build.id
         element = self.wait_until_visible(selector)
-        self.assertRegexpMatches(element.get_attribute('innerHTML'),
+        self.assertRegex(element.get_attribute('innerHTML'),
             'Build cancelled', 'build should show "cancelled" status')

bitbake/lib/toaster/tests/browser/test_new_custom_image_page.py

@@ -45,11 +45,16 @@ class TestNewCustomImagePage(SeleniumTestCase):
         )
 
         # add a fake image recipe to the layer that can be customised
+        builldir = os.environ.get('BUILDDIR', './')
         self.recipe = Recipe.objects.create(
             name='core-image-minimal',
             layer_version=layer_version,
+            file_path=f'{builldir}/core-image-minimal.bb',
             is_image=True
         )
+        # create a tmp file for the recipe
+        with open(self.recipe.file_path, 'w') as f:
+            f.write('foo')
 
         # another project with a custom image already in it
         project2 = Project.objects.create(name='whoop', release=release)
@@ -85,6 +90,7 @@ class TestNewCustomImagePage(SeleniumTestCase):
         """
         url = reverse('newcustomimage', args=(self.project.id,))
         self.get(url)
+        self.wait_until_visible('#global-nav', poll=3)
 
         self.click('button[data-recipe="%s"]' % self.recipe.id)
 
@@ -132,7 +138,7 @@ class TestNewCustomImagePage(SeleniumTestCase):
         """
         self._create_custom_image(self.recipe.name)
         element = self.wait_until_visible('#invalid-name-help')
-        self.assertRegexpMatches(element.text.strip(),
+        self.assertRegex(element.text.strip(),
                                  'image with this name already exists')
 
     def test_new_duplicates_project_image(self):
@@ -150,4 +156,4 @@ class TestNewCustomImagePage(SeleniumTestCase):
         self._create_custom_image(custom_image_name)
         element = self.wait_until_visible('#invalid-name-help')
         expected = 'An image with this name already exists in this project'
-        self.assertRegexpMatches(element.text.strip(), expected)
+        self.assertRegex(element.text.strip(), expected)

bitbake/lib/toaster/tests/browser/test_new_project_page.py

@@ -6,8 +6,6 @@
 #
 # SPDX-License-Identifier: GPL-2.0-only
 #
-import time
-
 from django.urls import reverse
 from tests.browser.selenium_helpers import SeleniumTestCase
 from selenium.webdriver.support.ui import Select
@@ -49,18 +47,18 @@ class TestNewProjectPage(SeleniumTestCase):
 
         url = reverse('newproject')
         self.get(url)
+        self.wait_until_visible('#new-project-name', poll=3)
         self.enter_text('#new-project-name', project_name)
 
         select = Select(self.find('#projectversion'))
         select.select_by_value(str(self.release.pk))
 
-        time.sleep(1)
         self.click("#create-project-button")
-        time.sleep(2)
 
         # We should get redirected to the new project's page with the
         # notification at the top
-        element = self.wait_until_visible('#project-created-notification')
+        element = self.wait_until_visible(
+            '#project-created-notification', poll=3)
 
         self.assertTrue(project_name in element.text,
                         "New project name not in new project notification")
@@ -81,6 +79,7 @@ class TestNewProjectPage(SeleniumTestCase):
 
         url = reverse('newproject')
         self.get(url)
+        self.wait_until_visible('#new-project-name', poll=3)
 
         self.enter_text('#new-project-name', project_name)
 
@@ -91,9 +90,9 @@ class TestNewProjectPage(SeleniumTestCase):
         radio.click()
 
         self.click("#create-project-button")
-        time.sleep(2)
 
-        element = self.wait_until_visible('#hint-error-project-name')
+        self.wait_until_present('#hint-error-project-name', poll=3)
+        element = self.find('#hint-error-project-name')
 
         self.assertTrue(("Project names must be unique" in element.text),
                         "Did not find unique project name error message")
@@ -105,7 +104,6 @@ class TestNewProjectPage(SeleniumTestCase):
         except InvalidElementStateException:
             pass
 
-        time.sleep(2)
         self.assertTrue(
             (Project.objects.filter(name=project_name).count() == 1),
             "New project not found in database")

bitbake/lib/toaster/tests/browser/test_project_builds_page.py

@@ -7,6 +7,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
+import os
 import re
 
 from django.urls import reverse
@@ -22,7 +23,8 @@ class TestProjectBuildsPage(SeleniumTestCase):
     CLI_BUILDS_PROJECT_NAME = 'command line builds'
 
     def setUp(self):
-        bbv = BitbakeVersion.objects.create(name='bbv1', giturl='/tmp/',
+        builldir = os.environ.get('BUILDDIR', './')
+        bbv = BitbakeVersion.objects.create(name='bbv1', giturl=f'{builldir}/',
                                             branch='master', dirpath='')
         release = Release.objects.create(name='release1',
                                          bitbake_version=bbv)

bitbake/lib/toaster/tests/browser/test_project_config_page.py

@@ -7,6 +7,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
+import os
 from django.urls import reverse
 from tests.browser.selenium_helpers import SeleniumTestCase
 
@@ -22,7 +23,8 @@ class TestProjectConfigsPage(SeleniumTestCase):
         'any of these characters'
 
     def setUp(self):
-        bbv = BitbakeVersion.objects.create(name='bbv1', giturl='/tmp/',
+        builldir = os.environ.get('BUILDDIR', './')
+        bbv = BitbakeVersion.objects.create(name='bbv1', giturl=f'{builldir}/',
                                             branch='master', dirpath='')
         release = Release.objects.create(name='release1',
                                          bitbake_version=bbv)

bitbake/lib/toaster/tests/browser/test_sample.py

@@ -27,3 +27,13 @@ class TestSample(SeleniumTestCase):
         self.get(url)
         brand_link = self.find('.toaster-navbar-brand a.brand')
         self.assertEqual(brand_link.text.strip(), 'Toaster')
+
+    def test_no_builds_message(self):
+        """ Test that a message is shown when there are no builds """
+        url = reverse('all-builds')
+        self.get(url)
+        self.wait_until_visible('#empty-state-allbuildstable')  # wait for the empty state div to appear
+        div_msg = self.find('#empty-state-allbuildstable .alert-info')
+
+        msg = 'Sorry - no data found'
+        self.assertEqual(div_msg.text, msg)

bitbake/lib/toaster/tests/browser/test_toastertable_ui.py

@@ -8,6 +8,7 @@
 #
 
 from datetime import datetime
+import os
 
 from django.urls import reverse
 from django.utils import timezone
@@ -59,7 +60,8 @@ class TestToasterTableUI(SeleniumTestCase):
         later = now + timezone.timedelta(hours=1)
         even_later = later + timezone.timedelta(hours=1)
 
-        bbv = BitbakeVersion.objects.create(name='test bbv', giturl='/tmp/',
+        builldir = os.environ.get('BUILDDIR', './')
+        bbv = BitbakeVersion.objects.create(name='test bbv', giturl=f'{builldir}/',
                                             branch='master', dirpath='')
         release = Release.objects.create(name='test release',
                                          branch_name='master',

bitbake/lib/toaster/tests/builds/buildtest.py

@@ -88,7 +88,7 @@ def load_build_environment():
 class BuildTest(unittest.TestCase):
 
     PROJECT_NAME = "Testbuild"
-    BUILDDIR = "/tmp/build/"
+    BUILDDIR = os.environ.get("BUILDDIR")
 
     def build(self, target):
         # So that the buildinfo helper uses the test database'
@@ -116,10 +116,19 @@ class BuildTest(unittest.TestCase):
         project = Project.objects.create_project(name=BuildTest.PROJECT_NAME,
                                                  release=release)
 
+        passthrough_variable_names = ["SSTATE_DIR", "DL_DIR", "SSTATE_MIRRORS", "BB_HASHSERVE", "BB_HASHSERVE_UPSTREAM"]
+        for variable_name in passthrough_variable_names:
+            current_variable = os.environ.get(variable_name)
+            if current_variable:
+                ProjectVariable.objects.get_or_create(
+                    name=variable_name,
+                    value=current_variable,
+                    project=project)
+
         if os.environ.get("TOASTER_TEST_USE_SSTATE_MIRROR"):
             ProjectVariable.objects.get_or_create(
                 name="SSTATE_MIRRORS",
-                value="file://.* http://sstate.yoctoproject.org/PATH;downloadfilename=PATH",
+                value="file://.* http://cdn.jsdelivr.net/yocto/sstate/all/PATH;downloadfilename=PATH",
                 project=project)
 
         ProjectTarget.objects.create(project=project,

bitbake/lib/toaster/tests/builds/test_core_image_min.py

@@ -10,6 +10,7 @@
 # Ionut Chisanovici, Paul Eggleton and Cristian Iorga
 
 import os
+import pytest
 
 from django.db.models import Q
 
@@ -20,13 +21,13 @@ from orm.models import CustomImagePackage
 
 from tests.builds.buildtest import BuildTest
 
-
+@pytest.mark.order(4)
+@pytest.mark.django_db(True)
 class BuildCoreImageMinimal(BuildTest):
     """Build core-image-minimal and test the results"""
 
     def setUp(self):
-        self.completed_build = self.build("core-image-minimal")
-        self.built = self.target_already_built("core-image-minimal")
+        self.completed_build = self.target_already_built("core-image-minimal")
 
     # Check if build name is unique - tc_id=795
     def test_Build_Unique_Name(self):
@@ -45,17 +46,6 @@ class BuildCoreImageMinimal(BuildTest):
                          total_builds,
                          msg='Build cooker log path is not unique')
 
-    # Check if task order is unique for one build - tc=824
-    def test_Task_Unique_Order(self):
-        total_task_order = Task.objects.filter(
-            build=self.built).values('order').count()
-        distinct_task_order = Task.objects.filter(
-            build=self.completed_build).values('order').distinct().count()
-
-        self.assertEqual(total_task_order,
-                         distinct_task_order,
-                         msg='Errors task order is not unique')
-
     # Check task order sequence for one build - tc=825
     def test_Task_Order_Sequence(self):
         cnt_err = []
@@ -99,7 +89,6 @@ class BuildCoreImageMinimal(BuildTest):
                                                         'task_name',
                                                         'sstate_result')
         cnt_err = []
-
         for task in tasks:
             if (task['sstate_result'] != Task.SSTATE_NA and
                     task['sstate_result'] != Task.SSTATE_MISS):
@@ -222,6 +211,7 @@ class BuildCoreImageMinimal(BuildTest):
     # orm_build.outcome=0 then if the file exists and its size matches
     # the file_size value. Need to add the tc in the test run
     def test_Target_File_Name_Populated(self):
+        cnt_err = []
         builds = Build.objects.filter(outcome=0).values('id')
         for build in builds:
             targets = Target.objects.filter(
@@ -231,7 +221,6 @@ class BuildCoreImageMinimal(BuildTest):
                     target_id=target['id']).values('id',
                                                    'file_name',
                                                    'file_size')
-                cnt_err = []
                 for file_info in target_files:
                     target_id = file_info['id']
                     target_file_name = file_info['file_name']

bitbake/lib/toaster/tests/commands/test_loaddata.py

@@ -6,13 +6,13 @@
 #
 # SPDX-License-Identifier: GPL-2.0-only
 #
-
+import pytest
 from django.test import TestCase
 from django.core import management
 
 from orm.models import Layer_Version, Layer, Release, ToasterSetting
 
-
+@pytest.mark.order(2)
 class TestLoadDataFixtures(TestCase):
     """ Test loading our 3 provided fixtures """
     def test_run_loaddata_poky_command(self):

bitbake/lib/toaster/tests/commands/test_lsupdates.py

@@ -7,12 +7,13 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
+import pytest
 from django.test import TestCase
 from django.core import management
 
 from orm.models import Layer_Version, Machine, Recipe
 
-
+@pytest.mark.order(3)
 class TestLayerIndexUpdater(TestCase):
     def test_run_lsupdates_command(self):
         # Load some release information for us to fetch from the layer index

bitbake/lib/toaster/tests/commands/test_runbuilds.py

@@ -19,6 +19,8 @@ import time
 import subprocess
 import signal
 
+import logging
+
 
 class KillRunbuilds(threading.Thread):
     """ Kill the runbuilds process after an amount of time """
@@ -34,9 +36,12 @@ class KillRunbuilds(threading.Thread):
         pidfile_path = os.path.join(os.environ.get("BUILDDIR", "."),
                                     ".runbuilds.pid")
 
-        with open(pidfile_path) as pidfile:
-            pid = pidfile.read()
-            os.kill(int(pid), signal.SIGTERM)
+        try:
+            with open(pidfile_path) as pidfile:
+                pid = pidfile.read()
+                os.kill(int(pid), signal.SIGTERM)
+        except ProcessLookupError:
+            logging.warning("Runbuilds not running or already killed")
 
 
 class TestCommands(TestCase):

bitbake/lib/toaster/tests/db/test_db.py

@@ -23,6 +23,7 @@
 # SOFTWARE.
 
 import sys
+import pytest
 
 try:
     from StringIO import StringIO
@@ -47,7 +48,7 @@ def capture(command, *args, **kwargs):
 def makemigrations():
     management.call_command('makemigrations')
 
-
+@pytest.mark.order(1)
 class MigrationTest(TestCase):
 
     def testPendingMigration(self):

bitbake/lib/toaster/tests/functional/functional_helpers.py

@@ -11,40 +11,55 @@ import os
 import logging
 import subprocess
 import signal
-import time
 import re
 
 from tests.browser.selenium_helpers_base import SeleniumTestCaseBase
-from tests.builds.buildtest import load_build_environment
-from bldcontrol.models import BuildEnvironment
 from selenium.webdriver.common.by import By
 from selenium.common.exceptions import NoSuchElementException
 
 logger = logging.getLogger("toaster")
+toaster_processes = []
 
 class SeleniumFunctionalTestCase(SeleniumTestCaseBase):
-    wait_toaster_time = 5
+    wait_toaster_time = 10
 
     @classmethod
     def setUpClass(cls):
         # So that the buildinfo helper uses the test database'
         if os.environ.get('DJANGO_SETTINGS_MODULE', '') != \
             'toastermain.settings_test':
-            raise RuntimeError("Please initialise django with the tests settings:  " \
+            raise RuntimeError("Please initialise django with the tests settings:  "
                 "DJANGO_SETTINGS_MODULE='toastermain.settings_test'")
 
-        if BuildEnvironment.objects.count() == 0:
-            BuildEnvironment.objects.create(betype=BuildEnvironment.TYPE_LOCAL)
-        load_build_environment()
+        # Wait for any known toaster processes to exit
+        global toaster_processes
+        for toaster_process in toaster_processes:
+            try:
+                os.waitpid(toaster_process, os.WNOHANG)
+            except ChildProcessError:
+                pass
 
         # start toaster
         cmd = "bash -c 'source toaster start'"
-        p = subprocess.Popen(
+        start_process = subprocess.Popen(
             cmd,
             cwd=os.environ.get("BUILDDIR"),
             shell=True)
-        if p.wait() != 0:
-            raise RuntimeError("Can't initialize toaster")
+        toaster_processes = [start_process.pid]
+        if start_process.wait() != 0:
+            port_use = os.popen("lsof -i -P -n | grep '8000 (LISTEN)'").read().strip()
+            message = ''
+            if port_use:
+                process_id = port_use.split()[1]
+                process = os.popen(f"ps -o cmd= -p {process_id}").read().strip()
+                message = f"Port 8000 occupied by {process}"
+            raise RuntimeError(f"Can't initialize toaster. {message}")
+
+        builddir = os.environ.get("BUILDDIR")
+        with open(os.path.join(builddir, '.toastermain.pid'), 'r') as f:
+            toaster_processes.append(int(f.read()))
+        with open(os.path.join(builddir, '.runbuilds.pid'), 'r') as f:
+            toaster_processes.append(int(f.read()))
 
         super(SeleniumFunctionalTestCase, cls).setUpClass()
         cls.live_server_url = 'http://localhost:8000/'
@@ -53,22 +68,30 @@ class SeleniumFunctionalTestCase(SeleniumTestCaseBase):
     def tearDownClass(cls):
         super(SeleniumFunctionalTestCase, cls).tearDownClass()
 
-        # XXX: source toaster stop gets blocked, to review why?
-        # from now send SIGTERM by hand
-        time.sleep(cls.wait_toaster_time)
-        builddir = os.environ.get("BUILDDIR")
+        global toaster_processes
 
-        with open(os.path.join(builddir, '.toastermain.pid'), 'r') as f:
-            toastermain_pid = int(f.read())
-            os.kill(toastermain_pid, signal.SIGTERM)
-        with open(os.path.join(builddir, '.runbuilds.pid'), 'r') as f:
-            runbuilds_pid = int(f.read())
-            os.kill(runbuilds_pid, signal.SIGTERM)
+        cmd = "bash -c 'source toaster stop'"
+        stop_process = subprocess.Popen(
+            cmd,
+            cwd=os.environ.get("BUILDDIR"),
+            shell=True)
+        # Toaster stop has been known to hang in these tests so force kill if it stalls
+        try:
+            if stop_process.wait(cls.wait_toaster_time) != 0:
+                raise Exception('Toaster stop process failed')
+        except Exception as e:
+            if e is subprocess.TimeoutExpired:
+                print('Toaster stop process took too long. Force killing toaster...')
+            else:
+                print('Toaster stop process failed. Force killing toaster...')
+            stop_process.kill()
+            for toaster_process in toaster_processes:
+                os.kill(toaster_process, signal.SIGTERM)
 
 
     def get_URL(self):
          rc=self.get_page_source()
-         project_url=re.search("(projectPageUrl\s:\s\")(.*)(\",)",rc)
+         project_url=re.search(r"(projectPageUrl\s:\s\")(.*)(\",)",rc)
          return project_url.group(2)
 
 

bitbake/lib/toaster/tests/functional/test_create_new_project.py

@@ -0,0 +1,179 @@
+#! /usr/bin/env python3
+# BitBake Toaster UI tests implementation
+#
+# Copyright (C) 2023 Savoir-faire Linux
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import re
+import pytest
+from django.urls import reverse
+from selenium.webdriver.support.select import Select
+from tests.functional.functional_helpers import SeleniumFunctionalTestCase
+from orm.models import Project
+from selenium.webdriver.common.by import By
+
+
+@pytest.mark.django_db
+@pytest.mark.order("last")
+class TestCreateNewProject(SeleniumFunctionalTestCase):
+
+    def _create_test_new_project(
+        self,
+        project_name,
+        release,
+        release_title,
+        merge_toaster_settings,
+    ):
+        """ Create/Test new project using:
+          - Project Name: Any string
+          - Release: Any string
+          - Merge Toaster settings: True or False
+        """
+        self.get(reverse('newproject'))
+        self.wait_until_visible('#new-project-name', poll=3)
+        self.driver.find_element(By.ID,
+                                 "new-project-name").send_keys(project_name)
+
+        select = Select(self.find('#projectversion'))
+        select.select_by_value(release)
+
+        # check merge toaster settings
+        checkbox = self.find('.checkbox-mergeattr')
+        if merge_toaster_settings:
+            if not checkbox.is_selected():
+                checkbox.click()
+        else:
+            if checkbox.is_selected():
+                checkbox.click()
+
+        self.driver.find_element(By.ID, "create-project-button").click()
+
+        element = self.wait_until_visible('#project-created-notification', poll=3)
+        self.assertTrue(
+            self.element_exists('#project-created-notification'),
+            f"Project:{project_name} creation notification not shown"
+        )
+        self.assertTrue(
+            project_name in element.text,
+            f"New project name:{project_name} not in new project notification"
+        )
+        self.assertTrue(
+            Project.objects.filter(name=project_name).count(),
+            f"New project:{project_name} not found in database"
+        )
+
+        # check release
+        self.assertTrue(re.search(
+            release_title,
+            self.driver.find_element(By.XPATH,
+                                     "//span[@id='project-release-title']"
+                                     ).text),
+                        'The project release is not defined')
+
+    def test_create_new_project_master(self):
+        """ Test create new project using:
+          - Project Name: Any string
+          - Release: Yocto Project master (option value: 3)
+          - Merge Toaster settings: False
+        """
+        release = '3'
+        release_title = 'Yocto Project master'
+        project_name = 'projectmaster'
+        self._create_test_new_project(
+            project_name,
+            release,
+            release_title,
+            False,
+        )
+
+    def test_create_new_project_kirkstone(self):
+        """ Test create new project using:
+          - Project Name: Any string
+          - Release: Yocto Project 4.0 "Kirkstone" (option value: 1)
+          - Merge Toaster settings: True
+        """
+        release = '1'
+        release_title = 'Yocto Project 4.0 "Kirkstone"'
+        project_name = 'projectkirkstone'
+        self._create_test_new_project(
+            project_name,
+            release,
+            release_title,
+            True,
+        )
+
+    def test_create_new_project_dunfell(self):
+        """ Test create new project using:
+          - Project Name: Any string
+          - Release: Yocto Project 3.1 "Dunfell" (option value: 5)
+          - Merge Toaster settings: False
+        """
+        release = '5'
+        release_title = 'Yocto Project 3.1 "Dunfell"'
+        project_name = 'projectdunfell'
+        self._create_test_new_project(
+            project_name,
+            release,
+            release_title,
+            False,
+        )
+
+    def test_create_new_project_local(self):
+        """ Test create new project using:
+          - Project Name: Any string
+          - Release: Local Yocto Project (option value: 2)
+          - Merge Toaster settings: True
+        """
+        release = '2'
+        release_title = 'Local Yocto Project'
+        project_name = 'projectlocal'
+        self._create_test_new_project(
+            project_name,
+            release,
+            release_title,
+            True,
+        )
+
+    def test_create_new_project_without_name(self):
+        """ Test create new project without project name """
+        self.get(reverse('newproject'))
+
+        select = Select(self.find('#projectversion'))
+        select.select_by_value(str(3))
+
+        # Check input name has required attribute
+        input_name = self.driver.find_element(By.ID, "new-project-name")
+        self.assertIsNotNone(input_name.get_attribute('required'),
+                        'Input name has not required attribute')
+
+        # Check create button is disabled
+        create_btn = self.driver.find_element(By.ID, "create-project-button")
+        self.assertIsNotNone(create_btn.get_attribute('disabled'),
+                        'Create button is not disabled')
+
+    def test_import_new_project(self):
+        """ Test import new project using:
+          - Project Name: Any string
+          - Project type: select (Import command line project)
+          - Import existing project directory: Wrong Path
+        """
+        project_name = 'projectimport'
+        self.get(reverse('newproject'))
+        self.driver.find_element(By.ID,
+                                 "new-project-name").send_keys(project_name)
+        # select import project
+        self.find('#type-import').click()
+
+        # set wrong path
+        wrong_path = '/wrongpath'
+        self.driver.find_element(By.ID,
+                                 "import-project-dir").send_keys(wrong_path)
+        self.driver.find_element(By.ID, "create-project-button").click()
+
+        # check error message
+        self.assertTrue(self.element_exists('.alert-danger'),
+                        'Allert message not shown')
+        self.assertTrue(wrong_path in self.find('.alert-danger').text,
+                        "Wrong path not in alert message")

bitbake/lib/toaster/tests/functional/test_functional_basic.py

@@ -7,93 +7,111 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-import re, time
+import re
 from django.urls import reverse
+import pytest
 from tests.functional.functional_helpers import SeleniumFunctionalTestCase
 from orm.models import Project
 from selenium.webdriver.common.by import By
 
+from tests.functional.utils import get_projectId_from_url
+
+
+@pytest.mark.django_db
+@pytest.mark.order("second_to_last")
 class FuntionalTestBasic(SeleniumFunctionalTestCase):
+    """Basic functional tests for Toaster"""
+    project_id = None
+
+    def setUp(self):
+        super(FuntionalTestBasic, self).setUp()
+        if not FuntionalTestBasic.project_id:
+            self._create_slenium_project()
+            current_url = self.driver.current_url
+            FuntionalTestBasic.project_id = get_projectId_from_url(current_url)
 
 #   testcase (1514)
-    def test_create_slenium_project(self):
+    def _create_slenium_project(self):
         project_name = 'selenium-project'
         self.get(reverse('newproject'))
+        self.wait_until_visible('#new-project-name', poll=3)
         self.driver.find_element(By.ID, "new-project-name").send_keys(project_name)
         self.driver.find_element(By.ID, 'projectversion').click()
         self.driver.find_element(By.ID, "create-project-button").click()
-        time.sleep(2)
-        element = self.wait_until_visible('#project-created-notification')
+        element = self.wait_until_visible('#project-created-notification', poll=10)
         self.assertTrue(self.element_exists('#project-created-notification'),'Project creation notification not shown')
         self.assertTrue(project_name in element.text,
                         "New project name not in new project notification")
         self.assertTrue(Project.objects.filter(name=project_name).count(),
                         "New project not found in database")
+        return Project.objects.last().id
 
  #  testcase (1515)
     def test_verify_left_bar_menu(self):
         self.get(reverse('all-projects'))
-        self.wait_until_visible('#projectstable')
+        self.wait_until_present('#projectstable', poll=10)
         self.find_element_by_link_text_in_table('projectstable', 'selenium-project').click()
-        time.sleep(2)
+        self.wait_until_present('#config-nav', poll=10)
         self.assertTrue(self.element_exists('#config-nav'),'Configuration Tab does not exist')
         project_URL=self.get_URL()
         self.driver.find_element(By.XPATH, '//a[@href="'+project_URL+'"]').click()
-        time.sleep(2)
+        self.wait_until_present('#config-nav', poll=10)
 
         try:
             self.driver.find_element(By.XPATH, "//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'customimages/"'+"]").click()
-            time.sleep(2)
+            self.wait_until_present('#config-nav', poll=10)
             self.assertTrue(re.search("Custom images",self.driver.find_element(By.XPATH, "//div[@class='col-md-10']").text),'Custom images information is not loading properly')
         except:
             self.fail(msg='No Custom images tab available')
 
         try:
             self.driver.find_element(By.XPATH, "//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'images/"'+"]").click()
+            self.wait_until_present('#config-nav', poll=10)
             self.assertTrue(re.search("Compatible image recipes",self.driver.find_element(By.XPATH, "//div[@class='col-md-10']").text),'The Compatible image recipes information is not loading properly')
         except:
             self.fail(msg='No Compatible image tab available')
 
         try:
             self.driver.find_element(By.XPATH, "//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'softwarerecipes/"'+"]").click()
+            self.wait_until_present('#config-nav', poll=10)
             self.assertTrue(re.search("Compatible software recipes",self.driver.find_element(By.XPATH, "//div[@class='col-md-10']").text),'The Compatible software recipe information is not loading properly')
         except:
             self.fail(msg='No Compatible software recipe tab available')
 
         try:
             self.driver.find_element(By.XPATH, "//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'machines/"'+"]").click()
+            self.wait_until_present('#config-nav', poll=10)
             self.assertTrue(re.search("Compatible machines",self.driver.find_element(By.XPATH, "//div[@class='col-md-10']").text),'The Compatible machine information is not loading properly')
         except:
             self.fail(msg='No Compatible machines tab available')
 
         try:
             self.driver.find_element(By.XPATH, "//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'layers/"'+"]").click()
+            self.wait_until_present('#config-nav', poll=10)
             self.assertTrue(re.search("Compatible layers",self.driver.find_element(By.XPATH, "//div[@class='col-md-10']").text),'The Compatible layer information is not loading properly')
         except:
             self.fail(msg='No Compatible layers tab available')
 
         try:
             self.driver.find_element(By.XPATH, "//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'configuration"'+"]").click()
+            self.wait_until_present('#config-nav', poll=10)
             self.assertTrue(re.search("Bitbake variables",self.driver.find_element(By.XPATH, "//div[@class='col-md-10']").text),'The Bitbake variables information is not loading properly')
         except:
             self.fail(msg='No Bitbake variables tab available')
 
 #   testcase (1516)
     def test_review_configuration_information(self):
-        self.get('')
-        self.driver.find_element(By.XPATH, "//div[@id='global-nav']/ul/li/a[@href="+'"'+'/toastergui/projects/'+'"'+"]").click()
-        time.sleep(2)
-        self.wait_until_visible('#projectstable')
+        self.get(reverse('all-projects'))
+        self.wait_until_present('#projectstable', poll=10)
         self.find_element_by_link_text_in_table('projectstable', 'selenium-project').click()
         project_URL=self.get_URL()
-        time.sleep(2)
+        self.wait_until_present('#config-nav', poll=10)
         try:
            self.assertTrue(self.element_exists('#machine-section'),'Machine section for the project configuration page does not exist')
-           self.assertTrue(re.search("qemux86",self.driver.find_element(By.XPATH, "//span[@id='project-machine-name']").text),'The machine type is not assigned')
+           self.assertTrue(re.search("qemux86-64",self.driver.find_element(By.XPATH, "//span[@id='project-machine-name']").text),'The machine type is not assigned')
            self.driver.find_element(By.XPATH, "//span[@id='change-machine-toggle']").click()
-           time.sleep(2)
-           self.wait_until_visible('#select-machine-form')
-           self.wait_until_visible('#cancel-machine-change')
+           self.wait_until_visible('#select-machine-form', poll=10)
+           self.wait_until_visible('#cancel-machine-change', poll=10)
            self.driver.find_element(By.XPATH, "//form[@id='select-machine-form']/a[@id='cancel-machine-change']").click()
         except:
            self.fail(msg='The machine information is wrong in the configuration page')
@@ -127,49 +145,42 @@ class FuntionalTestBasic(SeleniumFunctionalTestCase):
 
 #   testcase (1517)
     def test_verify_machine_information(self):
-        self.get('')
-        self.driver.find_element(By.XPATH, "//div[@id='global-nav']/ul/li/a[@href="+'"'+'/toastergui/projects/'+'"'+"]").click()
-        time.sleep(2)
-        self.wait_until_visible('#projectstable')
+        self.get(reverse('all-projects'))
+        self.wait_until_present('#projectstable', poll=10)
         self.find_element_by_link_text_in_table('projectstable', 'selenium-project').click()
-        time.sleep(2)
+        self.wait_until_present('#config-nav', poll=10)
 
         try:
             self.assertTrue(self.element_exists('#machine-section'),'Machine section for the project configuration page does not exist')
-            self.assertTrue(re.search("qemux86",self.driver.find_element(By.ID, "project-machine-name").text),'The machine type is not assigned')
+            self.assertTrue(re.search("qemux86-64",self.driver.find_element(By.ID, "project-machine-name").text),'The machine type is not assigned')
             self.driver.find_element(By.ID, "change-machine-toggle").click()
-            time.sleep(2)
-            self.wait_until_visible('#select-machine-form')
-            self.wait_until_visible('#cancel-machine-change')
+            self.wait_until_visible('#select-machine-form', poll=10)
+            self.wait_until_visible('#cancel-machine-change', poll=10)
             self.driver.find_element(By.ID, "cancel-machine-change").click()
         except:
             self.fail(msg='The machine information is wrong in the configuration page')
 
 #   testcase (1518)
     def test_verify_most_built_recipes_information(self):
-        self.get('')
-        self.driver.find_element(By.XPATH, "//div[@id='global-nav']/ul/li/a[@href="+'"'+'/toastergui/projects/'+'"'+"]").click()
-        time.sleep(2)
-        self.wait_until_visible('#projectstable')
+        self.get(reverse('all-projects'))
+        self.wait_until_present('#projectstable', poll=10)
         self.find_element_by_link_text_in_table('projectstable', 'selenium-project').click()
+        self.wait_until_present('#config-nav', poll=10)
         project_URL=self.get_URL()
-        time.sleep(2)
         try:
             self.assertTrue(re.search("You haven't built any recipes yet",self.driver.find_element(By.ID, "no-most-built").text),'Default message of no builds is not present')
             self.driver.find_element(By.XPATH, "//div[@id='no-most-built']/p/a[@href="+'"'+project_URL+'images/"'+"]").click()
-            time.sleep(2)
+            self.wait_until_present('#config-nav', poll=10)
             self.assertTrue(re.search("Compatible image recipes",self.driver.find_element(By.XPATH, "//div[@class='col-md-10']").text),'The Choose a recipe to build link  is not working  properly')
         except:
             self.fail(msg='No Most built information in project detail page')
 
 #   testcase (1519)
     def test_verify_project_release_information(self):
-        self.get('')
-        self.driver.find_element(By.XPATH, "//div[@id='global-nav']/ul/li/a[@href="+'"'+'/toastergui/projects/'+'"'+"]").click()
-        time.sleep(2)
-        self.wait_until_visible('#projectstable')
+        self.get(reverse('all-projects'))
+        self.wait_until_present('#projectstable', poll=10)
         self.find_element_by_link_text_in_table('projectstable', 'selenium-project').click()
-        time.sleep(2)
+        self.wait_until_present('#config-nav', poll=10)
 
         try:
             self.assertTrue(re.search("Yocto Project master",self.driver.find_element(By.ID, "project-release-title").text),'The project release is not defined')
@@ -178,12 +189,11 @@ class FuntionalTestBasic(SeleniumFunctionalTestCase):
 
 #   testcase (1520)
     def test_verify_layer_information(self):
-        self.get('')
-        self.driver.find_element(By.XPATH, "//div[@id='global-nav']/ul/li/a[@href="+'"'+'/toastergui/projects/'+'"'+"]").click()
-        self.wait_until_visible('#projectstable')
+        self.get(reverse('all-projects'))
+        self.wait_until_present('#projectstable', poll=10)
         self.find_element_by_link_text_in_table('projectstable', 'selenium-project').click()
+        self.wait_until_present('#config-nav', poll=10)
         project_URL=self.get_URL()
-        time.sleep(2)
         try:
            self.driver.find_element(By.XPATH, "//div[@id='layer-container']")
            self.assertTrue(re.search("3",self.driver.find_element(By.ID, "project-layers-count").text),'There should be 3 layers listed in the layer count')
@@ -209,18 +219,18 @@ class FuntionalTestBasic(SeleniumFunctionalTestCase):
 
 #   testcase (1521)
     def test_verify_project_detail_links(self):
-        self.get('')
-        self.driver.find_element(By.XPATH, "//div[@id='global-nav']/ul/li/a[@href="+'"'+'/toastergui/projects/'+'"'+"]").click()
-        time.sleep(2)
-        self.wait_until_visible('#projectstable')
+        self.get(reverse('all-projects'))
+        self.wait_until_present('#projectstable', poll=10)
         self.find_element_by_link_text_in_table('projectstable', 'selenium-project').click()
+        self.wait_until_present('#config-nav', poll=10)
         project_URL=self.get_URL()
-        time.sleep(2)
         self.driver.find_element(By.XPATH, "//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li[@id='topbar-configuration-tab']/a[@href="+'"'+project_URL+'"'+"]").click()
+        self.wait_until_present('#config-nav', poll=10)
         self.assertTrue(re.search("Configuration",self.driver.find_element(By.XPATH, "//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li[@id='topbar-configuration-tab']/a[@href="+'"'+project_URL+'"'+"]").text), 'Configuration tab in project topbar is misspelled')
 
         try:
             self.driver.find_element(By.XPATH, "//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'builds/"'+"]").click()
+            self.wait_until_visible('#project-topbar', poll=10)
             self.assertTrue(re.search("Builds",self.driver.find_element(By.XPATH, "//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'builds/"'+"]").text), 'Builds tab in project topbar is misspelled')
             self.driver.find_element(By.XPATH, "//div[@id='empty-state-projectbuildstable']")
         except:
@@ -228,6 +238,7 @@ class FuntionalTestBasic(SeleniumFunctionalTestCase):
 
         try:
             self.driver.find_element(By.XPATH, "//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'importlayer"'+"]").click()
+            self.wait_until_visible('#project-topbar', poll=10)
             self.assertTrue(re.search("Import layer",self.driver.find_element(By.XPATH, "//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'importlayer"'+"]").text), 'Import layer tab in project topbar is misspelled')
             self.driver.find_element(By.XPATH, "//fieldset[@id='repo-select']")
             self.driver.find_element(By.XPATH, "//fieldset[@id='git-repo']")
@@ -236,6 +247,7 @@ class FuntionalTestBasic(SeleniumFunctionalTestCase):
 
         try:
             self.driver.find_element(By.XPATH, "//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'newcustomimage/"'+"]").click()
+            self.wait_until_visible('#project-topbar', poll=10)
             self.assertTrue(re.search("New custom image",self.driver.find_element(By.XPATH, "//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'newcustomimage/"'+"]").text), 'New custom image tab in project topbar is misspelled')
             self.assertTrue(re.search("Select the image recipe you want to customise",self.driver.find_element(By.XPATH, "//div[@class='col-md-12']/h2").text),'The new custom image tab is not loading correctly')
         except:

bitbake/lib/toaster/tests/functional/test_project_config.py

@@ -0,0 +1,341 @@
+#! /usr/bin/env python3 #
+# BitBake Toaster UI tests implementation
+#
+# Copyright (C) 2023 Savoir-faire Linux
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import string
+import random
+import pytest
+from django.urls import reverse
+from selenium.webdriver import Keys
+from selenium.webdriver.support.select import Select
+from selenium.common.exceptions import TimeoutException
+from tests.functional.functional_helpers import SeleniumFunctionalTestCase
+from selenium.webdriver.common.by import By
+
+from .utils import get_projectId_from_url
+
+
+@pytest.mark.django_db
+@pytest.mark.order("last")
+class TestProjectConfig(SeleniumFunctionalTestCase):
+    project_id = None
+    PROJECT_NAME = 'TestProjectConfig'
+    INVALID_PATH_START_TEXT = 'The directory path should either start with a /'
+    INVALID_PATH_CHAR_TEXT = 'The directory path cannot include spaces or ' \
+        'any of these characters'
+
+    def _create_project(self, project_name):
+        """ Create/Test new project using:
+          - Project Name: Any string
+          - Release: Any string
+          - Merge Toaster settings: True or False
+        """
+        self.get(reverse('newproject'))
+        self.wait_until_visible('#new-project-name', poll=2)
+        self.find("#new-project-name").send_keys(project_name)
+        select = Select(self.find("#projectversion"))
+        select.select_by_value('3')
+
+        # check merge toaster settings
+        checkbox = self.find('.checkbox-mergeattr')
+        if not checkbox.is_selected():
+            checkbox.click()
+
+        if self.PROJECT_NAME != 'TestProjectConfig':
+            # Reset project name if it's not the default one
+            self.PROJECT_NAME = 'TestProjectConfig'
+
+        self.find("#create-project-button").click()
+
+        try:
+            self.wait_until_visible('#hint-error-project-name', poll=2)
+            url = reverse('project', args=(TestProjectConfig.project_id, ))
+            self.get(url)
+            self.wait_until_visible('#config-nav', poll=3)
+        except TimeoutException:
+            self.wait_until_visible('#config-nav', poll=3)
+
+    def _random_string(self, length):
+        return ''.join(
+            random.choice(string.ascii_letters) for _ in range(length)
+        )
+
+    def _get_config_nav_item(self, index):
+        config_nav = self.find('#config-nav')
+        return config_nav.find_elements(By.TAG_NAME, 'li')[index]
+
+    def _navigate_bbv_page(self):
+        """ Navigate to project BitBake variables page """
+        # check if the menu is displayed
+        if TestProjectConfig.project_id is None:
+            self._create_project(project_name=self._random_string(10))
+            current_url = self.driver.current_url
+            TestProjectConfig.project_id = get_projectId_from_url(current_url)
+        else:
+            url = reverse('projectconf', args=(TestProjectConfig.project_id,))
+            self.get(url)
+        self.wait_until_visible('#config-nav', poll=3)
+        bbv_page_link = self._get_config_nav_item(9)
+        bbv_page_link.click()
+        self.wait_until_visible('#config-nav', poll=3)
+
+    def test_no_underscore_iamgefs_type(self):
+        """
+        Should not accept IMAGEFS_TYPE with an underscore
+        """
+        self._navigate_bbv_page()
+        imagefs_type = "foo_bar"
+
+        self.wait_until_visible('#change-image_fstypes-icon', poll=2)
+
+        self.click('#change-image_fstypes-icon')
+
+        self.enter_text('#new-imagefs_types', imagefs_type)
+
+        element = self.wait_until_visible('#hintError-image-fs_type', poll=2)
+
+        self.assertTrue(("A valid image type cannot include underscores" in element.text),
+                        "Did not find underscore error message")
+
+    def test_checkbox_verification(self):
+        """
+        Should automatically check the checkbox if user enters value
+        text box, if value is there in the checkbox.
+        """
+        self._navigate_bbv_page()
+
+        imagefs_type = "btrfs"
+
+        self.wait_until_visible('#change-image_fstypes-icon', poll=2)
+
+        self.click('#change-image_fstypes-icon')
+
+        self.enter_text('#new-imagefs_types', imagefs_type)
+
+        checkboxes = self.driver.find_elements(By.XPATH, "//input[@class='fs-checkbox-fstypes']")
+
+        for checkbox in checkboxes:
+            if checkbox.get_attribute("value") == "btrfs":
+               self.assertEqual(checkbox.is_selected(), True)
+
+    def test_textbox_with_checkbox_verification(self):
+        """
+        Should automatically add or remove value in textbox, if user checks
+        or unchecks checkboxes.
+        """
+        self._navigate_bbv_page()
+
+        self.wait_until_visible('#change-image_fstypes-icon', poll=2)
+
+        self.click('#change-image_fstypes-icon')
+
+        checkboxes_selector = '.fs-checkbox-fstypes'
+
+        self.wait_until_visible(checkboxes_selector, poll=2)
+        checkboxes = self.find_all(checkboxes_selector)
+
+        for checkbox in checkboxes:
+            if checkbox.get_attribute("value") == "cpio":
+               checkbox.click()
+               element = self.driver.find_element(By.ID, 'new-imagefs_types')
+
+               self.wait_until_visible('#new-imagefs_types', poll=2)
+
+               self.assertTrue(("cpio" in element.get_attribute('value'),
+                               "Imagefs not added into the textbox"))
+               checkbox.click()
+               self.assertTrue(("cpio" not in element.text),
+                               "Image still present in the textbox")
+
+    def test_set_download_dir(self):
+        """
+        Validate the allowed and disallowed types in the directory field for
+        DL_DIR
+        """
+        self._navigate_bbv_page()
+
+        # activate the input to edit download dir
+        try:
+            change_dl_dir_btn = self.wait_until_visible('#change-dl_dir-icon', poll=2)
+        except TimeoutException:
+            # If download dir is not displayed, test is skipped
+            change_dl_dir_btn = None
+
+        if change_dl_dir_btn:
+            change_dl_dir_btn = self.wait_until_visible('#change-dl_dir-icon', poll=2)
+            change_dl_dir_btn.click()
+
+            # downloads dir path doesn't start with / or ${...}
+            input_field = self.wait_until_visible('#new-dl_dir', poll=2)
+            input_field.clear()
+            self.enter_text('#new-dl_dir', 'home/foo')
+            element = self.wait_until_visible('#hintError-initialChar-dl_dir', poll=2)
+
+            msg = 'downloads directory path starts with invalid character but ' \
+                'treated as valid'
+            self.assertTrue((self.INVALID_PATH_START_TEXT in element.text), msg)
+
+            # downloads dir path has a space
+            self.driver.find_element(By.ID, 'new-dl_dir').clear()
+            self.enter_text('#new-dl_dir', '/foo/bar a')
+
+            element = self.wait_until_visible('#hintError-dl_dir', poll=2)
+            msg = 'downloads directory path characters invalid but treated as valid'
+            self.assertTrue((self.INVALID_PATH_CHAR_TEXT in element.text), msg)
+
+            # downloads dir path starts with ${...} but has a space
+            self.driver.find_element(By.ID,'new-dl_dir').clear()
+            self.enter_text('#new-dl_dir', '${TOPDIR}/down foo')
+
+            element = self.wait_until_visible('#hintError-dl_dir', poll=2)
+            msg = 'downloads directory path characters invalid but treated as valid'
+            self.assertTrue((self.INVALID_PATH_CHAR_TEXT in element.text), msg)
+
+            # downloads dir path starts with /
+            self.driver.find_element(By.ID,'new-dl_dir').clear()
+            self.enter_text('#new-dl_dir', '/bar/foo')
+
+            hidden_element = self.driver.find_element(By.ID,'hintError-dl_dir')
+            self.assertEqual(hidden_element.is_displayed(), False,
+                'downloads directory path valid but treated as invalid')
+
+            # downloads dir path starts with ${...}
+            self.driver.find_element(By.ID,'new-dl_dir').clear()
+            self.enter_text('#new-dl_dir', '${TOPDIR}/down')
+
+            hidden_element = self.driver.find_element(By.ID,'hintError-dl_dir')
+            self.assertEqual(hidden_element.is_displayed(), False,
+                'downloads directory path valid but treated as invalid')
+
+    def test_set_sstate_dir(self):
+        """
+        Validate the allowed and disallowed types in the directory field for
+        SSTATE_DIR
+        """
+        self._navigate_bbv_page()
+
+        try:
+            btn_chg_sstate_dir = self.wait_until_visible(
+                '#change-sstate_dir-icon',
+                poll=2
+            )
+            self.click('#change-sstate_dir-icon')
+        except TimeoutException:
+            # If sstate_dir is not displayed, test is skipped
+            btn_chg_sstate_dir = None
+
+        if btn_chg_sstate_dir:  # Skip continuation if sstate_dir is not displayed
+            # path doesn't start with / or ${...}
+            input_field = self.wait_until_visible('#new-sstate_dir', poll=2)
+            input_field.clear()
+            self.enter_text('#new-sstate_dir', 'home/foo')
+            element = self.wait_until_visible('#hintError-initialChar-sstate_dir', poll=2)
+
+            msg = 'sstate directory path starts with invalid character but ' \
+                'treated as valid'
+            self.assertTrue((self.INVALID_PATH_START_TEXT in element.text), msg)
+
+            # path has a space
+            self.driver.find_element(By.ID, 'new-sstate_dir').clear()
+            self.enter_text('#new-sstate_dir', '/foo/bar a')
+
+            element = self.wait_until_visible('#hintError-sstate_dir', poll=2)
+            msg = 'sstate directory path characters invalid but treated as valid'
+            self.assertTrue((self.INVALID_PATH_CHAR_TEXT in element.text), msg)
+
+            # path starts with ${...} but has a space
+            self.driver.find_element(By.ID,'new-sstate_dir').clear()
+            self.enter_text('#new-sstate_dir', '${TOPDIR}/down foo')
+
+            element = self.wait_until_visible('#hintError-sstate_dir', poll=2)
+            msg = 'sstate directory path characters invalid but treated as valid'
+            self.assertTrue((self.INVALID_PATH_CHAR_TEXT in element.text), msg)
+
+            # path starts with /
+            self.driver.find_element(By.ID,'new-sstate_dir').clear()
+            self.enter_text('#new-sstate_dir', '/bar/foo')
+
+            hidden_element = self.driver.find_element(By.ID, 'hintError-sstate_dir')
+            self.assertEqual(hidden_element.is_displayed(), False,
+                'sstate directory path valid but treated as invalid')
+
+            # paths starts with ${...}
+            self.driver.find_element(By.ID, 'new-sstate_dir').clear()
+            self.enter_text('#new-sstate_dir', '${TOPDIR}/down')
+
+            hidden_element = self.driver.find_element(By.ID, 'hintError-sstate_dir')
+            self.assertEqual(hidden_element.is_displayed(), False,
+                'sstate directory path valid but treated as invalid')
+
+    def _change_bbv_value(self, **kwargs):
+        var_name, field, btn_id, input_id, value, save_btn, *_ = kwargs.values()
+        """ Change bitbake variable value """
+        self._navigate_bbv_page()
+        self.wait_until_visible(f'#{btn_id}', poll=2)
+        if kwargs.get('new_variable'):
+            self.find(f"#{btn_id}").clear()
+            self.enter_text(f"#{btn_id}", f"{var_name}")
+        else:
+            self.click(f'#{btn_id}')
+            self.wait_until_visible(f'#{input_id}', poll=2)
+
+        if kwargs.get('is_select'):
+            select = Select(self.find(f'#{input_id}'))
+            select.select_by_visible_text(value)
+        else:
+            self.find(f"#{input_id}").clear()
+            self.enter_text(f'#{input_id}', f'{value}')
+        self.click(f'#{save_btn}')
+        value_displayed = str(self.wait_until_visible(f'#{field}').text).lower()
+        msg = f'{var_name} variable not changed'
+        self.assertTrue(str(value).lower() in value_displayed, msg)
+
+    def test_change_distro_var(self):
+        """ Test changing distro variable """
+        self._change_bbv_value(
+            var_name='DISTRO',
+            field='distro',
+            btn_id='change-distro-icon',
+            input_id='new-distro',
+            value='poky-changed',
+            save_btn="apply-change-distro",
+        )
+
+    def test_set_image_install_append_var(self):
+        """ Test setting IMAGE_INSTALL:append variable """
+        self._change_bbv_value(
+            var_name='IMAGE_INSTALL:append',
+            field='image_install',
+            btn_id='change-image_install-icon',
+            input_id='new-image_install',
+            value='bash, apt, busybox',
+            save_btn="apply-change-image_install",
+        )
+
+    def test_set_package_classes_var(self):
+        """ Test setting PACKAGE_CLASSES variable """
+        self._change_bbv_value(
+            var_name='PACKAGE_CLASSES',
+            field='package_classes',
+            btn_id='change-package_classes-icon',
+            input_id='package_classes-select',
+            value='package_deb',
+            save_btn="apply-change-package_classes",
+            is_select=True,
+        )
+
+    def test_create_new_bbv(self):
+        """ Test creating new bitbake variable """
+        self._change_bbv_value(
+            var_name='New_Custom_Variable',
+            field='configvar-list',
+            btn_id='variable',
+            input_id='value',
+            value='new variable value',
+            save_btn="add-configvar-button",
+            new_variable=True
+        )

bitbake/lib/toaster/tests/functional/test_project_page.py

@@ -0,0 +1,792 @@
+#! /usr/bin/env python3 #
+# BitBake Toaster UI tests implementation
+#
+# Copyright (C) 2023 Savoir-faire Linux
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import os
+import random
+import string
+from unittest import skip
+import pytest
+from django.urls import reverse
+from django.utils import timezone
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.support.select import Select
+from selenium.common.exceptions import TimeoutException
+from tests.functional.functional_helpers import SeleniumFunctionalTestCase
+from orm.models import Build, Project, Target
+from selenium.webdriver.common.by import By
+
+from .utils import get_projectId_from_url, wait_until_build, wait_until_build_cancelled
+
+
+@pytest.mark.django_db
+@pytest.mark.order("last")
+class TestProjectPage(SeleniumFunctionalTestCase):
+    project_id = None
+    PROJECT_NAME = 'TestProjectPage'
+
+    def _create_project(self, project_name):
+        """ Create/Test new project using:
+          - Project Name: Any string
+          - Release: Any string
+          - Merge Toaster settings: True or False
+        """
+        self.get(reverse('newproject'))
+        self.wait_until_visible('#new-project-name')
+        self.find("#new-project-name").send_keys(project_name)
+        select = Select(self.find("#projectversion"))
+        select.select_by_value('3')
+
+        # check merge toaster settings
+        checkbox = self.find('.checkbox-mergeattr')
+        if not checkbox.is_selected():
+            checkbox.click()
+
+        if self.PROJECT_NAME != 'TestProjectPage':
+            # Reset project name if it's not the default one
+            self.PROJECT_NAME = 'TestProjectPage'
+
+        self.find("#create-project-button").click()
+
+        try:
+            self.wait_until_visible('#hint-error-project-name')
+            url = reverse('project', args=(TestProjectPage.project_id, ))
+            self.get(url)
+            self.wait_until_visible('#config-nav', poll=3)
+        except TimeoutException:
+            self.wait_until_visible('#config-nav', poll=3)
+
+    def _random_string(self, length):
+        return ''.join(
+            random.choice(string.ascii_letters) for _ in range(length)
+        )
+
+    def _navigate_to_project_page(self):
+        # Navigate to project page
+        if TestProjectPage.project_id is None:
+            self._create_project(project_name=self._random_string(10))
+            current_url = self.driver.current_url
+            TestProjectPage.project_id = get_projectId_from_url(current_url)
+        else:
+            url = reverse('project', args=(TestProjectPage.project_id,))
+            self.get(url)
+        self.wait_until_visible('#config-nav')
+
+    def _get_create_builds(self, **kwargs):
+        """ Create a build and return the build object """
+        # parameters for builds to associate with the projects
+        now = timezone.now()
+        self.project1_build_success = {
+            'project': Project.objects.get(id=TestProjectPage.project_id),
+            'started_on': now,
+            'completed_on': now,
+            'outcome': Build.SUCCEEDED
+        }
+
+        self.project1_build_failure = {
+            'project': Project.objects.get(id=TestProjectPage.project_id),
+            'started_on': now,
+            'completed_on': now,
+            'outcome': Build.FAILED
+        }
+        build1 = Build.objects.create(**self.project1_build_success)
+        build2 = Build.objects.create(**self.project1_build_failure)
+
+        # add some targets to these builds so they have recipe links
+        # (and so we can find the row in the ToasterTable corresponding to
+        # a particular build)
+        Target.objects.create(build=build1, target='foo')
+        Target.objects.create(build=build2, target='bar')
+
+        if kwargs:
+            # Create kwargs.get('success') builds with success status with target
+            # and kwargs.get('failure') builds with failure status with target
+            for i in range(kwargs.get('success', 0)):
+                now = timezone.now()
+                self.project1_build_success['started_on'] = now
+                self.project1_build_success[
+                    'completed_on'] = now - timezone.timedelta(days=i)
+                build = Build.objects.create(**self.project1_build_success)
+                Target.objects.create(build=build,
+                                      target=f'{i}_success_recipe',
+                                      task=f'{i}_success_task')
+
+            for i in range(kwargs.get('failure', 0)):
+                now = timezone.now()
+                self.project1_build_failure['started_on'] = now
+                self.project1_build_failure[
+                    'completed_on'] = now - timezone.timedelta(days=i)
+                build = Build.objects.create(**self.project1_build_failure)
+                Target.objects.create(build=build,
+                                      target=f'{i}_fail_recipe',
+                                      task=f'{i}_fail_task')
+        return build1, build2
+
+    def _mixin_test_table_edit_column(
+            self,
+            table_id,
+            edit_btn_id,
+            list_check_box_id: list
+    ):
+        # Check edit column
+        edit_column = self.find(f'#{edit_btn_id}')
+        self.assertTrue(edit_column.is_displayed())
+        edit_column.click()
+        # Check dropdown is visible
+        self.wait_until_visible('ul.dropdown-menu.editcol')
+        for check_box_id in list_check_box_id:
+            # Check that we can hide/show table column
+            check_box = self.find(f'#{check_box_id}')
+            th_class = str(check_box_id).replace('checkbox-', '')
+            if check_box.is_selected():
+                # check if column is visible in table
+                self.assertTrue(
+                    self.find(
+                        f'#{table_id} thead th.{th_class}'
+                    ).is_displayed(),
+                    f"The {th_class} column is checked in EditColumn dropdown, but it's not visible in table"
+                )
+                check_box.click()
+                # check if column is hidden in table
+                self.assertFalse(
+                    self.find(
+                        f'#{table_id} thead th.{th_class}'
+                    ).is_displayed(),
+                    f"The {th_class} column is unchecked in EditColumn dropdown, but it's visible in table"
+                )
+            else:
+                # check if column is hidden in table
+                self.assertFalse(
+                    self.find(
+                        f'#{table_id} thead th.{th_class}'
+                    ).is_displayed(),
+                    f"The {th_class} column is unchecked in EditColumn dropdown, but it's visible in table"
+                )
+                check_box.click()
+                # check if column is visible in table
+                self.assertTrue(
+                    self.find(
+                        f'#{table_id} thead th.{th_class}'
+                    ).is_displayed(),
+                    f"The {th_class} column is checked in EditColumn dropdown, but it's not visible in table"
+                )
+
+    def _get_config_nav_item(self, index):
+        config_nav = self.find('#config-nav')
+        return config_nav.find_elements(By.TAG_NAME, 'li')[index]
+
+    def _navigate_to_config_nav(self, nav_id, nav_index):
+        # navigate to the project page
+        self._navigate_to_project_page()
+        # click on "Software recipe" tab
+        soft_recipe = self._get_config_nav_item(nav_index)
+        soft_recipe.click()
+        self.wait_until_visible(f'#{nav_id}')
+
+    def _mixin_test_table_show_rows(self, table_selector, **kwargs):
+        """ Test the show rows feature in the builds table on the all builds page """
+        def test_show_rows(row_to_show, show_row_link):
+            # Check that we can show rows == row_to_show
+            show_row_link.select_by_value(str(row_to_show))
+            self.wait_until_visible(f'#{table_selector} tbody tr', poll=3)
+            # check at least some rows are visible
+            self.assertTrue(
+                len(self.find_all(f'#{table_selector} tbody tr')) > 0
+            )
+        self.wait_until_present(f'#{table_selector} tbody tr')
+        show_rows = self.driver.find_elements(
+            By.XPATH,
+            f'//select[@class="form-control pagesize-{table_selector}"]'
+        )
+        rows_to_show = [10, 25, 50, 100, 150]
+        to_skip = kwargs.get('to_skip', [])
+        # Check show rows
+        for show_row_link in show_rows:
+            show_row_link = Select(show_row_link)
+            for row_to_show in rows_to_show:
+                if row_to_show not in to_skip:
+                    test_show_rows(row_to_show, show_row_link)
+
+    def _mixin_test_table_search_input(self, **kwargs):
+        input_selector, input_text, searchBtn_selector, table_selector, *_ = kwargs.values()
+        # Test search input
+        self.wait_until_visible(f'#{input_selector}')
+        recipe_input = self.find(f'#{input_selector}')
+        recipe_input.send_keys(input_text)
+        self.find(f'#{searchBtn_selector}').click()
+        self.wait_until_visible(f'#{table_selector} tbody tr')
+        rows = self.find_all(f'#{table_selector} tbody tr')
+        self.assertTrue(len(rows) > 0)
+
+    def test_create_project(self):
+        """ Create/Test new project using:
+          - Project Name: Any string
+          - Release: Any string
+          - Merge Toaster settings: True or False
+        """
+        self._create_project(project_name=self.PROJECT_NAME)
+
+    def test_image_recipe_editColumn(self):
+        """ Test the edit column feature in image recipe table on project page """
+        self._get_create_builds(success=10, failure=10)
+
+        url = reverse('projectimagerecipes', args=(TestProjectPage.project_id,))
+        self.get(url)
+        self.wait_until_present('#imagerecipestable tbody tr')
+
+        column_list = [
+            'get_description_or_summary', 'layer_version__get_vcs_reference',
+            'layer_version__layer__name', 'license', 'recipe-file', 'section',
+            'version'
+        ]
+
+        # Check that we can hide the edit column
+        self._mixin_test_table_edit_column(
+            'imagerecipestable',
+            'edit-columns-button',
+            [f'checkbox-{column}' for column in column_list]
+        )
+
+    def test_page_header_on_project_page(self):
+        """ Check page header in project page:
+          - AT LEFT -> Logo of Yocto project, displayed, clickable
+          - "Toaster"+" Information icon", displayed, clickable
+          - "Server Icon" + "All builds", displayed, clickable
+          - "Directory Icon" + "All projects", displayed, clickable
+          - "Book Icon" + "Documentation", displayed, clickable
+          - AT RIGHT -> button "New project", displayed, clickable
+        """
+        # navigate to the project page
+        self._navigate_to_project_page()
+
+        # check page header
+        # AT LEFT -> Logo of Yocto project
+        logo = self.driver.find_element(
+            By.XPATH,
+            "//div[@class='toaster-navbar-brand']",
+        )
+        logo_img = logo.find_element(By.TAG_NAME, 'img')
+        self.assertTrue(logo_img.is_displayed(),
+                        'Logo of Yocto project not found')
+        self.assertTrue(
+            '/static/img/logo.png' in str(logo_img.get_attribute('src')),
+            'Logo of Yocto project not found'
+        )
+        # "Toaster"+" Information icon", clickable
+        toaster = self.driver.find_element(
+            By.XPATH,
+            "//div[@class='toaster-navbar-brand']//a[@class='brand']",
+        )
+        self.assertTrue(toaster.is_displayed(), 'Toaster not found')
+        self.assertTrue(toaster.text == 'Toaster')
+        info_sign = self.find('.glyphicon-info-sign')
+        self.assertTrue(info_sign.is_displayed())
+
+        # "Server Icon" + "All builds"
+        all_builds = self.find('#navbar-all-builds')
+        all_builds_link = all_builds.find_element(By.TAG_NAME, 'a')
+        self.assertTrue("All builds" in all_builds_link.text)
+        self.assertTrue(
+            '/toastergui/builds/' in str(all_builds_link.get_attribute('href'))
+        )
+        server_icon = all_builds.find_element(By.TAG_NAME, 'i')
+        self.assertTrue(
+            server_icon.get_attribute('class') == 'glyphicon glyphicon-tasks'
+        )
+        self.assertTrue(server_icon.is_displayed())
+
+        # "Directory Icon" + "All projects"
+        all_projects = self.find('#navbar-all-projects')
+        all_projects_link = all_projects.find_element(By.TAG_NAME, 'a')
+        self.assertTrue("All projects" in all_projects_link.text)
+        self.assertTrue(
+            '/toastergui/projects/' in str(all_projects_link.get_attribute(
+                'href'))
+        )
+        dir_icon = all_projects.find_element(By.TAG_NAME, 'i')
+        self.assertTrue(
+            dir_icon.get_attribute('class') == 'icon-folder-open'
+        )
+        self.assertTrue(dir_icon.is_displayed())
+
+        # "Book Icon" + "Documentation"
+        toaster_docs_link = self.find('#navbar-docs')
+        toaster_docs_link_link = toaster_docs_link.find_element(By.TAG_NAME,
+                                                                'a')
+        self.assertTrue("Documentation" in toaster_docs_link_link.text)
+        self.assertTrue(
+            toaster_docs_link_link.get_attribute('href') == 'http://docs.yoctoproject.org/toaster-manual/index.html#toaster-user-manual'
+        )
+        book_icon = toaster_docs_link.find_element(By.TAG_NAME, 'i')
+        self.assertTrue(
+            book_icon.get_attribute('class') == 'glyphicon glyphicon-book'
+        )
+        self.assertTrue(book_icon.is_displayed())
+
+        # AT RIGHT -> button "New project"
+        new_project_button = self.find('#new-project-button')
+        self.assertTrue(new_project_button.is_displayed())
+        self.assertTrue(new_project_button.text == 'New project')
+        new_project_button.click()
+        self.assertTrue(
+            '/toastergui/newproject/' in str(self.driver.current_url)
+        )
+
+    def test_edit_project_name(self):
+        """ Test edit project name:
+          - Click on "Edit" icon button
+          - Change project name
+          - Click on "Save" button
+          - Check project name is changed
+        """
+        # navigate to the project page
+        self._navigate_to_project_page()
+
+        # click on "Edit" icon button
+        self.wait_until_visible('#project-name-container')
+        edit_button = self.find('#project-change-form-toggle')
+        edit_button.click()
+        project_name_input = self.find('#project-name-change-input')
+        self.assertTrue(project_name_input.is_displayed())
+        project_name_input.clear()
+        project_name_input.send_keys('New Name')
+        self.find('#project-name-change-btn').click()
+
+        # check project name is changed
+        self.wait_until_visible('#project-name-container')
+        self.assertTrue(
+            'New Name' in str(self.find('#project-name-container').text)
+        )
+
+    def test_project_page_tabs(self):
+        """ Test project tabs:
+          - "configuration" tab
+          - "Builds" tab
+          - "Import layers" tab
+          - "New custom image" tab
+          Check search box used to build recipes
+        """
+        # navigate to the project page
+        self._navigate_to_project_page()
+
+        # check "configuration" tab
+        self.wait_until_visible('#topbar-configuration-tab')
+        config_tab = self.find('#topbar-configuration-tab')
+        self.assertTrue(config_tab.get_attribute('class') == 'active')
+        self.assertTrue('Configuration' in str(config_tab.text))
+        self.assertTrue(
+            f"/toastergui/project/{TestProjectPage.project_id}" in str(self.driver.current_url)
+        )
+
+        def get_tabs():
+            # tabs links list
+            return self.driver.find_elements(
+                By.XPATH,
+                '//div[@id="project-topbar"]//li'
+            )
+
+        def check_tab_link(tab_index, tab_name, url):
+            tab = get_tabs()[tab_index]
+            tab_link = tab.find_element(By.TAG_NAME, 'a')
+            self.assertTrue(url in tab_link.get_attribute('href'))
+            self.assertTrue(tab_name in tab_link.text)
+            self.assertTrue(tab.get_attribute('class') == 'active')
+
+        # check "Builds" tab
+        builds_tab = get_tabs()[1]
+        builds_tab.find_element(By.TAG_NAME, 'a').click()
+        check_tab_link(
+            1,
+            'Builds',
+            f"/toastergui/project/{TestProjectPage.project_id}/builds"
+        )
+
+        # check "Import layers" tab
+        import_layers_tab = get_tabs()[2]
+        import_layers_tab.find_element(By.TAG_NAME, 'a').click()
+        check_tab_link(
+            2,
+            'Import layer',
+            f"/toastergui/project/{TestProjectPage.project_id}/importlayer"
+        )
+
+        # check "New custom image" tab
+        new_custom_image_tab = get_tabs()[3]
+        new_custom_image_tab.find_element(By.TAG_NAME, 'a').click()
+        check_tab_link(
+            3,
+            'New custom image',
+            f"/toastergui/project/{TestProjectPage.project_id}/newcustomimage"
+        )
+
+        # check search box can be use to build recipes
+        search_box = self.find('#build-input')
+        search_box.send_keys('core-image-minimal')
+        self.find('#build-button').click()
+        self.wait_until_visible('#latest-builds')
+        lastest_builds = self.driver.find_elements(
+            By.XPATH,
+            '//div[@id="latest-builds"]',
+        )
+        last_build = lastest_builds[0]
+        self.assertTrue(
+            'core-image-minimal' in str(last_build.text)
+        )
+
+    def test_softwareRecipe_page(self):
+        """ Test software recipe page
+            - Check title "Compatible software recipes" is displayed
+            - Check search input
+            - Check "build recipe" button works
+            - Check software recipe table feature(show/hide column, pagination)
+        """
+        self._navigate_to_config_nav('softwarerecipestable', 4)
+        # check title "Compatible software recipes" is displayed
+        self.assertTrue("Compatible software recipes" in self.get_page_source())
+        # Test search input
+        self._mixin_test_table_search_input(
+            input_selector='search-input-softwarerecipestable',
+            input_text='busybox',
+            searchBtn_selector='search-submit-softwarerecipestable',
+            table_selector='softwarerecipestable'
+        )
+        # check "build recipe" button works
+        rows = self.find_all('#softwarerecipestable tbody tr')
+        image_to_build = rows[0]
+        build_btn = image_to_build.find_element(
+            By.XPATH,
+            '//td[@class="add-del-layers"]//a[1]'
+        )
+        build_btn.click()
+        build_state = wait_until_build(self, 'queued cloning starting parsing failed')
+        lastest_builds = self.driver.find_elements(
+            By.XPATH,
+            '//div[@id="latest-builds"]/div'
+        )
+        self.assertTrue(len(lastest_builds) > 0)
+        last_build = lastest_builds[0]
+        cancel_button = last_build.find_element(
+            By.XPATH,
+            '//span[@class="cancel-build-btn pull-right alert-link"]',
+        )
+        cancel_button.click()
+        if 'starting' not in build_state:  # change build state when cancelled in starting state
+            wait_until_build_cancelled(self)
+
+        # check software recipe table feature(show/hide column, pagination)
+        self._navigate_to_config_nav('softwarerecipestable', 4)
+        column_list = [
+            'get_description_or_summary',
+            'layer_version__get_vcs_reference',
+            'layer_version__layer__name',
+            'license',
+            'recipe-file',
+            'section',
+            'version',
+        ]
+        self._mixin_test_table_edit_column(
+            'softwarerecipestable',
+            'edit-columns-button',
+            [f'checkbox-{column}' for column in column_list]
+        )
+        self._navigate_to_config_nav('softwarerecipestable', 4)
+        # check show rows(pagination)
+        self._mixin_test_table_show_rows(
+            table_selector='softwarerecipestable',
+            to_skip=[150],
+        )
+
+    def test_machines_page(self):
+        """ Test Machine page
+            - Check if title "Compatible machines" is displayed
+            - Check search input
+            - Check "Select machine" button works
+            - Check "Add layer" button works
+            - Check Machine table feature(show/hide column, pagination)
+        """
+        self._navigate_to_config_nav('machinestable', 5)
+        # check title "Compatible software recipes" is displayed
+        self.assertTrue("Compatible machines" in self.get_page_source())
+        # Test search input
+        self._mixin_test_table_search_input(
+            input_selector='search-input-machinestable',
+            input_text='qemux86-64',
+            searchBtn_selector='search-submit-machinestable',
+            table_selector='machinestable'
+        )
+        # check "Select machine" button works
+        rows = self.find_all('#machinestable tbody tr')
+        machine_to_select = rows[0]
+        select_btn = machine_to_select.find_element(
+            By.XPATH,
+            '//td[@class="add-del-layers"]//a[1]'
+        )
+        select_btn.send_keys(Keys.RETURN)
+        self.wait_until_visible('#config-nav')
+        project_machine_name = self.find('#project-machine-name')
+        self.assertTrue(
+            'qemux86-64' in project_machine_name.text
+        )
+        # check "Add layer" button works
+        self._navigate_to_config_nav('machinestable', 5)
+        # Search for a machine whit layer not in project
+        self._mixin_test_table_search_input(
+            input_selector='search-input-machinestable',
+            input_text='qemux86-64-tpm2',
+            searchBtn_selector='search-submit-machinestable',
+            table_selector='machinestable'
+        )
+        self.wait_until_visible('#machinestable tbody tr', poll=3)
+        rows = self.find_all('#machinestable tbody tr')
+        machine_to_add = rows[0]
+        add_btn = machine_to_add.find_element(By.XPATH, '//td[@class="add-del-layers"]')
+        add_btn.click()
+        self.wait_until_visible('#change-notification')
+        change_notification = self.find('#change-notification')
+        self.assertTrue(
+            f'You have added 1 layer to your project' in str(change_notification.text)
+        )
+        # check Machine table feature(show/hide column, pagination)
+        self._navigate_to_config_nav('machinestable', 5)
+        column_list = [
+            'description',
+            'layer_version__get_vcs_reference',
+            'layer_version__layer__name',
+            'machinefile',
+        ]
+        self._mixin_test_table_edit_column(
+            'machinestable',
+            'edit-columns-button',
+            [f'checkbox-{column}' for column in column_list]
+        )
+        self._navigate_to_config_nav('machinestable', 5)
+        # check show rows(pagination)
+        self._mixin_test_table_show_rows(
+            table_selector='machinestable',
+            to_skip=[150],
+        )
+
+    def test_layers_page(self):
+        """ Test layers page
+            - Check if title "Compatible layerss" is displayed
+            - Check search input
+            - Check "Add layer" button works
+            - Check "Remove layer" button works
+            - Check layers table feature(show/hide column, pagination)
+        """
+        self._navigate_to_config_nav('layerstable', 6)
+        # check title "Compatible layers" is displayed
+        self.assertTrue("Compatible layers" in self.get_page_source())
+        # Test search input
+        input_text='meta-tanowrt'
+        self._mixin_test_table_search_input(
+            input_selector='search-input-layerstable',
+            input_text=input_text,
+            searchBtn_selector='search-submit-layerstable',
+            table_selector='layerstable'
+        )
+        # check "Add layer" button works
+        self.wait_until_visible('#layerstable tbody tr', poll=3)
+        rows = self.find_all('#layerstable tbody tr')
+        layer_to_add = rows[0]
+        add_btn = layer_to_add.find_element(
+            By.XPATH,
+            '//td[@class="add-del-layers"]'
+        )
+        add_btn.click()
+        # check modal is displayed
+        self.wait_until_visible('#dependencies-modal', poll=3)
+        list_dependencies = self.find_all('#dependencies-list li')
+        # click on add-layers button
+        add_layers_btn = self.driver.find_element(
+            By.XPATH,
+            '//form[@id="dependencies-modal-form"]//button[@class="btn btn-primary"]'
+        )
+        add_layers_btn.click()
+        self.wait_until_visible('#change-notification')
+        change_notification = self.find('#change-notification')
+        self.assertTrue(
+            f'You have added {len(list_dependencies)+1} layers to your project: {input_text} and its dependencies' in str(change_notification.text)
+        )
+        # check "Remove layer" button works
+        self.wait_until_visible('#layerstable tbody tr', poll=3)
+        rows = self.find_all('#layerstable tbody tr')
+        layer_to_remove = rows[0]
+        remove_btn = layer_to_remove.find_element(
+            By.XPATH,
+            '//td[@class="add-del-layers"]'
+        )
+        remove_btn.click()
+        self.wait_until_visible('#change-notification', poll=2)
+        change_notification = self.find('#change-notification')
+        self.assertTrue(
+            f'You have removed 1 layer from your project: {input_text}' in str(change_notification.text)
+        )
+        # check layers table feature(show/hide column, pagination)
+        self._navigate_to_config_nav('layerstable', 6)
+        column_list = [
+            'dependencies',
+            'revision',
+            'layer__vcs_url',
+            'git_subdir',
+            'layer__summary',
+        ]
+        self._mixin_test_table_edit_column(
+            'layerstable',
+            'edit-columns-button',
+            [f'checkbox-{column}' for column in column_list]
+        )
+        self._navigate_to_config_nav('layerstable', 6)
+        # check show rows(pagination)
+        self._mixin_test_table_show_rows(
+            table_selector='layerstable',
+            to_skip=[150],
+        )
+
+    def test_distro_page(self):
+        """ Test distros page
+            - Check if title "Compatible distros" is displayed
+            - Check search input
+            - Check "Add layer" button works
+            - Check distro table feature(show/hide column, pagination)
+        """
+        self._navigate_to_config_nav('distrostable', 7)
+        # check title "Compatible distros" is displayed
+        self.assertTrue("Compatible Distros" in self.get_page_source())
+        # Test search input
+        input_text='poky-altcfg'
+        self._mixin_test_table_search_input(
+            input_selector='search-input-distrostable',
+            input_text=input_text,
+            searchBtn_selector='search-submit-distrostable',
+            table_selector='distrostable'
+        )
+        # check "Add distro" button works
+        rows = self.find_all('#distrostable tbody tr')
+        distro_to_add = rows[0]
+        add_btn = distro_to_add.find_element(
+            By.XPATH,
+            '//td[@class="add-del-layers"]//a[1]'
+        )
+        add_btn.click()
+        self.wait_until_visible('#change-notification', poll=2)
+        change_notification = self.find('#change-notification')
+        self.assertTrue(
+            f'You have changed the distro to: {input_text}' in str(change_notification.text)
+        )
+        # check distro table feature(show/hide column, pagination)
+        self._navigate_to_config_nav('distrostable', 7)
+        column_list = [
+            'description',
+            'templatefile',
+            'layer_version__get_vcs_reference',
+            'layer_version__layer__name',
+        ]
+        self._mixin_test_table_edit_column(
+            'distrostable',
+            'edit-columns-button',
+            [f'checkbox-{column}' for column in column_list]
+        )
+        self._navigate_to_config_nav('distrostable', 7)
+        # check show rows(pagination)
+        self._mixin_test_table_show_rows(
+            table_selector='distrostable',
+            to_skip=[150],
+        )
+
+    def test_single_layer_page(self):
+        """ Test layer page
+            - Check if title is displayed
+            - Check add/remove layer button works
+            - Check tabs(layers, recipes, machines) are displayed
+            - Check left section is displayed
+                - Check layer name
+                - Check layer summary
+                - Check layer description
+        """
+        url = reverse("layerdetails", args=(TestProjectPage.project_id, 8))
+        self.get(url)
+        self.wait_until_visible('.page-header')
+        # check title is displayed
+        self.assertTrue(self.find('.page-header h1').is_displayed())
+
+        # check add layer button works
+        remove_layer_btn = self.find('#add-remove-layer-btn')
+        remove_layer_btn.click()
+        self.wait_until_visible('#change-notification', poll=2)
+        change_notification = self.find('#change-notification')
+        self.assertTrue(
+            f'You have removed 1 layer from your project' in str(change_notification.text)
+        )
+        # check add layer button works, 18 is the random layer id
+        add_layer_btn = self.find('#add-remove-layer-btn')
+        add_layer_btn.click()
+        self.wait_until_visible('#change-notification')
+        change_notification = self.find('#change-notification')
+        self.assertTrue(
+            f'You have added 1 layer to your project' in str(change_notification.text)
+        )
+        # check tabs(layers, recipes, machines) are displayed
+        tabs = self.find_all('.nav-tabs li')
+        self.assertEqual(len(tabs), 3)
+        # Check first tab
+        tabs[0].click()
+        self.assertTrue(
+            'active' in str(self.find('#information').get_attribute('class'))
+        )
+        # Check second tab
+        tabs[1].click()
+        self.assertTrue(
+            'active' in str(self.find('#recipes').get_attribute('class'))
+        )
+        # Check third tab
+        tabs[2].click()
+        self.assertTrue(
+            'active' in str(self.find('#machines').get_attribute('class'))
+        )
+        # Check left section is displayed
+        section = self.find('.well')
+        # Check layer name
+        self.assertTrue(
+            section.find_element(By.XPATH, '//h2[1]').is_displayed()
+        )
+        # Check layer summary
+        self.assertTrue("Summary" in section.text)
+        # Check layer description
+        self.assertTrue("Description" in section.text)
+
+    def test_single_recipe_page(self):
+        """ Test recipe page
+            - Check if title is displayed
+            - Check add recipe layer displayed
+            - Check left section is displayed
+                - Check recipe: name, summary, description, Version, Section,
+                License, Approx. packages included, Approx. size, Recipe file
+        """
+        url = reverse("recipedetails", args=(TestProjectPage.project_id, 53428))
+        self.get(url)
+        self.wait_until_visible('.page-header')
+        # check title is displayed
+        self.assertTrue(self.find('.page-header h1').is_displayed())
+        # check add recipe layer displayed
+        add_recipe_layer_btn = self.find('#add-layer-btn')
+        self.assertTrue(add_recipe_layer_btn.is_displayed())
+        # check left section is displayed
+        section = self.find('.well')
+        # Check recipe name
+        self.assertTrue(
+            section.find_element(By.XPATH, '//h2[1]').is_displayed()
+        )
+        # Check recipe sections details info are displayed
+        self.assertTrue("Summary" in section.text)
+        self.assertTrue("Description" in section.text)
+        self.assertTrue("Version" in section.text)
+        self.assertTrue("Section" in section.text)
+        self.assertTrue("License" in section.text)
+        self.assertTrue("Approx. packages included" in section.text)
+        self.assertTrue("Approx. package size" in section.text)
+        self.assertTrue("Recipe file" in section.text)

bitbake/lib/toaster/tests/functional/test_project_page_tab_config.py

@@ -0,0 +1,528 @@
+#! /usr/bin/env python3 #
+# BitBake Toaster UI tests implementation
+#
+# Copyright (C) 2023 Savoir-faire Linux
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import string
+import random
+import pytest
+from django.urls import reverse
+from selenium.webdriver import Keys
+from selenium.webdriver.support.select import Select
+from selenium.common.exceptions import ElementClickInterceptedException, NoSuchElementException, TimeoutException
+from orm.models import Project
+from tests.functional.functional_helpers import SeleniumFunctionalTestCase
+from selenium.webdriver.common.by import By
+
+from .utils import get_projectId_from_url, wait_until_build, wait_until_build_cancelled
+
+
+@pytest.mark.django_db
+@pytest.mark.order("last")
+class TestProjectConfigTab(SeleniumFunctionalTestCase):
+    PROJECT_NAME = 'TestProjectConfigTab'
+    project_id = None
+
+    def _create_project(self, project_name, **kwargs):
+        """ Create/Test new project using:
+          - Project Name: Any string
+          - Release: Any string
+          - Merge Toaster settings: True or False
+        """
+        release = kwargs.get('release', '3')
+        self.get(reverse('newproject'))
+        self.wait_until_visible('#new-project-name')
+        self.find("#new-project-name").send_keys(project_name)
+        select = Select(self.find("#projectversion"))
+        select.select_by_value(release)
+
+        # check merge toaster settings
+        checkbox = self.find('.checkbox-mergeattr')
+        if not checkbox.is_selected():
+            checkbox.click()
+
+        if self.PROJECT_NAME != 'TestProjectConfigTab':
+            # Reset project name if it's not the default one
+            self.PROJECT_NAME = 'TestProjectConfigTab'
+
+        self.find("#create-project-button").click()
+
+        try:
+            self.wait_until_visible('#hint-error-project-name', poll=3)
+            url = reverse('project', args=(TestProjectConfigTab.project_id, ))
+            self.get(url)
+            self.wait_until_visible('#config-nav', poll=3)
+        except TimeoutException:
+            self.wait_until_visible('#config-nav', poll=3)
+
+    def _random_string(self, length):
+        return ''.join(
+            random.choice(string.ascii_letters) for _ in range(length)
+        )
+
+    def _navigate_to_project_page(self):
+        # Navigate to project page
+        if TestProjectConfigTab.project_id is None:
+            self._create_project(project_name=self._random_string(10))
+            current_url = self.driver.current_url
+            TestProjectConfigTab.project_id = get_projectId_from_url(
+                current_url)
+        else:
+            url = reverse('project', args=(TestProjectConfigTab.project_id,))
+            self.get(url)
+        self.wait_until_visible('#config-nav')
+
+    def _create_builds(self):
+        # check search box can be use to build recipes
+        search_box = self.find('#build-input')
+        search_box.send_keys('foo')
+        self.find('#build-button').click()
+        self.wait_until_present('#latest-builds')
+        # loop until reach the parsing state
+        wait_until_build(self, 'queued cloning starting parsing failed')
+        lastest_builds = self.driver.find_elements(
+            By.XPATH,
+            '//div[@id="latest-builds"]/div',
+        )
+        last_build = lastest_builds[0]
+        self.assertTrue(
+            'foo' in str(last_build.text)
+        )
+        last_build = lastest_builds[0]
+        try:
+            cancel_button = last_build.find_element(
+                By.XPATH,
+                '//span[@class="cancel-build-btn pull-right alert-link"]',
+            )
+            cancel_button.click()
+        except NoSuchElementException:
+            # Skip if the build is already cancelled
+            pass
+        wait_until_build_cancelled(self)
+
+    def _get_tabs(self):
+        # tabs links list
+        return self.driver.find_elements(
+            By.XPATH,
+            '//div[@id="project-topbar"]//li'
+        )
+
+    def _get_config_nav_item(self, index):
+        config_nav = self.find('#config-nav')
+        return config_nav.find_elements(By.TAG_NAME, 'li')[index]
+
+    def test_project_config_nav(self):
+        """ Test project config tab navigation:
+        - Check if the menu is displayed and contains the right elements:
+            - Configuration
+            - COMPATIBLE METADATA
+            - Custom images
+            - Image recipes
+            - Software recipes
+            - Machines
+            - Layers
+            - Distro
+            - EXTRA CONFIGURATION
+            - Bitbake variables
+            - Actions
+            - Delete project
+        """
+        self._navigate_to_project_page()
+
+        def _get_config_nav_item(index):
+            config_nav = self.find('#config-nav')
+            return config_nav.find_elements(By.TAG_NAME, 'li')[index]
+
+        def check_config_nav_item(index, item_name, url):
+            item = _get_config_nav_item(index)
+            self.assertTrue(item_name in item.text)
+            self.assertTrue(item.get_attribute('class') == 'active')
+            self.assertTrue(url in self.driver.current_url)
+
+        # check if the menu contains the right elements
+        # COMPATIBLE METADATA
+        compatible_metadata = _get_config_nav_item(1)
+        self.assertTrue(
+            "compatible metadata" in compatible_metadata.text.lower()
+        )
+        # EXTRA CONFIGURATION
+        extra_configuration = _get_config_nav_item(8)
+        self.assertTrue(
+            "extra configuration" in extra_configuration.text.lower()
+        )
+        # Actions
+        actions = _get_config_nav_item(10)
+        self.assertTrue("actions" in str(actions.text).lower())
+
+        conf_nav_list = [
+            # config
+            [0, 'Configuration',
+                f"/toastergui/project/{TestProjectConfigTab.project_id}"],
+            # custom images
+            [2, 'Custom images',
+                f"/toastergui/project/{TestProjectConfigTab.project_id}/customimages"],
+            # image recipes
+            [3, 'Image recipes',
+                f"/toastergui/project/{TestProjectConfigTab.project_id}/images"],
+            # software recipes
+            [4, 'Software recipes',
+                f"/toastergui/project/{TestProjectConfigTab.project_id}/softwarerecipes"],
+            # machines
+            [5, 'Machines',
+                f"/toastergui/project/{TestProjectConfigTab.project_id}/machines"],
+            # layers
+            [6, 'Layers',
+                f"/toastergui/project/{TestProjectConfigTab.project_id}/layers"],
+            # distro
+            [7, 'Distros',
+                f"/toastergui/project/{TestProjectConfigTab.project_id}/distros"],
+            #  [9, 'BitBake variables', f"/toastergui/project/{TestProjectConfigTab.project_id}/configuration"],  # bitbake variables
+        ]
+        for index, item_name, url in conf_nav_list:
+            item = _get_config_nav_item(index)
+            if item.get_attribute('class') != 'active':
+                item.click()
+            check_config_nav_item(index, item_name, url)
+
+    def test_image_recipe_editColumn(self):
+        """ Test the edit column feature in image recipe table on project page """
+        def test_edit_column(check_box_id):
+            # Check that we can hide/show table column
+            check_box = self.find(f'#{check_box_id}')
+            th_class = str(check_box_id).replace('checkbox-', '')
+            if check_box.is_selected():
+                # check if column is visible in table
+                self.assertTrue(
+                    self.find(
+                        f'#imagerecipestable thead th.{th_class}'
+                    ).is_displayed(),
+                    f"The {th_class} column is checked in EditColumn dropdown, but it's not visible in table"
+                )
+                check_box.click()
+                # check if column is hidden in table
+                self.assertFalse(
+                    self.find(
+                        f'#imagerecipestable thead th.{th_class}'
+                    ).is_displayed(),
+                    f"The {th_class} column is unchecked in EditColumn dropdown, but it's visible in table"
+                )
+            else:
+                # check if column is hidden in table
+                self.assertFalse(
+                    self.find(
+                        f'#imagerecipestable thead th.{th_class}'
+                    ).is_displayed(),
+                    f"The {th_class} column is unchecked in EditColumn dropdown, but it's visible in table"
+                )
+                check_box.click()
+                # check if column is visible in table
+                self.assertTrue(
+                    self.find(
+                        f'#imagerecipestable thead th.{th_class}'
+                    ).is_displayed(),
+                    f"The {th_class} column is checked in EditColumn dropdown, but it's not visible in table"
+                )
+
+        self._navigate_to_project_page()
+        # navigate to project image recipe page
+        recipe_image_page_link = self._get_config_nav_item(3)
+        recipe_image_page_link.click()
+        self.wait_until_present('#imagerecipestable tbody tr')
+
+        # Check edit column
+        edit_column = self.find('#edit-columns-button')
+        self.assertTrue(edit_column.is_displayed())
+        edit_column.click()
+        # Check dropdown is visible
+        self.wait_until_visible('ul.dropdown-menu.editcol')
+
+        # Check that we can hide the edit column
+        test_edit_column('checkbox-get_description_or_summary')
+        test_edit_column('checkbox-layer_version__get_vcs_reference')
+        test_edit_column('checkbox-layer_version__layer__name')
+        test_edit_column('checkbox-license')
+        test_edit_column('checkbox-recipe-file')
+        test_edit_column('checkbox-section')
+        test_edit_column('checkbox-version')
+
+    def test_image_recipe_show_rows(self):
+        """ Test the show rows feature in image recipe table on project page """
+        def test_show_rows(row_to_show, show_row_link):
+            # Check that we can show rows == row_to_show
+            show_row_link.select_by_value(str(row_to_show))
+            self.wait_until_visible('#imagerecipestable tbody tr', poll=3)
+            # check at least some rows are visible
+            self.assertTrue(
+                len(self.find_all('#imagerecipestable tbody tr'))  > 0
+            )
+
+        self._navigate_to_project_page()
+        # navigate to project image recipe page
+        recipe_image_page_link = self._get_config_nav_item(3)
+        recipe_image_page_link.click()
+        self.wait_until_present('#imagerecipestable tbody tr')
+
+        show_rows = self.driver.find_elements(
+            By.XPATH,
+            '//select[@class="form-control pagesize-imagerecipestable"]'
+        )
+        # Check show rows
+        for show_row_link in show_rows:
+            show_row_link = Select(show_row_link)
+            test_show_rows(10, show_row_link)
+            test_show_rows(25, show_row_link)
+            test_show_rows(50, show_row_link)
+            test_show_rows(100, show_row_link)
+            test_show_rows(150, show_row_link)
+
+    def test_project_config_tab_right_section(self):
+        """ Test project config tab right section contains five blocks:
+            - Machine:
+                - check 'Machine' is displayed
+                - check can change Machine
+            - Distro:
+                - check 'Distro' is displayed
+                - check can change Distro
+            - Most built recipes:
+                - check 'Most built recipes' is displayed
+                - check can select a recipe and build it
+            - Project release:
+                - check 'Project release' is displayed
+                - check project has right release displayed
+            - Layers:
+                - check can add a layer if exists
+                - check at least three layers are displayed
+                    - openembedded-core
+                    - meta-poky
+                    - meta-yocto-bsp
+        """
+        # Create a new project for this test
+        project_name = self._random_string(10)
+        self._create_project(project_name=project_name)
+        # check if the menu is displayed
+        self.wait_until_visible('#project-page')
+        block_l = self.driver.find_element(
+            By.XPATH, '//*[@id="project-page"]/div[2]')
+        project_release = self.driver.find_element(
+            By.XPATH, '//*[@id="project-page"]/div[1]/div[4]')
+        layers = block_l.find_element(By.ID, 'layer-container')
+
+        def check_machine_distro(self, item_name, new_item_name, block_id):
+            block = self.find(f'#{block_id}')
+            title = block.find_element(By.TAG_NAME, 'h3')
+            self.assertTrue(item_name.capitalize() in title.text)
+            edit_btn = self.find(f'#change-{item_name}-toggle')
+            edit_btn.click()
+            self.wait_until_visible(f'#{item_name}-change-input')
+            name_input = self.find(f'#{item_name}-change-input')
+            name_input.clear()
+            name_input.send_keys(new_item_name)
+            change_btn = self.find(f'#{item_name}-change-btn')
+            change_btn.click()
+            self.wait_until_visible(f'#project-{item_name}-name')
+            project_name = self.find(f'#project-{item_name}-name')
+            self.assertTrue(new_item_name in project_name.text)
+            # check change notificaiton is displayed
+            change_notification = self.find('#change-notification')
+            self.assertTrue(
+                f'You have changed the {item_name} to: {new_item_name}' in change_notification.text
+            )
+
+        # Machine
+        check_machine_distro(self, 'machine', 'qemux86-64', 'machine-section')
+        # Distro
+        check_machine_distro(self, 'distro', 'poky-altcfg', 'distro-section')
+
+        # Project release
+        title = project_release.find_element(By.TAG_NAME, 'h3')
+        self.assertTrue("Project release" in title.text)
+        self.assertTrue(
+            "Yocto Project master" in self.find('#project-release-title').text
+        )
+        # Layers
+        title = layers.find_element(By.TAG_NAME, 'h3')
+        self.assertTrue("Layers" in title.text)
+        # check at least three layers are displayed
+        # openembedded-core
+        # meta-poky
+        # meta-yocto-bsp
+        layers_list = layers.find_element(By.ID, 'layers-in-project-list')
+        layers_list_items = layers_list.find_elements(By.TAG_NAME, 'li')
+        # remove all layers except the first three layers
+        for i in range(3, len(layers_list_items)):
+            layers_list_items[i].find_element(By.TAG_NAME, 'span').click()
+        # check can add a layer if exists
+        add_layer_input = layers.find_element(By.ID, 'layer-add-input')
+        add_layer_input.send_keys('meta-oe')
+        self.wait_until_visible('#layer-container > form > div > span > div')
+        dropdown_item = self.driver.find_element(
+            By.XPATH,
+            '//*[@id="layer-container"]/form/div/span/div'
+        )
+        try:
+            dropdown_item.click()
+        except ElementClickInterceptedException:
+            self.skipTest(
+                "layer-container dropdown item click intercepted. Element not properly visible.")
+        add_layer_btn = layers.find_element(By.ID, 'add-layer-btn')
+        add_layer_btn.click()
+        self.wait_until_visible('#layers-in-project-list')
+        # check layer is added
+        layers_list_items = layers_list.find_elements(By.TAG_NAME, 'li')
+        self.assertTrue(len(layers_list_items) == 4)
+
+    def test_most_build_recipes(self):
+        """ Test most build recipes block contains"""
+        def rebuild_from_most_build_recipes(recipe_list_items):
+            checkbox = recipe_list_items[0].find_element(By.TAG_NAME, 'input')
+            checkbox.click()
+            build_btn = self.find('#freq-build-btn')
+            build_btn.click()
+            self.wait_until_visible('#latest-builds')
+            wait_until_build(self, 'queued cloning starting parsing failed')
+            lastest_builds = self.driver.find_elements(
+                By.XPATH,
+                '//div[@id="latest-builds"]/div'
+            )
+            self.assertTrue(len(lastest_builds) >= 2)
+            last_build = lastest_builds[0]
+            try:
+                cancel_button = last_build.find_element(
+                    By.XPATH,
+                    '//span[@class="cancel-build-btn pull-right alert-link"]',
+                )
+                cancel_button.click()
+            except NoSuchElementException:
+                # Skip if the build is already cancelled
+                pass
+            wait_until_build_cancelled(self)
+        # Create a new project for remaining asserts
+        project_name = self._random_string(10)
+        self._create_project(project_name=project_name, release='2')
+        current_url = self.driver.current_url
+        TestProjectConfigTab.project_id = get_projectId_from_url(current_url)
+        url = current_url.split('?')[0]
+
+        # Create a new builds
+        self._create_builds()
+
+        # back to project page
+        self.driver.get(url)
+
+        self.wait_until_visible('#project-page', poll=3)
+
+        # Most built recipes
+        most_built_recipes = self.driver.find_element(
+            By.XPATH, '//*[@id="project-page"]/div[1]/div[3]')
+        title = most_built_recipes.find_element(By.TAG_NAME, 'h3')
+        self.assertTrue("Most built recipes" in title.text)
+        # check can select a recipe and build it
+        self.wait_until_visible('#freq-build-list', poll=3)
+        recipe_list = self.find('#freq-build-list')
+        recipe_list_items = recipe_list.find_elements(By.TAG_NAME, 'li')
+        self.assertTrue(
+            len(recipe_list_items) > 0,
+            msg="Any recipes found in the most built recipes list",
+        )
+        rebuild_from_most_build_recipes(recipe_list_items)
+        TestProjectConfigTab.project_id = None  # reset project id
+
+    def test_project_page_tab_importlayer(self):
+        """ Test project page tab import layer """
+        self._navigate_to_project_page()
+        # navigate to "Import layers" tab
+        import_layers_tab = self._get_tabs()[2]
+        import_layers_tab.find_element(By.TAG_NAME, 'a').click()
+        self.wait_until_visible('#layer-git-repo-url')
+
+        # Check git repo radio button
+        git_repo_radio = self.find('#git-repo-radio')
+        git_repo_radio.click()
+
+        # Set git repo url
+        input_repo_url = self.find('#layer-git-repo-url')
+        input_repo_url.send_keys('git://git.yoctoproject.org/meta-fake')
+        # Blur the input to trigger the validation
+        input_repo_url.send_keys(Keys.TAB)
+
+        # Check name is set
+        input_layer_name = self.find('#import-layer-name')
+        self.assertTrue(input_layer_name.get_attribute('value') == 'meta-fake')
+
+        # Set branch
+        input_branch = self.find('#layer-git-ref')
+        input_branch.send_keys('master')
+
+        # Import layer
+        self.find('#import-and-add-btn').click()
+
+        # Check layer is added
+        self.wait_until_visible('#layer-container')
+        block_l = self.driver.find_element(
+            By.XPATH, '//*[@id="project-page"]/div[2]')
+        layers = block_l.find_element(By.ID, 'layer-container')
+        layers_list = layers.find_element(By.ID, 'layers-in-project-list')
+        layers_list_items = layers_list.find_elements(By.TAG_NAME, 'li')
+        self.assertTrue(
+            'meta-fake' in str(layers_list_items[-1].text)
+        )
+
+    def test_project_page_custom_image_no_image(self):
+        """ Test project page tab "New custom image" when no custom image """
+        project_name = self._random_string(10)
+        self._create_project(project_name=project_name)
+        current_url = self.driver.current_url
+        TestProjectConfigTab.project_id = get_projectId_from_url(current_url)
+        # navigate to "Custom image" tab
+        custom_image_section = self._get_config_nav_item(2)
+        custom_image_section.click()
+        self.wait_until_visible('#empty-state-customimagestable')
+
+        # Check message when no custom image
+        self.assertTrue(
+            "You have not created any custom images yet." in str(
+                self.find('#empty-state-customimagestable').text
+            )
+        )
+        div_empty_msg = self.find('#empty-state-customimagestable')
+        link_create_custom_image = div_empty_msg.find_element(
+            By.TAG_NAME, 'a')
+        self.assertTrue(TestProjectConfigTab.project_id is not None)
+        self.assertTrue(
+            f"/toastergui/project/{TestProjectConfigTab.project_id}/newcustomimage" in str(
+                link_create_custom_image.get_attribute('href')
+            )
+        )
+        self.assertTrue(
+            "Create your first custom image" in str(
+                link_create_custom_image.text
+            )
+        )
+        TestProjectConfigTab.project_id = None  # reset project id
+
+    def test_project_page_image_recipe(self):
+        """ Test project page section images
+            - Check image recipes are displayed
+            - Check search input
+            - Check image recipe build button works
+            - Check image recipe table features(show/hide column, pagination)
+        """
+        self._navigate_to_project_page()
+        # navigate to "Images section"
+        images_section = self._get_config_nav_item(3)
+        images_section.click()
+        self.wait_until_visible('#imagerecipestable')
+        rows = self.find_all('#imagerecipestable tbody tr')
+        self.assertTrue(len(rows) > 0)
+
+        # Test search input
+        self.wait_until_visible('#search-input-imagerecipestable')
+        recipe_input = self.find('#search-input-imagerecipestable')
+        recipe_input.send_keys('core-image-minimal')
+        self.find('#search-submit-imagerecipestable').click()
+        self.wait_until_visible('#imagerecipestable tbody tr')
+        rows = self.find_all('#imagerecipestable tbody tr')
+        self.assertTrue(len(rows) > 0)

bitbake/lib/toaster/tests/functional/utils.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# BitBake Toaster UI tests implementation
+#
+# Copyright (C) 2023 Savoir-faire Linux
+#
+# SPDX-License-Identifier: GPL-2.0-only
+
+
+from time import sleep
+from selenium.common.exceptions import NoSuchElementException, StaleElementReferenceException, TimeoutException
+from selenium.webdriver.common.by import By
+
+from orm.models import Build
+
+
+def wait_until_build(test_instance, state):
+    timeout = 60
+    start_time = 0
+    build_state = ''
+    while True:
+        try:
+            if start_time > timeout:
+                raise TimeoutException(
+                    f'Build did not reach {state} state within {timeout} seconds'
+                )
+            last_build_state = test_instance.driver.find_element(
+                By.XPATH,
+                '//*[@id="latest-builds"]/div[1]//div[@class="build-state"]',
+            )
+            build_state = last_build_state.get_attribute(
+                'data-build-state')
+            state_text = state.lower().split()
+            if any(x in str(build_state).lower() for x in state_text):
+                return str(build_state).lower()
+            if 'failed' in str(build_state).lower():
+                break
+        except NoSuchElementException:
+            continue
+        except TimeoutException:
+            break
+        start_time += 1
+        sleep(1) # take a breath and try again
+
+def wait_until_build_cancelled(test_instance):
+    """ Cancel build take a while sometime, the method is to wait driver action
+        until build being cancelled
+    """
+    timeout = 30
+    start_time = 0
+    build = None
+    while True:
+        try:
+            if start_time > timeout:
+                raise TimeoutException(
+                    f'Build did not reach cancelled state within {timeout} seconds'
+                )
+            last_build_state = test_instance.driver.find_element(
+                By.XPATH,
+                '//*[@id="latest-builds"]/div[1]//div[@class="build-state"]',
+            )
+            build_state = last_build_state.get_attribute(
+                'data-build-state')
+            if 'failed' in str(build_state).lower():
+                break
+            if 'cancelling' in str(build_state).lower():
+                # Change build state to cancelled
+                if not build:  # get build object only once
+                    build = Build.objects.last()
+                    build.outcome = Build.CANCELLED
+                    build.save()
+            if 'cancelled' in str(build_state).lower():
+                break
+        except NoSuchElementException:
+            continue
+        except StaleElementReferenceException:
+            continue
+        except TimeoutException:
+            break
+        start_time += 1
+        sleep(1) # take a breath and try again
+
+def get_projectId_from_url(url):
+    # url = 'http://domainename.com/toastergui/project/1656/whatever
+    # or url = 'http://domainename.com/toastergui/project/1/
+    # or url = 'http://domainename.com/toastergui/project/186
+    assert '/toastergui/project/' in url, "URL is not valid"
+    url_to_list = url.split('/toastergui/project/')
+    return  int(url_to_list[1].split('/')[0])  # project_id