From cdb1cb2ed8f689a0a7ecc874885c0ba7c708cbd6 Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Mon, 12 Feb 2024 23:08:44 +0000 Subject: [PATCH] bitbake: runqueue: Improve setcene performance when encoutering many 'hard' dependencies "bitbake world -n --setscene-only" shows poor performance as the numbers of tasks increases. Analysys shows this is due to the "deferred" hard dependencies being repeatedly processed which doesn't allow much forward progress in overall task execution. To avoid this, mark when it has been done and don't reprocess until dependencies are updated. We have to be careful as we've seen bugs where these dependencies aren't processed at the right time. They also have to be reproceseed during task migration/rehashing so the code has to "self heal" the data structures. (Bitbake rev: e5609bac06c17dabcf6286b47b1a3f19f5a1160f) Signed-off-by: Richard Purdie --- bitbake/lib/bb/runqueue.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py index 95cab5132f..e86ccd8c61 100644 --- a/bitbake/lib/bb/runqueue.py +++ b/bitbake/lib/bb/runqueue.py @@ -1840,6 +1840,7 @@ class RunQueueExecute: self.failed_tids = [] self.sq_deferred = {} self.sq_needed_harddeps = set() + self.sq_harddep_deferred = set() self.stampcache = {} @@ -2163,7 +2164,7 @@ class RunQueueExecute: if not self.sqdone and self.can_start_task(): # Find the next setscene to run for nexttask in self.sorted_setscene_tids: - if nexttask in self.sq_buildable and nexttask not in self.sq_running and self.sqdata.stamps[nexttask] not in self.build_stamps.values(): + if nexttask in self.sq_buildable and nexttask not in self.sq_running and self.sqdata.stamps[nexttask] not in self.build_stamps.values() and nexttask not in self.sq_harddep_deferred: if nexttask not in self.sqdata.unskippable and self.sqdata.sq_revdeps[nexttask] and \ nexttask not in self.sq_needed_harddeps and \ self.sqdata.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and \ @@ -2184,6 +2185,7 @@ class RunQueueExecute: self.sq_buildable.add(dep) self.sq_needed_harddeps.add(dep) updated = True + self.sq_harddep_deferred.add(nexttask) if updated: return True continue @@ -2678,6 +2680,7 @@ class RunQueueExecute: if changed: self.stats.updateCovered(len(self.scenequeue_covered), len(self.scenequeue_notcovered)) self.sq_needed_harddeps = set() + self.sq_harddep_deferred = set() self.holdoff_need_update = True def scenequeue_updatecounters(self, task, fail=False): @@ -2712,6 +2715,13 @@ class RunQueueExecute: new.add(dep) next = new + # If this task was one which other setscene tasks have a hard dependency upon, we need + # to walk through the hard dependencies and allow execution of those which have completed dependencies. + if task in self.sqdata.sq_harddeps: + for dep in self.sq_harddep_deferred.copy(): + if self.sqdata.sq_harddeps_rev[dep].issubset(self.scenequeue_covered | self.scenequeue_notcovered): + self.sq_harddep_deferred.remove(dep) + self.stats.updateCovered(len(self.scenequeue_covered), len(self.scenequeue_notcovered)) self.holdoff_need_update = True