mirrors/poky
1
0
Fork 0
mirror of https://git.yoctoproject.org/poky synced 2026-09-16 09:49:35 +02:00
Code Issues Packages Projects Releases Wiki Activity

bitbake: runqueue: Change buildable/running lists to sets

Browse Source 
Using positions in lists for flags is an odd choice and makes the code
hard to maintain. Maintaining a list is slow since list searches are
slow (watch bitbake -n slow massively with it) but we can use a set()
instead.

This patch uses python sets to maintain the lists of tasks in each state
and this prepares for changing the task IDs from being integers.

(Bitbake rev: 8c1ed57f6ea475b714eca6673b48e8e5f5f0f9c3)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Richard Purdie
2016-06-12 23:55:08 +01:00
parent 7e54da0581
commit 29b4fb1796
1 changed files with 27 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
bitbake/lib/bb/runqueue.py
View File

@@ -108,7 +108,7 @@ class RunQueueScheduler(object):
fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[taskid]] fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[taskid]]
taskname = self.rqdata.runq_task[taskid] taskname = self.rqdata.runq_task[taskid]
self.stamps[taskid] = bb.build.stampfile(taskname, self.rqdata.dataCache, fn) self.stamps[taskid] = bb.build.stampfile(taskname, self.rqdata.dataCache, fn)
if self.rq.runq_buildable[taskid] == 1: if taskid in self.rq.runq_buildable:
self.buildable.append(taskid) self.buildable.append(taskid)
self.rev_prio_map = None self.rev_prio_map = None
@@ -117,7 +117,7 @@ class RunQueueScheduler(object):
""" """
Return the id of the first task we find that is buildable Return the id of the first task we find that is buildable
""" """
self.buildable = [x for x in self.buildable if not self.rq.runq_running[x] == 1] self.buildable = [x for x in self.buildable if x not in self.rq.runq_running]
if not self.buildable: if not self.buildable:
return None return None
if len(self.buildable) == 1: if len(self.buildable) == 1:
@@ -1361,9 +1361,9 @@ class RunQueueExecute:
self.number_tasks = int(self.cfgData.getVar("BB_NUMBER_THREADS", True) or 1) self.number_tasks = int(self.cfgData.getVar("BB_NUMBER_THREADS", True) or 1)
self.scheduler = self.cfgData.getVar("BB_SCHEDULER", True) or "speed" self.scheduler = self.cfgData.getVar("BB_SCHEDULER", True) or "speed"
self.runq_buildable = [] self.runq_buildable = set()
self.runq_running = [] self.runq_running = set()
self.runq_complete = [] self.runq_complete = set()
self.build_stamps = {} self.build_stamps = {}
self.build_stamps2 = [] self.build_stamps2 = []
@@ -1464,12 +1464,8 @@ class RunQueueExecuteTasks(RunQueueExecute):
# Mark initial buildable tasks # Mark initial buildable tasks
for task in range(self.stats.total): for task in range(self.stats.total):
self.runq_running.append(0)
self.runq_complete.append(0)
if len(self.rqdata.runq_depends[task]) == 0: if len(self.rqdata.runq_depends[task]) == 0:
self.runq_buildable.append(1) self.runq_buildable.add(task)
else:
self.runq_buildable.append(0)
if len(self.rqdata.runq_revdeps[task]) > 0 and self.rqdata.runq_revdeps[task].issubset(self.rq.scenequeue_covered): if len(self.rqdata.runq_revdeps[task]) > 0 and self.rqdata.runq_revdeps[task].issubset(self.rq.scenequeue_covered):
self.rq.scenequeue_covered.add(task) self.rq.scenequeue_covered.add(task)
@@ -1574,7 +1570,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
return schedulers return schedulers
def setbuildable(self, task): def setbuildable(self, task):
self.runq_buildable[task] = 1 self.runq_buildable.add(task)
self.sched.newbuilable(task) self.sched.newbuilable(task)
def task_completeoutright(self, task): def task_completeoutright(self, task):
@@ -1583,15 +1579,15 @@ class RunQueueExecuteTasks(RunQueueExecute):
Look at the reverse dependencies and mark any task with Look at the reverse dependencies and mark any task with
completed dependencies as buildable completed dependencies as buildable
""" """
self.runq_complete[task] = 1 self.runq_complete.add(task)
for revdep in self.rqdata.runq_revdeps[task]: for revdep in self.rqdata.runq_revdeps[task]:
if self.runq_running[revdep] == 1: if revdep in self.runq_running:
continue continue
if self.runq_buildable[revdep] == 1: if revdep in self.runq_buildable:
continue continue
alldeps = 1 alldeps = 1
for dep in self.rqdata.runq_depends[revdep]: for dep in self.rqdata.runq_depends[revdep]:
if self.runq_complete[dep] != 1: if dep not in self.runq_complete:
alldeps = 0 alldeps = 0
if alldeps == 1: if alldeps == 1:
self.setbuildable(revdep) self.setbuildable(revdep)
@@ -1617,7 +1613,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
self.rq.state = runQueueCleanUp self.rq.state = runQueueCleanUp
def task_skip(self, task, reason): def task_skip(self, task, reason):
self.runq_running[task] = 1 self.runq_running.add(task)
self.setbuildable(task) self.setbuildable(task)
bb.event.fire(runQueueTaskSkipped(task, self.stats, self.rq, reason), self.cfgData) bb.event.fire(runQueueTaskSkipped(task, self.stats, self.rq, reason), self.cfgData)
self.task_completeoutright(task) self.task_completeoutright(task)
@@ -1658,7 +1654,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
startevent = runQueueTaskStarted(task, self.stats, self.rq, startevent = runQueueTaskStarted(task, self.stats, self.rq,
noexec=True) noexec=True)
bb.event.fire(startevent, self.cfgData) bb.event.fire(startevent, self.cfgData)
self.runq_running[task] = 1 self.runq_running.add(task)
self.stats.taskActive() self.stats.taskActive()
if not self.cooker.configuration.dry_run: if not self.cooker.configuration.dry_run:
bb.build.make_stamp(taskname, self.rqdata.dataCache, fn) bb.build.make_stamp(taskname, self.rqdata.dataCache, fn)
@@ -1687,7 +1683,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCache, fn) self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCache, fn)
self.build_stamps2.append(self.build_stamps[task]) self.build_stamps2.append(self.build_stamps[task])
self.runq_running[task] = 1 self.runq_running.add(task)
self.stats.taskActive() self.stats.taskActive()
if self.stats.active < self.number_tasks: if self.stats.active < self.number_tasks:
return True return True
@@ -1702,11 +1698,11 @@ class RunQueueExecuteTasks(RunQueueExecute):
# Sanity Checks # Sanity Checks
for task in range(self.stats.total): for task in range(self.stats.total):
if self.runq_buildable[task] == 0: if task not in self.runq_buildable:
logger.error("Task %s never buildable!", task) logger.error("Task %s never buildable!", task)
if self.runq_running[task] == 0: if task not in self.runq_running:
logger.error("Task %s never ran!", task) logger.error("Task %s never ran!", task)
if self.runq_complete[task] == 0: if task not in self.runq_complete:
logger.error("Task %s never completed!", task) logger.error("Task %s never completed!", task)
self.rq.state = runQueueComplete self.rq.state = runQueueComplete
@@ -1759,11 +1755,6 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
# therefore aims to collapse the huge runqueue dependency tree into a smaller one # therefore aims to collapse the huge runqueue dependency tree into a smaller one
# only containing the setscene functions. # only containing the setscene functions.
for task in range(self.stats.total):
self.runq_running.append(0)
self.runq_complete.append(0)
self.runq_buildable.append(0)
# First process the chains up to the first setscene task. # First process the chains up to the first setscene task.
endpoints = {} endpoints = {}
for task in range(len(self.rqdata.runq_fnid)): for task in range(len(self.rqdata.runq_fnid)):
@@ -1894,7 +1885,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
for task in range(len(self.sq_revdeps)): for task in range(len(self.sq_revdeps)):
if len(self.sq_revdeps[task]) == 0: if len(self.sq_revdeps[task]) == 0:
self.runq_buildable[task] = 1 self.runq_buildable.add(task)
self.outrightfail = [] self.outrightfail = []
if self.rq.hashvalidate: if self.rq.hashvalidate:
@@ -1966,7 +1957,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
continue continue
self.sq_revdeps2[dep].remove(task) self.sq_revdeps2[dep].remove(task)
if len(self.sq_revdeps2[dep]) == 0: if len(self.sq_revdeps2[dep]) == 0:
self.runq_buildable[dep] = 1 self.runq_buildable.add(dep)
def task_completeoutright(self, task): def task_completeoutright(self, task):
""" """
@@ -1994,8 +1985,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
self.scenequeue_updatecounters(task, True) self.scenequeue_updatecounters(task, True)
def task_failoutright(self, task): def task_failoutright(self, task):
self.runq_running[task] = 1 self.runq_running.add(task)
self.runq_buildable[task] = 1 self.runq_buildable.add(task)
self.stats.taskCompleted() self.stats.taskCompleted()
self.stats.taskSkipped() self.stats.taskSkipped()
index = self.rqdata.runq_setscene[task] index = self.rqdata.runq_setscene[task]
@@ -2003,8 +1994,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
self.scenequeue_updatecounters(task, True) self.scenequeue_updatecounters(task, True)
def task_skip(self, task): def task_skip(self, task):
self.runq_running[task] = 1 self.runq_running.add(task)
self.runq_buildable[task] = 1 self.runq_buildable.add(task)
self.task_completeoutright(task) self.task_completeoutright(task)
self.stats.taskCompleted() self.stats.taskCompleted()
self.stats.taskSkipped() self.stats.taskSkipped()
@@ -2020,7 +2011,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
if self.stats.active < self.number_tasks: if self.stats.active < self.number_tasks:
# Find the next setscene to run # Find the next setscene to run
for nexttask in range(self.stats.total): for nexttask in range(self.stats.total):
if self.runq_buildable[nexttask] == 1 and self.runq_running[nexttask] != 1: if nexttask in self.runq_buildable and nexttask not in self.runq_running:
if nexttask in self.unskippable: if nexttask in self.unskippable:
logger.debug(2, "Setscene task %s is unskippable" % self.rqdata.get_user_idstring(self.rqdata.runq_setscene[nexttask])) logger.debug(2, "Setscene task %s is unskippable" % self.rqdata.get_user_idstring(self.rqdata.runq_setscene[nexttask]))
if nexttask not in self.unskippable and len(self.sq_revdeps[nexttask]) > 0 and self.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sq_revdeps[nexttask], True): if nexttask not in self.unskippable and len(self.sq_revdeps[nexttask]) > 0 and self.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sq_revdeps[nexttask], True):
@@ -2077,7 +2068,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
self.rq.worker.stdin.write(b"<runtask>" + pickle.dumps((fn, realtask, taskname, True, self.cooker.collection.get_file_appends(fn), None)) + b"</runtask>") self.rq.worker.stdin.write(b"<runtask>" + pickle.dumps((fn, realtask, taskname, True, self.cooker.collection.get_file_appends(fn), None)) + b"</runtask>")
self.rq.worker.stdin.flush() self.rq.worker.stdin.flush()
self.runq_running[task] = 1 self.runq_running.add(task)
self.stats.taskActive() self.stats.taskActive()
if self.stats.active < self.number_tasks: if self.stats.active < self.number_tasks:
return True return True
@@ -2087,8 +2078,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
return self.rq.active_fds() return self.rq.active_fds()
#for task in range(self.stats.total): #for task in range(self.stats.total):
# if self.runq_running[task] != 1: # if task not in self.runq_running:
# buildable = self.runq_buildable[task] # buildable = task in self.runq_buildable
# revdeps = self.sq_revdeps[task] # revdeps = self.sq_revdeps[task]
# bb.warn("Found we didn't run %s %s %s %s" % (task, buildable, str(revdeps), self.rqdata.get_user_idstring(self.rqdata.runq_setscene[task]))) # bb.warn("Found we didn't run %s %s %s %s" % (task, buildable, str(revdeps), self.rqdata.get_user_idstring(self.rqdata.runq_setscene[task])))