mirrors/poky
1
0
Fork 0
mirror of https://git.yoctoproject.org/poky synced 2026-04-05 08:02:25 +02:00
Code Issues Packages Projects Releases Wiki Activity

bitbake: bitbake: runqueue: add memory pressure regulation

Browse Source 
Prevent new tasks from being scheduled if the memory pressure is above
a certain threshold, specified through the "BB_MAX_PRESSURE_MEMORY"
variable in the conf/local.conf file. This is an extension to the
following commit and hence regulates pressure in the same way:
   48a6d84de1 bitbake: runqueue: add cpu/io pressure regulation

Memory pressure is experienced when time is spent swapping, refaulting
pages from the page cache or performing direct reclaim. This is why
memory pressure is rarely seen but might be useful as a last resort to
prevent OOM errors.

(Bitbake rev: 44c395434c7be8dab968630a610c8807f512920c)

(Bitbake rev: 4ada86cb6b05e6e3aabc8015a6e73aacb14a3388)

Signed-off-by: Aryaman Gupta <aryaman.gupta@windriver.com>
Signed-off-by: Randy Macleod <Randy.Macleod@windriver.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Signed-off-by: Steve Sakoman <steve@sakoman.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Aryaman Gupta
2022-08-16 14:27:56 -04:00
committed by Richard Purdie
parent 3b18438166
commit 2e193726bd
1 changed files with 22 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
bitbake/lib/bb/runqueue.py
View File

@@ -168,11 +168,15 @@ class RunQueueScheduler(object):
openSUSE /proc/pressure/* files have readable file permissions but when read the error EOPNOTSUPP (Operation not supported)
is returned.
"""
if self.rq.max_cpu_pressure or self.rq.max_io_pressure:
if self.rq.max_cpu_pressure or self.rq.max_io_pressure or self.rq.max_memory_pressure:
try:
with open("/proc/pressure/cpu") as cpu_pressure_fds, open("/proc/pressure/io") as io_pressure_fds:
with open("/proc/pressure/cpu") as cpu_pressure_fds, \
open("/proc/pressure/io") as io_pressure_fds, \
open("/proc/pressure/memory") as memory_pressure_fds:
self.prev_cpu_pressure = cpu_pressure_fds.readline().split()[4].split("=")[1]
self.prev_io_pressure = io_pressure_fds.readline().split()[4].split("=")[1]
self.prev_memory_pressure = memory_pressure_fds.readline().split()[4].split("=")[1]
self.prev_pressure_time = time.time()
self.check_pressure = True
except:
@@ -184,21 +188,26 @@ class RunQueueScheduler(object):
def exceeds_max_pressure(self):
"""
Monitor the difference in total pressure at least once per second, if
BB_PRESSURE_MAX_{CPU|IO} are set, return True if above threshold.
BB_PRESSURE_MAX_{CPU|IO|MEMORY} are set, return True if above threshold.
"""
if self.check_pressure:
with open("/proc/pressure/cpu") as cpu_pressure_fds, open("/proc/pressure/io") as io_pressure_fds:
with open("/proc/pressure/cpu") as cpu_pressure_fds, \
open("/proc/pressure/io") as io_pressure_fds, \
open("/proc/pressure/memory") as memory_pressure_fds:
# extract "total" from /proc/pressure/{cpu|io}
curr_cpu_pressure = cpu_pressure_fds.readline().split()[4].split("=")[1]
curr_io_pressure = io_pressure_fds.readline().split()[4].split("=")[1]
curr_memory_pressure = memory_pressure_fds.readline().split()[4].split("=")[1]
exceeds_cpu_pressure = self.rq.max_cpu_pressure and (float(curr_cpu_pressure) - float(self.prev_cpu_pressure)) > self.rq.max_cpu_pressure
exceeds_io_pressure = self.rq.max_io_pressure and (float(curr_io_pressure) - float(self.prev_io_pressure)) > self.rq.max_io_pressure
exceeds_memory_pressure = self.rq.max_memory_pressure and (float(curr_memory_pressure) - float(self.prev_memory_pressure)) > self.rq.max_memory_pressure
now = time.time()
if now - self.prev_pressure_time > 1.0:
self.prev_cpu_pressure = curr_cpu_pressure
self.prev_io_pressure = curr_io_pressure
self.prev_memory_pressure = curr_memory_pressure
self.prev_pressure_time = now
return (exceeds_cpu_pressure or exceeds_io_pressure)
return (exceeds_cpu_pressure or exceeds_io_pressure or exceeds_memory_pressure)
return False
def next_buildable_task(self):
@@ -1748,6 +1757,7 @@ class RunQueueExecute:
self.scheduler = self.cfgData.getVar("BB_SCHEDULER") or "speed"
self.max_cpu_pressure = self.cfgData.getVar("BB_PRESSURE_MAX_CPU")
self.max_io_pressure = self.cfgData.getVar("BB_PRESSURE_MAX_IO")
self.max_memory_pressure = self.cfgData.getVar("BB_PRESSURE_MAX_MEMORY")
self.sq_buildable = set()
self.sq_running = set()
@@ -1798,6 +1808,13 @@ class RunQueueExecute:
if self.max_io_pressure > upper_limit:
bb.warn("Your build will be largely unregulated since BB_PRESSURE_MAX_IO is set to %s. It is very unlikely that such high pressure will be experienced." % (self.max_io_pressure))
if self.max_memory_pressure:
self.max_memory_pressure = float(self.max_memory_pressure)
if self.max_memory_pressure < lower_limit:
bb.fatal("Invalid BB_PRESSURE_MAX_MEMORY %s, minimum value is %s." % (self.max_memory_pressure, lower_limit))
if self.max_memory_pressure > upper_limit:
bb.warn("Your build will be largely unregulated since BB_PRESSURE_MAX_MEMORY is set to %s. It is very unlikely that such high pressure will be experienced." % (self.max_io_pressure))
# List of setscene tasks which we've covered
self.scenequeue_covered = set()
# List of tasks which are covered (including setscene ones)