bitbake: server/process: Improve idle thread exception handling

If the inotifier code has an exception, bitbake currently hangs. Catch any
exception and exit if seen. Also check the idle thread is alive and exit
if it disappears. This should stop bitbake hanging if such a situation arises
in future such as this example:

3323260 21:48:31.554468 Running command ['getVariable', 'BBINCLUDELOGS']
Exception in thread Thread-1 (idle_thread):
Traceback (most recent call last):
  File "/usr/lib64/python3.10/threading.py", line 1016, in _bootstrap_inner
    self.run()
  File "/usr/lib64/python3.10/threading.py", line 953, in run
    self._target(*self._args, **self._kwargs)
  File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/bitbake/lib/bb/server/process.py", line 408, in idle_thread
    self.cooker.process_inotify_updates()
  File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/bitbake/lib/bb/cooker.py", line 256, in process_inotify_updates
    n.read_events()
  File "/home/pokybuild/yocto-worker/oe-selftest-fedora/build/bitbake/lib/pyinotify.py", line 1207, in read_events
    if fcntl.ioctl(self._fd, termios.FIONREAD, buf_, 1) == -1:
OSError: [Errno 9] Bad file descriptor
3323260 21:48:32.206995 Command Completed (socket: True)

(Bitbake rev: 358b5b02d5de1ab0f98104c4ec4953e46999b9a5)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Richard Purdie
2023-02-20 09:30:44 +00:00
parent 7244bf2457
commit 76db0baba2

View File

@@ -405,7 +405,11 @@ class ProcessServer():
nextsleep = 0.1
fds = []
self.cooker.process_inotify_updates()
try:
self.cooker.process_inotify_updates()
except Exception as exc:
serverlog("Exception %s in inofify updates broke the idle_thread, exiting" % traceback.format_exc())
self.quit = True
with bb.utils.lock_timeout(self._idlefuncsLock):
items = list(self._idlefuns.items())
@@ -473,6 +477,10 @@ class ProcessServer():
if not self.idle:
self.idle = threading.Thread(target=self.idle_thread)
self.idle.start()
elif self.idle and not self.idle.is_alive():
serverlog("Idle thread terminated, main thread exiting too")
bb.error("Idle thread terminated, main thread exiting too")
self.quit = True
if nextsleep is not None:
if self.xmlrpc: