Files
poky/bitbake/lib/bb/parse/__init__.py
Richard Purdie 37c31a5adc bitbake: lib: Drop inotify support and replace with mtime checks
With the flush in serverlog() removed and a memory resident bitbake with a
60s timeout, the following could fail in strange ways:

rm bitbake-cookerdaemon.log
bitbake-layers add-layer ../meta-virtualization/
bitbake-layers add-layer ../meta-openembedded/meta-oe/
bitbake -m

specifically that it might error adding meta-oe with an error related to meta-virt.

This clearly shows that whilst bblayers.conf was modified, bitbake was not
recognising that. This would fit with the random autobuilder issues seen when
the serverlog flush() call was removed.

The issue appears to be that you have no way to "sync()" the inotify events with
the command stream coming over the socket. There is no way to know if there are
changes in the IO queue which bitbake needs to wait for before proceeding with
the next command.

I did experiment with os.sync() and fsync on the inotify fd, however nothing
addressed the issue. Since it is extremely important we have accurate cache data,
the only realistic thing to do is to switch to stat() calls and check mtime.

For bitbake commands, this is straightforward since we can revalidate the cache
upon new connections/commands. For tinfoil this is problematic and we need to
introduce and explict command "revalidateCaches" that the code can use to force
bitbake to re-check it's cache validity. I've exposed this through tinfoil with
a new "modified_files" function.

So, this patch:

a) drops inotify support within bitbake's cooker/server and switch to using mtime
b) requires a new function call in tinfoil when metadata has been modified

(Bitbake rev: da3ec3801bdb80180b3f1ac24edb27a698415ff7)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2023-09-18 11:35:05 +01:00

176 lines
4.7 KiB
Python

"""
BitBake Parsers
File parsers for the BitBake build tools.
"""
# Copyright (C) 2003, 2004 Chris Larson
# Copyright (C) 2003, 2004 Phil Blundell
#
# SPDX-License-Identifier: GPL-2.0-only
#
# Based on functions from the base bb module, Copyright 2003 Holger Schurig
#
handlers = []
import errno
import logging
import os
import stat
import bb
import bb.utils
import bb.siggen
logger = logging.getLogger("BitBake.Parsing")
class ParseError(Exception):
"""Exception raised when parsing fails"""
def __init__(self, msg, filename, lineno=0):
self.msg = msg
self.filename = filename
self.lineno = lineno
Exception.__init__(self, msg, filename, lineno)
def __str__(self):
if self.lineno:
return "ParseError at %s:%d: %s" % (self.filename, self.lineno, self.msg)
else:
return "ParseError in %s: %s" % (self.filename, self.msg)
class SkipRecipe(Exception):
"""Exception raised to skip this recipe"""
class SkipPackage(SkipRecipe):
"""Exception raised to skip this recipe (use SkipRecipe in new code)"""
__mtime_cache = {}
def cached_mtime(f):
if f not in __mtime_cache:
__mtime_cache[f] = os.stat(f)[stat.ST_MTIME]
return __mtime_cache[f]
def cached_mtime_noerror(f):
if f not in __mtime_cache:
try:
__mtime_cache[f] = os.stat(f)[stat.ST_MTIME]
except OSError:
return 0
return __mtime_cache[f]
def check_mtime(f, mtime):
try:
current_mtime = os.stat(f)[stat.ST_MTIME]
__mtime_cache[f] = current_mtime
except OSError:
current_mtime = 0
return current_mtime == mtime
def update_mtime(f):
try:
__mtime_cache[f] = os.stat(f)[stat.ST_MTIME]
except OSError:
if f in __mtime_cache:
del __mtime_cache[f]
return 0
return __mtime_cache[f]
def update_cache(f):
if f in __mtime_cache:
logger.debug("Updating mtime cache for %s" % f)
update_mtime(f)
def clear_cache():
global __mtime_cache
__mtime_cache = {}
def mark_dependency(d, f):
if f.startswith('./'):
f = "%s/%s" % (os.getcwd(), f[2:])
deps = (d.getVar('__depends', False) or [])
s = (f, cached_mtime_noerror(f))
if s not in deps:
deps.append(s)
d.setVar('__depends', deps)
def check_dependency(d, f):
s = (f, cached_mtime_noerror(f))
deps = (d.getVar('__depends', False) or [])
return s in deps
def supports(fn, data):
"""Returns true if we have a handler for this file, false otherwise"""
for h in handlers:
if h['supports'](fn, data):
return 1
return 0
def handle(fn, data, include=0, baseconfig=False):
"""Call the handler that is appropriate for this file"""
for h in handlers:
if h['supports'](fn, data):
with data.inchistory.include(fn):
return h['handle'](fn, data, include, baseconfig)
raise ParseError("not a BitBake file", fn)
def init(fn, data):
for h in handlers:
if h['supports'](fn):
return h['init'](data)
def init_parser(d):
if hasattr(bb.parse, "siggen"):
bb.parse.siggen.exit()
bb.parse.siggen = bb.siggen.init(d)
def resolve_file(fn, d):
if not os.path.isabs(fn):
bbpath = d.getVar("BBPATH")
newfn, attempts = bb.utils.which(bbpath, fn, history=True)
for af in attempts:
mark_dependency(d, af)
if not newfn:
raise IOError(errno.ENOENT, "file %s not found in %s" % (fn, bbpath))
fn = newfn
else:
mark_dependency(d, fn)
if not os.path.isfile(fn):
raise IOError(errno.ENOENT, "file %s not found" % fn)
return fn
# Used by OpenEmbedded metadata
__pkgsplit_cache__={}
def vars_from_file(mypkg, d):
if not mypkg or not mypkg.endswith((".bb", ".bbappend")):
return (None, None, None)
if mypkg in __pkgsplit_cache__:
return __pkgsplit_cache__[mypkg]
myfile = os.path.splitext(os.path.basename(mypkg))
parts = myfile[0].split('_')
__pkgsplit_cache__[mypkg] = parts
if len(parts) > 3:
raise ParseError("Unable to generate default variables from filename (too many underscores)", mypkg)
exp = 3 - len(parts)
tmplist = []
while exp != 0:
exp -= 1
tmplist.append(None)
parts.extend(tmplist)
return parts
def get_file_depends(d):
'''Return the dependent files'''
dep_files = []
depends = d.getVar('__base_depends', False) or []
depends = depends + (d.getVar('__depends', False) or [])
for (fn, _) in depends:
dep_files.append(os.path.abspath(fn))
return " ".join(dep_files)
from bb.parse.parse_py import __version__, ConfHandler, BBHandler