mirror of
https://git.yoctoproject.org/poky
synced 2026-02-06 08:48:45 +01:00
a87aa7b7af
There are two problems with the _logged_communicate that are both caused as a result of buffering I/O issues: 1) log truncation when python fails 2) While a bitbake task is running it is impossible to see what is going on if it is only writing a small incremental log that is smaller than the buffer, or you get only a partial log, up until the task exists. It is worse in the case that stderr and stdout are separate file handles, because previous code blocks on the read of stdout and then stderr, serially. The right approach is simply to use select() to determine if there is data available and also flush the log before exiting. This is based on a patch from Jason Wessel <jason.wessel@windriver.com> with some changes to flush upon exit, abstract the non blocking file descriptor setup and drop the buffer size parameter. (Bitbake rev: 361fb71e907aa84c28ecec79fefc6ca39c39172f) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
569 lines
17 KiB
Python
569 lines
17 KiB
Python
# ex:ts=4:sw=4:sts=4:et
|
|
# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
|
|
#
|
|
# BitBake 'Build' implementation
|
|
#
|
|
# Core code for function execution and task handling in the
|
|
# BitBake build tools.
|
|
#
|
|
# Copyright (C) 2003, 2004 Chris Larson
|
|
#
|
|
# Based on Gentoo's portage.py.
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License version 2 as
|
|
# published by the Free Software Foundation.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License along
|
|
# with this program; if not, write to the Free Software Foundation, Inc.,
|
|
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
#
|
|
#Based on functions from the base bb module, Copyright 2003 Holger Schurig
|
|
|
|
import os
|
|
import sys
|
|
import logging
|
|
import shlex
|
|
import bb
|
|
import bb.msg
|
|
import bb.process
|
|
from contextlib import nested
|
|
from bb import data, event, utils
|
|
|
|
bblogger = logging.getLogger('BitBake')
|
|
logger = logging.getLogger('BitBake.Build')
|
|
|
|
NULL = open(os.devnull, 'r+')
|
|
|
|
|
|
# When we execute a python function we'd like certain things
|
|
# in all namespaces, hence we add them to __builtins__
|
|
# If we do not do this and use the exec globals, they will
|
|
# not be available to subfunctions.
|
|
__builtins__['bb'] = bb
|
|
__builtins__['os'] = os
|
|
|
|
class FuncFailed(Exception):
|
|
def __init__(self, name = None, logfile = None):
|
|
self.logfile = logfile
|
|
self.name = name
|
|
if name:
|
|
self.msg = 'Function failed: %s' % name
|
|
else:
|
|
self.msg = "Function failed"
|
|
|
|
def __str__(self):
|
|
if self.logfile and os.path.exists(self.logfile):
|
|
msg = ("%s (see %s for further information)" %
|
|
(self.msg, self.logfile))
|
|
else:
|
|
msg = self.msg
|
|
return msg
|
|
|
|
class TaskBase(event.Event):
|
|
"""Base class for task events"""
|
|
|
|
def __init__(self, t, d ):
|
|
self._task = t
|
|
self._package = d.getVar("PF", True)
|
|
event.Event.__init__(self)
|
|
self._message = "package %s: task %s: %s" % (d.getVar("PF", True), t, self.getDisplayName())
|
|
|
|
def getTask(self):
|
|
return self._task
|
|
|
|
def setTask(self, task):
|
|
self._task = task
|
|
|
|
def getDisplayName(self):
|
|
return bb.event.getName(self)[4:]
|
|
|
|
task = property(getTask, setTask, None, "task property")
|
|
|
|
class TaskStarted(TaskBase):
|
|
"""Task execution started"""
|
|
|
|
class TaskSucceeded(TaskBase):
|
|
"""Task execution completed"""
|
|
|
|
class TaskFailed(TaskBase):
|
|
"""Task execution failed"""
|
|
|
|
def __init__(self, task, logfile, metadata, errprinted = False):
|
|
self.logfile = logfile
|
|
self.errprinted = errprinted
|
|
super(TaskFailed, self).__init__(task, metadata)
|
|
|
|
class TaskFailedSilent(TaskBase):
|
|
"""Task execution failed (silently)"""
|
|
def __init__(self, task, logfile, metadata):
|
|
self.logfile = logfile
|
|
super(TaskFailedSilent, self).__init__(task, metadata)
|
|
|
|
def getDisplayName(self):
|
|
# Don't need to tell the user it was silent
|
|
return "Failed"
|
|
|
|
class TaskInvalid(TaskBase):
|
|
|
|
def __init__(self, task, metadata):
|
|
super(TaskInvalid, self).__init__(task, metadata)
|
|
self._message = "No such task '%s'" % task
|
|
|
|
|
|
class LogTee(object):
|
|
def __init__(self, logger, outfile):
|
|
self.outfile = outfile
|
|
self.logger = logger
|
|
self.name = self.outfile.name
|
|
|
|
def write(self, string):
|
|
self.logger.plain(string)
|
|
self.outfile.write(string)
|
|
|
|
def __enter__(self):
|
|
self.outfile.__enter__()
|
|
return self
|
|
|
|
def __exit__(self, *excinfo):
|
|
self.outfile.__exit__(*excinfo)
|
|
|
|
def __repr__(self):
|
|
return '<LogTee {0}>'.format(self.name)
|
|
def flush(self):
|
|
self.outfile.flush()
|
|
|
|
def exec_func(func, d, dirs = None):
|
|
"""Execute an BB 'function'"""
|
|
|
|
body = data.getVar(func, d)
|
|
if not body:
|
|
if body is None:
|
|
logger.warn("Function %s doesn't exist", func)
|
|
return
|
|
|
|
flags = data.getVarFlags(func, d)
|
|
cleandirs = flags.get('cleandirs')
|
|
if cleandirs:
|
|
for cdir in data.expand(cleandirs, d).split():
|
|
bb.utils.remove(cdir, True)
|
|
|
|
if dirs is None:
|
|
dirs = flags.get('dirs')
|
|
if dirs:
|
|
dirs = data.expand(dirs, d).split()
|
|
|
|
if dirs:
|
|
for adir in dirs:
|
|
bb.utils.mkdirhier(adir)
|
|
adir = dirs[-1]
|
|
else:
|
|
adir = data.getVar('B', d, 1)
|
|
bb.utils.mkdirhier(adir)
|
|
|
|
ispython = flags.get('python')
|
|
|
|
lockflag = flags.get('lockfiles')
|
|
if lockflag:
|
|
lockfiles = [data.expand(f, d) for f in lockflag.split()]
|
|
else:
|
|
lockfiles = None
|
|
|
|
tempdir = data.getVar('T', d, 1)
|
|
|
|
# or func allows items to be executed outside of the normal
|
|
# task set, such as buildhistory
|
|
task = data.getVar('BB_RUNTASK', d, 1) or func
|
|
if task == func:
|
|
taskfunc = task
|
|
else:
|
|
taskfunc = "%s.%s" % (task, func)
|
|
|
|
runfmt = data.getVar('BB_RUNFMT', d, 1) or "run.{func}.{pid}"
|
|
runfn = runfmt.format(taskfunc=taskfunc, task=task, func=func, pid=os.getpid())
|
|
runfile = os.path.join(tempdir, runfn)
|
|
bb.utils.mkdirhier(os.path.dirname(runfile))
|
|
|
|
with bb.utils.fileslocked(lockfiles):
|
|
if ispython:
|
|
exec_func_python(func, d, runfile, cwd=adir)
|
|
else:
|
|
exec_func_shell(func, d, runfile, cwd=adir)
|
|
|
|
_functionfmt = """
|
|
def {function}(d):
|
|
{body}
|
|
|
|
{function}(d)
|
|
"""
|
|
logformatter = bb.msg.BBLogFormatter("%(levelname)s: %(message)s")
|
|
def exec_func_python(func, d, runfile, cwd=None):
|
|
"""Execute a python BB 'function'"""
|
|
|
|
bbfile = d.getVar('FILE', True)
|
|
code = _functionfmt.format(function=func, body=d.getVar(func, True))
|
|
bb.utils.mkdirhier(os.path.dirname(runfile))
|
|
with open(runfile, 'w') as script:
|
|
script.write(code)
|
|
|
|
if cwd:
|
|
try:
|
|
olddir = os.getcwd()
|
|
except OSError:
|
|
olddir = None
|
|
os.chdir(cwd)
|
|
|
|
bb.debug(2, "Executing python function %s" % func)
|
|
|
|
try:
|
|
comp = utils.better_compile(code, func, bbfile)
|
|
utils.better_exec(comp, {"d": d}, code, bbfile)
|
|
except:
|
|
if sys.exc_info()[0] in (bb.parse.SkipPackage, bb.build.FuncFailed):
|
|
raise
|
|
|
|
raise FuncFailed(func, None)
|
|
finally:
|
|
bb.debug(2, "Python function %s finished" % func)
|
|
|
|
if cwd and olddir:
|
|
try:
|
|
os.chdir(olddir)
|
|
except OSError:
|
|
pass
|
|
|
|
def exec_func_shell(func, d, runfile, cwd=None):
|
|
"""Execute a shell function from the metadata
|
|
|
|
Note on directory behavior. The 'dirs' varflag should contain a list
|
|
of the directories you need created prior to execution. The last
|
|
item in the list is where we will chdir/cd to.
|
|
"""
|
|
|
|
# Don't let the emitted shell script override PWD
|
|
d.delVarFlag('PWD', 'export')
|
|
|
|
with open(runfile, 'w') as script:
|
|
script.write('#!/bin/sh -e\n')
|
|
data.emit_func(func, script, d)
|
|
|
|
if bb.msg.loggerVerboseLogs:
|
|
script.write("set -x\n")
|
|
if cwd:
|
|
script.write("cd %s\n" % cwd)
|
|
script.write("%s\n" % func)
|
|
|
|
os.chmod(runfile, 0775)
|
|
|
|
cmd = runfile
|
|
if d.getVarFlag(func, 'fakeroot'):
|
|
fakerootcmd = d.getVar('FAKEROOT', True)
|
|
if fakerootcmd:
|
|
cmd = [fakerootcmd, runfile]
|
|
|
|
if bb.msg.loggerDefaultVerbose:
|
|
logfile = LogTee(logger, sys.stdout)
|
|
else:
|
|
logfile = sys.stdout
|
|
|
|
bb.debug(2, "Executing shell function %s" % func)
|
|
|
|
try:
|
|
bb.process.run(cmd, shell=False, stdin=NULL, log=logfile)
|
|
except bb.process.CmdError:
|
|
logfn = d.getVar('BB_LOGFILE', True)
|
|
raise FuncFailed(func, logfn)
|
|
|
|
bb.debug(2, "Shell function %s finished" % func)
|
|
|
|
def _task_data(fn, task, d):
|
|
localdata = data.createCopy(d)
|
|
localdata.setVar('BB_FILENAME', fn)
|
|
localdata.setVar('BB_CURRENTTASK', task[3:])
|
|
localdata.setVar('OVERRIDES', 'task-%s:%s' %
|
|
(task[3:], d.getVar('OVERRIDES', False)))
|
|
localdata.finalize()
|
|
data.expandKeys(localdata)
|
|
return localdata
|
|
|
|
def _exec_task(fn, task, d, quieterr):
|
|
"""Execute a BB 'task'
|
|
|
|
Execution of a task involves a bit more setup than executing a function,
|
|
running it with its own local metadata, and with some useful variables set.
|
|
"""
|
|
if not data.getVarFlag(task, 'task', d):
|
|
event.fire(TaskInvalid(task, d), d)
|
|
logger.error("No such task: %s" % task)
|
|
return 1
|
|
|
|
logger.debug(1, "Executing task %s", task)
|
|
|
|
localdata = _task_data(fn, task, d)
|
|
tempdir = localdata.getVar('T', True)
|
|
if not tempdir:
|
|
bb.fatal("T variable not set, unable to build")
|
|
|
|
bb.utils.mkdirhier(tempdir)
|
|
|
|
# Determine the logfile to generate
|
|
logfmt = localdata.getVar('BB_LOGFMT', True) or 'log.{task}.{pid}'
|
|
logbase = logfmt.format(task=task, pid=os.getpid())
|
|
|
|
# Document the order of the tasks...
|
|
logorder = os.path.join(tempdir, 'log.task_order')
|
|
try:
|
|
logorderfile = file(logorder, 'a')
|
|
except OSError:
|
|
logger.exception("Opening log file '%s'", logorder)
|
|
pass
|
|
logorderfile.write('{0} ({1}): {2}\n'.format(task, os.getpid(), logbase))
|
|
logorderfile.close()
|
|
|
|
# Setup the courtesy link to the logfn
|
|
loglink = os.path.join(tempdir, 'log.{0}'.format(task))
|
|
logfn = os.path.join(tempdir, logbase)
|
|
if loglink:
|
|
bb.utils.remove(loglink)
|
|
|
|
try:
|
|
os.symlink(logbase, loglink)
|
|
except OSError:
|
|
pass
|
|
|
|
prefuncs = localdata.getVarFlag(task, 'prefuncs', expand=True)
|
|
postfuncs = localdata.getVarFlag(task, 'postfuncs', expand=True)
|
|
|
|
class ErrorCheckHandler(logging.Handler):
|
|
def __init__(self):
|
|
self.triggered = False
|
|
logging.Handler.__init__(self, logging.ERROR)
|
|
def emit(self, record):
|
|
self.triggered = True
|
|
|
|
# Handle logfiles
|
|
si = file('/dev/null', 'r')
|
|
try:
|
|
bb.utils.mkdirhier(os.path.dirname(logfn))
|
|
logfile = file(logfn, 'w')
|
|
except OSError:
|
|
logger.exception("Opening log file '%s'", logfn)
|
|
pass
|
|
|
|
# Dup the existing fds so we dont lose them
|
|
osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()]
|
|
oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()]
|
|
ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()]
|
|
|
|
# Replace those fds with our own
|
|
os.dup2(si.fileno(), osi[1])
|
|
os.dup2(logfile.fileno(), oso[1])
|
|
os.dup2(logfile.fileno(), ose[1])
|
|
|
|
# Ensure python logging goes to the logfile
|
|
handler = logging.StreamHandler(logfile)
|
|
handler.setFormatter(logformatter)
|
|
# Always enable full debug output into task logfiles
|
|
handler.setLevel(logging.DEBUG - 2)
|
|
bblogger.addHandler(handler)
|
|
|
|
errchk = ErrorCheckHandler()
|
|
bblogger.addHandler(errchk)
|
|
|
|
localdata.setVar('BB_LOGFILE', logfn)
|
|
localdata.setVar('BB_RUNTASK', task)
|
|
|
|
event.fire(TaskStarted(task, localdata), localdata)
|
|
try:
|
|
for func in (prefuncs or '').split():
|
|
exec_func(func, localdata)
|
|
exec_func(task, localdata)
|
|
for func in (postfuncs or '').split():
|
|
exec_func(func, localdata)
|
|
except FuncFailed as exc:
|
|
if quieterr:
|
|
event.fire(TaskFailedSilent(task, logfn, localdata), localdata)
|
|
else:
|
|
errprinted = errchk.triggered
|
|
logger.error(str(exc))
|
|
event.fire(TaskFailed(task, logfn, localdata, errprinted), localdata)
|
|
return 1
|
|
finally:
|
|
sys.stdout.flush()
|
|
sys.stderr.flush()
|
|
|
|
bblogger.removeHandler(handler)
|
|
|
|
# Restore the backup fds
|
|
os.dup2(osi[0], osi[1])
|
|
os.dup2(oso[0], oso[1])
|
|
os.dup2(ose[0], ose[1])
|
|
|
|
# Close the backup fds
|
|
os.close(osi[0])
|
|
os.close(oso[0])
|
|
os.close(ose[0])
|
|
si.close()
|
|
|
|
logfile.close()
|
|
if os.path.exists(logfn) and os.path.getsize(logfn) == 0:
|
|
logger.debug(2, "Zero size logfn %s, removing", logfn)
|
|
bb.utils.remove(logfn)
|
|
bb.utils.remove(loglink)
|
|
event.fire(TaskSucceeded(task, localdata), localdata)
|
|
|
|
if not localdata.getVarFlag(task, 'nostamp') and not localdata.getVarFlag(task, 'selfstamp'):
|
|
make_stamp(task, localdata)
|
|
|
|
return 0
|
|
|
|
def exec_task(fn, task, d):
|
|
try:
|
|
quieterr = False
|
|
if d.getVarFlag(task, "quieterrors") is not None:
|
|
quieterr = True
|
|
|
|
return _exec_task(fn, task, d, quieterr)
|
|
except Exception:
|
|
from traceback import format_exc
|
|
if not quieterr:
|
|
logger.error("Build of %s failed" % (task))
|
|
logger.error(format_exc())
|
|
failedevent = TaskFailed(task, None, d, True)
|
|
event.fire(failedevent, d)
|
|
return 1
|
|
|
|
def stamp_internal(taskname, d, file_name):
|
|
"""
|
|
Internal stamp helper function
|
|
Makes sure the stamp directory exists
|
|
Returns the stamp path+filename
|
|
|
|
In the bitbake core, d can be a CacheData and file_name will be set.
|
|
When called in task context, d will be a data store, file_name will not be set
|
|
"""
|
|
taskflagname = taskname
|
|
if taskname.endswith("_setscene") and taskname != "do_setscene":
|
|
taskflagname = taskname.replace("_setscene", "")
|
|
|
|
if file_name:
|
|
stamp = d.stamp_base[file_name].get(taskflagname) or d.stamp[file_name]
|
|
extrainfo = d.stamp_extrainfo[file_name].get(taskflagname) or ""
|
|
else:
|
|
stamp = d.getVarFlag(taskflagname, 'stamp-base', True) or d.getVar('STAMP', True)
|
|
file_name = d.getVar('BB_FILENAME', True)
|
|
extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info', True) or ""
|
|
|
|
if not stamp:
|
|
return
|
|
|
|
stamp = bb.parse.siggen.stampfile(stamp, file_name, taskname, extrainfo)
|
|
|
|
bb.utils.mkdirhier(os.path.dirname(stamp))
|
|
|
|
return stamp
|
|
|
|
def make_stamp(task, d, file_name = None):
|
|
"""
|
|
Creates/updates a stamp for a given task
|
|
(d can be a data dict or dataCache)
|
|
"""
|
|
stamp = stamp_internal(task, d, file_name)
|
|
# Remove the file and recreate to force timestamp
|
|
# change on broken NFS filesystems
|
|
if stamp:
|
|
bb.utils.remove(stamp)
|
|
f = open(stamp, "w")
|
|
f.close()
|
|
|
|
# If we're in task context, write out a signature file for each task
|
|
# as it completes
|
|
if not task.endswith("_setscene") and task != "do_setscene" and not file_name:
|
|
file_name = d.getVar('BB_FILENAME', True)
|
|
bb.parse.siggen.dump_sigtask(file_name, task, d.getVar('STAMP', True), True)
|
|
|
|
def del_stamp(task, d, file_name = None):
|
|
"""
|
|
Removes a stamp for a given task
|
|
(d can be a data dict or dataCache)
|
|
"""
|
|
stamp = stamp_internal(task, d, file_name)
|
|
bb.utils.remove(stamp)
|
|
|
|
def write_taint(task, d, file_name = None):
|
|
"""
|
|
Creates a "taint" file which will force the specified task and its
|
|
dependents to be re-run the next time by influencing the value of its
|
|
taskhash.
|
|
(d can be a data dict or dataCache)
|
|
"""
|
|
import uuid
|
|
if file_name:
|
|
taintfn = d.stamp[file_name] + '.' + task + '.taint'
|
|
else:
|
|
taintfn = d.getVar('STAMP', True) + '.' + task + '.taint'
|
|
bb.utils.mkdirhier(os.path.dirname(taintfn))
|
|
# The specific content of the taint file is not really important,
|
|
# we just need it to be random, so a random UUID is used
|
|
with open(taintfn, 'w') as taintf:
|
|
taintf.write(str(uuid.uuid4()))
|
|
|
|
def stampfile(taskname, d, file_name = None):
|
|
"""
|
|
Return the stamp for a given task
|
|
(d can be a data dict or dataCache)
|
|
"""
|
|
return stamp_internal(taskname, d, file_name)
|
|
|
|
def add_tasks(tasklist, d):
|
|
task_deps = data.getVar('_task_deps', d)
|
|
if not task_deps:
|
|
task_deps = {}
|
|
if not 'tasks' in task_deps:
|
|
task_deps['tasks'] = []
|
|
if not 'parents' in task_deps:
|
|
task_deps['parents'] = {}
|
|
|
|
for task in tasklist:
|
|
task = data.expand(task, d)
|
|
data.setVarFlag(task, 'task', 1, d)
|
|
|
|
if not task in task_deps['tasks']:
|
|
task_deps['tasks'].append(task)
|
|
|
|
flags = data.getVarFlags(task, d)
|
|
def getTask(name):
|
|
if not name in task_deps:
|
|
task_deps[name] = {}
|
|
if name in flags:
|
|
deptask = data.expand(flags[name], d)
|
|
task_deps[name][task] = deptask
|
|
getTask('depends')
|
|
getTask('rdepends')
|
|
getTask('deptask')
|
|
getTask('rdeptask')
|
|
getTask('recrdeptask')
|
|
getTask('nostamp')
|
|
getTask('fakeroot')
|
|
getTask('noexec')
|
|
getTask('umask')
|
|
task_deps['parents'][task] = []
|
|
for dep in flags['deps']:
|
|
dep = data.expand(dep, d)
|
|
task_deps['parents'][task].append(dep)
|
|
|
|
# don't assume holding a reference
|
|
data.setVar('_task_deps', task_deps, d)
|
|
|
|
def remove_task(task, kill, d):
|
|
"""Remove an BB 'task'.
|
|
|
|
If kill is 1, also remove tasks that depend on this task."""
|
|
|
|
data.delVarFlag(task, 'task', d)
|