mirrors/poky
1
0
Fork 0
mirror of https://git.yoctoproject.org/poky synced 2026-02-11 19:23:03 +01:00
Code Issues Packages Projects Releases Wiki Activity
Files
42b9bdfd76d88f4155eb46f2b8acdcc4e4192147
poky/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
Elliot Smith 1f7f077466 bitbake: buildinfohelper: unset brbe variable when build finishes 
The brbe variable is set on the bitbake server when the Toaster
UI starts. This enables Toaster to associate events with the
correct build and build environment.

However, the brbe variable is also used when a build starts to
identify whether a new build needs to be created, or an existing
one looked up. This causes a bug for command-line builds which
happen after a Toaster-triggered build: because the brbe variable
is never unset on the server or the buildinfohelper, the new
command-line build events are treated as originating from the
previous build.

Ensure the brbe variable is reset when the buildinfohelper "closes"
a build, so that each build then either sets the brbe variable
(Toaster-triggered builds) or leaves it blank (command-line builds).

Also modify the localhostbecontroller so that the brbe variable
is not set on the server and not looked up from the server. This
ensures that it is only set when the triggerBuild() method is
called, and that it remains as None for command-line builds.

[YOCTO #9021]

(Bitbake rev: 4a6a8d0074f62208d843b06344be31ae73d9b745)

Signed-off-by: Elliot Smith <elliot.smith@intel.com>
Signed-off-by: Michael Wood <michael.g.wood@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2016-02-11 22:34:19 +00:00

339 lines
15 KiB
Python
Raw Blame History

#
# ex:ts=4:sw=4:sts=4:et
# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
#
# BitBake Toaster Implementation
#
# Copyright (C) 2014 Intel Corporation
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import os
import sys
import re
import shutil
from django.db import transaction
from django.db.models import Q
from bldcontrol.models import BuildEnvironment, BRLayer, BRVariable, BRTarget, BRBitbake
from orm.models import CustomImageRecipe, Layer, Layer_Version, ProjectLayer
import subprocess
from toastermain import settings
from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException
import logging
logger = logging.getLogger("toaster")
from pprint import pprint, pformat
class LocalhostBEController(BuildEnvironmentController):
""" Implementation of the BuildEnvironmentController for the localhost;
this controller manages the default build directory,
the server setup and system start and stop for the localhost-type build environment
"""
def __init__(self, be):
super(LocalhostBEController, self).__init__(be)
self.pokydirname = None
self.islayerset = False
def _shellcmd(self, command, cwd = None):
if cwd is None:
cwd = self.be.sourcedir
logger.debug("lbc_shellcmmd: (%s) %s" % (cwd, command))
p = subprocess.Popen(command, cwd = cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
(out,err) = p.communicate()
p.wait()
if p.returncode:
if len(err) == 0:
err = "command: %s \n%s" % (command, out)
else:
err = "command: %s \n%s" % (command, err)
logger.warn("localhostbecontroller: shellcmd error %s" % err)
raise ShellCmdException(err)
else:
logger.debug("localhostbecontroller: shellcmd success")
return out
def startBBServer(self):
assert self.pokydirname and os.path.exists(self.pokydirname)
assert self.islayerset
# find our own toasterui listener/bitbake
from toaster.bldcontrol.management.commands.loadconf import _reduce_canon_path
toaster = _reduce_canon_path(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../../bin/toaster"))
assert os.path.exists(toaster) and os.path.isfile(toaster)
# restart bitbake server and toastergui observer
self._shellcmd("bash -c 'source %s restart-bitbake'" % toaster, self.be.builddir)
logger.debug("localhostbecontroller: restarted bitbake server")
# read port number from bitbake.lock
self.be.bbport = ""
bblock = os.path.join(self.be.builddir, 'bitbake.lock')
if os.path.exists(bblock):
with open(bblock) as fplock:
for line in fplock:
if ":" in line:
self.be.bbport = line.split(":")[-1].strip()
logger.debug("localhostbecontroller: bitbake port %s", self.be.bbport)
break
if not self.be.bbport:
raise BuildSetupException("localhostbecontroller: can't read bitbake port from %s" % bblock)
self.be.bbaddress = "localhost"
self.be.bbstate = BuildEnvironment.SERVER_STARTED
self.be.save()
def getGitCloneDirectory(self, url, branch):
"""Construct unique clone directory name out of url and branch."""
if branch != "HEAD":
return "_toaster_clones/_%s_%s" % (re.sub('[:/@%]', '_', url), branch)
# word of attention; this is a localhost-specific issue; only on the localhost we expect to have "HEAD" releases
# which _ALWAYS_ means the current poky checkout
from os.path import dirname as DN
local_checkout_path = DN(DN(DN(DN(DN(os.path.abspath(__file__))))))
#logger.debug("localhostbecontroller: using HEAD checkout in %s" % local_checkout_path)
return local_checkout_path
def setLayers(self, bitbake, layers, targets):
""" a word of attention: by convention, the first layer for any build will be poky! """
assert self.be.sourcedir is not None
# set layers in the layersource
# 1. get a list of repos with branches, and map dirpaths for each layer
gitrepos = {}
gitrepos[(bitbake.giturl, bitbake.commit)] = []
gitrepos[(bitbake.giturl, bitbake.commit)].append( ("bitbake", bitbake.dirpath) )
for layer in layers:
# we don't process local URLs
if layer.giturl.startswith("file://"):
continue
if not (layer.giturl, layer.commit) in gitrepos:
gitrepos[(layer.giturl, layer.commit)] = []
gitrepos[(layer.giturl, layer.commit)].append( (layer.name, layer.dirpath) )
logger.debug("localhostbecontroller, our git repos are %s" % pformat(gitrepos))
# 2. Note for future use if the current source directory is a
# checked-out git repos that could match a layer's vcs_url and therefore
# be used to speed up cloning (rather than fetching it again).
cached_layers = {}
try:
for remotes in self._shellcmd("git remote -v", self.be.sourcedir).split("\n"):
try:
remote = remotes.split("\t")[1].split(" ")[0]
if remote not in cached_layers:
cached_layers[remote] = self.be.sourcedir
except IndexError:
pass
except ShellCmdException:
# ignore any errors in collecting git remotes this is an optional
# step
pass
logger.info("Using pre-checked out source for layer %s", cached_layers)
layerlist = []
# 3. checkout the repositories
for giturl, commit in gitrepos.keys():
localdirname = os.path.join(self.be.sourcedir, self.getGitCloneDirectory(giturl, commit))
logger.debug("localhostbecontroller: giturl %s:%s checking out in current directory %s" % (giturl, commit, localdirname))
# make sure our directory is a git repository
if os.path.exists(localdirname):
localremotes = self._shellcmd("git remote -v", localdirname)
if not giturl in localremotes:
raise BuildSetupException("Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something." % (localdirname, ", ".join(localremotes.split("\n")), giturl))
else:
if giturl in cached_layers:
logger.debug("localhostbecontroller git-copying %s to %s" % (cached_layers[giturl], localdirname))
self._shellcmd("git clone \"%s\" \"%s\"" % (cached_layers[giturl], localdirname))
self._shellcmd("git remote remove origin", localdirname)
self._shellcmd("git remote add origin \"%s\"" % giturl, localdirname)
else:
logger.debug("localhostbecontroller: cloning %s in %s" % (giturl, localdirname))
self._shellcmd('git clone "%s" "%s"' % (giturl, localdirname))
# branch magic name "HEAD" will inhibit checkout
if commit != "HEAD":
logger.debug("localhostbecontroller: checking out commit %s to %s " % (commit, localdirname))
ref = commit if re.match('^[a-fA-F0-9]+$', commit) else 'origin/%s' % commit
self._shellcmd('git fetch --all && git reset --hard "%s"' % ref, localdirname)
# take the localdirname as poky dir if we can find the oe-init-build-env
if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")):
logger.debug("localhostbecontroller: selected poky dir name %s" % localdirname)
self.pokydirname = localdirname
# make sure we have a working bitbake
if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')):
logger.debug("localhostbecontroller: checking bitbake into the poky dirname %s " % self.pokydirname)
self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbake.commit, bitbake.giturl, os.path.join(self.pokydirname, 'bitbake')))
# verify our repositories
for name, dirpath in gitrepos[(giturl, commit)]:
localdirpath = os.path.join(localdirname, dirpath)
logger.debug("localhostbecontroller: localdirpath expected '%s'" % localdirpath)
if not os.path.exists(localdirpath):
raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit))
if name != "bitbake":
layerlist.append(localdirpath.rstrip("/"))
logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist))
# 4. update the bblayers.conf
bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf")
if not os.path.exists(bblayerconf):
raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf)
# 5. create custom layer and add custom recipes to it
layerpath = os.path.join(self.be.sourcedir, "_meta-toaster-custom")
if os.path.isdir(layerpath):
shutil.rmtree(layerpath) # remove leftovers from previous builds
for target in targets:
try:
customrecipe = CustomImageRecipe.objects.get(name=target.target,
project=bitbake.req.project)
except CustomImageRecipe.DoesNotExist:
continue # not a custom recipe, skip
# create directory structure
for name in ("conf", "recipes"):
path = os.path.join(layerpath, name)
if not os.path.isdir(path):
os.makedirs(path)
# create layer.oonf
config = os.path.join(layerpath, "conf", "layer.conf")
if not os.path.isfile(config):
with open(config, "w") as conf:
conf.write('BBPATH .= ":${LAYERDIR}"\nBBFILES += "${LAYERDIR}/recipes/*.bb"\n')
# Update the Layer_Version dirpath that has our base_recipe in
# to be able to read the base recipe to then generate the
# custom recipe.
br_layer_base_recipe = layers.get(
layer_version=customrecipe.base_recipe.layer_version)
br_layer_base_dirpath = \
os.path.join(self.be.sourcedir,
self.getGitCloneDirectory(
br_layer_base_recipe.giturl,
br_layer_base_recipe.commit),
customrecipe.base_recipe.layer_version.dirpath
)
customrecipe.base_recipe.layer_version.dirpath = \
br_layer_base_dirpath
customrecipe.base_recipe.layer_version.save()
# create recipe
recipe_path = \
os.path.join(layerpath, "recipes", "%s.bb" % target.target)
with open(recipe_path, "w") as recipef:
recipef.write(customrecipe.generate_recipe_file_contents())
# Update the layer and recipe objects
customrecipe.layer_version.dirpath = layerpath
customrecipe.layer_version.save()
customrecipe.file_path = recipe_path
customrecipe.save()
# create *Layer* objects needed for build machinery to work
BRLayer.objects.get_or_create(req=target.req,
name=layer.name,
dirpath=layerpath,
giturl="file://%s" % layerpath)
if os.path.isdir(layerpath):
layerlist.append(layerpath)
BuildEnvironmentController._updateBBLayers(bblayerconf, layerlist)
self.islayerset = True
return True
def readServerLogFile(self):
return open(os.path.join(self.be.builddir, "toaster_server.log"), "r").read()
def release(self):
assert self.be.sourcedir and os.path.exists(self.be.builddir)
import shutil
shutil.rmtree(os.path.join(self.be.sourcedir, "build"))
assert not os.path.exists(self.be.builddir)
def triggerBuild(self, bitbake, layers, variables, targets):
# set up the build environment with the needed layers
self.setLayers(bitbake, layers, targets)
# write configuration file
filepath = os.path.join(self.be.builddir, "conf/toaster.conf")
with open(filepath, 'w') as conf:
for var in variables:
conf.write('%s="%s"\n' % (var.name, var.value))
conf.write('INHERIT+="toaster buildhistory"')
# get the bb server running with the build req id and build env id
bbctrl = self.getBBController()
# set variables; TOASTER_BRBE is not set on the server, as this
# causes events from command-line builds to be attached to the last
# Toaster-triggered build; instead, TOASTER_BRBE is fired as an event so
# that toasterui can set it on the buildinfohelper;
# see https://bugzilla.yoctoproject.org/show_bug.cgi?id=9021
for var in variables:
if var.name == 'TOASTER_BRBE':
bbctrl.triggerEvent('bb.event.MetadataEvent("SetBRBE", "%s")' \
% var.value)
else:
bbctrl.setVariable(var.name, var.value)
# Add 'toaster' and 'buildhistory' to INHERIT variable
inherit = {item.strip() for item in bbctrl.getVariable('INHERIT').split()}
inherit = inherit.union(["toaster", "buildhistory"])
bbctrl.setVariable('INHERIT', ' '.join(inherit))
# trigger the build command
task = reduce(lambda x, y: x if len(y)== 0 else y, map(lambda y: y.task, targets))
if len(task) == 0:
task = None
bbctrl.build(list(map(lambda x:x.target, targets)), task)
logger.debug("localhostbecontroller: Build launched, exiting. Follow build logs at %s/toaster_ui.log" % self.be.builddir)
# disconnect from the server
bbctrl.disconnect()
Reference in New Issue View Git Blame Copy Permalink