mirror of
https://git.yoctoproject.org/poky
synced 2026-02-20 08:29:42 +01:00
e44bb5b5c8
Toaster depends on pokydirname for identifying the location of the oe-init-build-env script (and there might be other purposes in the future). The problem with current approach is that it only checks/sets the variable with git based repos, whereas toaster provides mechanisms to allow having layers that are all locally available. The evaluation of the variable fails in such scenarios, so use a more flexible mechanism in this case and try to locate poky in the local layers as well, if not already set. [YOCTO #12891] (Bitbake rev: 971c728075af05e71edfd8e5212728c3dd0787b6) Signed-off-by: Awais Belal <awais.belal@mentor.com> Signed-off-by: David Reyna <David.Reyna@windriver.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
414 lines
18 KiB
Python
414 lines
18 KiB
Python
#
|
|
# ex:ts=4:sw=4:sts=4:et
|
|
# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
|
|
#
|
|
# BitBake Toaster Implementation
|
|
#
|
|
# Copyright (C) 2014 Intel Corporation
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License version 2 as
|
|
# published by the Free Software Foundation.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License along
|
|
# with this program; if not, write to the Free Software Foundation, Inc.,
|
|
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
|
|
|
|
import os
|
|
import sys
|
|
import re
|
|
import shutil
|
|
import time
|
|
from django.db import transaction
|
|
from django.db.models import Q
|
|
from bldcontrol.models import BuildEnvironment, BRLayer, BRVariable, BRTarget, BRBitbake
|
|
from orm.models import CustomImageRecipe, Layer, Layer_Version, ProjectLayer, ToasterSetting
|
|
import subprocess
|
|
|
|
from toastermain import settings
|
|
|
|
from bldcontrol.bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException, BitbakeController
|
|
|
|
import logging
|
|
logger = logging.getLogger("toaster")
|
|
|
|
from pprint import pprint, pformat
|
|
|
|
class LocalhostBEController(BuildEnvironmentController):
|
|
""" Implementation of the BuildEnvironmentController for the localhost;
|
|
this controller manages the default build directory,
|
|
the server setup and system start and stop for the localhost-type build environment
|
|
|
|
"""
|
|
|
|
def __init__(self, be):
|
|
super(LocalhostBEController, self).__init__(be)
|
|
self.pokydirname = None
|
|
self.islayerset = False
|
|
|
|
def _shellcmd(self, command, cwd=None, nowait=False,env=None):
|
|
if cwd is None:
|
|
cwd = self.be.sourcedir
|
|
if env is None:
|
|
env=os.environ.copy()
|
|
|
|
logger.debug("lbc_shellcmd: (%s) %s" % (cwd, command))
|
|
p = subprocess.Popen(command, cwd = cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
|
|
if nowait:
|
|
return
|
|
(out,err) = p.communicate()
|
|
p.wait()
|
|
if p.returncode:
|
|
if len(err) == 0:
|
|
err = "command: %s \n%s" % (command, out)
|
|
else:
|
|
err = "command: %s \n%s" % (command, err)
|
|
logger.warning("localhostbecontroller: shellcmd error %s" % err)
|
|
raise ShellCmdException(err)
|
|
else:
|
|
logger.debug("localhostbecontroller: shellcmd success")
|
|
return out.decode('utf-8')
|
|
|
|
def getGitCloneDirectory(self, url, branch):
|
|
"""Construct unique clone directory name out of url and branch."""
|
|
if branch != "HEAD":
|
|
return "_toaster_clones/_%s_%s" % (re.sub('[:/@+%]', '_', url), branch)
|
|
|
|
# word of attention; this is a localhost-specific issue; only on the localhost we expect to have "HEAD" releases
|
|
# which _ALWAYS_ means the current poky checkout
|
|
from os.path import dirname as DN
|
|
local_checkout_path = DN(DN(DN(DN(DN(os.path.abspath(__file__))))))
|
|
#logger.debug("localhostbecontroller: using HEAD checkout in %s" % local_checkout_path)
|
|
return local_checkout_path
|
|
|
|
|
|
def setCloneStatus(self,bitbake,status,total,current):
|
|
bitbake.req.build.repos_cloned=current
|
|
bitbake.req.build.repos_to_clone=total
|
|
bitbake.req.build.save()
|
|
|
|
def setLayers(self, bitbake, layers, targets):
|
|
""" a word of attention: by convention, the first layer for any build will be poky! """
|
|
|
|
assert self.be.sourcedir is not None
|
|
|
|
layerlist = []
|
|
nongitlayerlist = []
|
|
git_env = os.environ.copy()
|
|
# (note: add custom environment settings here)
|
|
|
|
# set layers in the layersource
|
|
|
|
# 1. get a list of repos with branches, and map dirpaths for each layer
|
|
gitrepos = {}
|
|
|
|
# if we're using a remotely fetched version of bitbake add its git
|
|
# details to the list of repos to clone
|
|
if bitbake.giturl and bitbake.commit:
|
|
gitrepos[(bitbake.giturl, bitbake.commit)] = []
|
|
gitrepos[(bitbake.giturl, bitbake.commit)].append(
|
|
("bitbake", bitbake.dirpath))
|
|
|
|
for layer in layers:
|
|
# We don't need to git clone the layer for the CustomImageRecipe
|
|
# as it's generated by us layer on if needed
|
|
if CustomImageRecipe.LAYER_NAME in layer.name:
|
|
continue
|
|
|
|
# If we have local layers then we don't need clone them
|
|
# For local layers giturl will be empty
|
|
if not layer.giturl:
|
|
nongitlayerlist.append(layer.layer_version.layer.local_source_dir)
|
|
continue
|
|
|
|
if not (layer.giturl, layer.commit) in gitrepos:
|
|
gitrepos[(layer.giturl, layer.commit)] = []
|
|
gitrepos[(layer.giturl, layer.commit)].append( (layer.name, layer.dirpath) )
|
|
|
|
|
|
logger.debug("localhostbecontroller, our git repos are %s" % pformat(gitrepos))
|
|
|
|
|
|
# 2. Note for future use if the current source directory is a
|
|
# checked-out git repos that could match a layer's vcs_url and therefore
|
|
# be used to speed up cloning (rather than fetching it again).
|
|
|
|
cached_layers = {}
|
|
|
|
try:
|
|
for remotes in self._shellcmd("git remote -v", self.be.sourcedir,env=git_env).split("\n"):
|
|
try:
|
|
remote = remotes.split("\t")[1].split(" ")[0]
|
|
if remote not in cached_layers:
|
|
cached_layers[remote] = self.be.sourcedir
|
|
except IndexError:
|
|
pass
|
|
except ShellCmdException:
|
|
# ignore any errors in collecting git remotes this is an optional
|
|
# step
|
|
pass
|
|
|
|
logger.info("Using pre-checked out source for layer %s", cached_layers)
|
|
|
|
# 3. checkout the repositories
|
|
clone_count=0
|
|
clone_total=len(gitrepos.keys())
|
|
self.setCloneStatus(bitbake,'Started',clone_total,clone_count)
|
|
for giturl, commit in gitrepos.keys():
|
|
self.setCloneStatus(bitbake,'progress',clone_total,clone_count)
|
|
clone_count += 1
|
|
|
|
localdirname = os.path.join(self.be.sourcedir, self.getGitCloneDirectory(giturl, commit))
|
|
logger.debug("localhostbecontroller: giturl %s:%s checking out in current directory %s" % (giturl, commit, localdirname))
|
|
|
|
# see if our directory is a git repository
|
|
if os.path.exists(localdirname):
|
|
try:
|
|
localremotes = self._shellcmd("git remote -v",
|
|
localdirname,env=git_env)
|
|
if not giturl in localremotes and commit != 'HEAD':
|
|
raise BuildSetupException("Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something." % (localdirname, ", ".join(localremotes.split("\n")), giturl))
|
|
except ShellCmdException:
|
|
# our localdirname might not be a git repository
|
|
#- that's fine
|
|
pass
|
|
else:
|
|
if giturl in cached_layers:
|
|
logger.debug("localhostbecontroller git-copying %s to %s" % (cached_layers[giturl], localdirname))
|
|
self._shellcmd("git clone \"%s\" \"%s\"" % (cached_layers[giturl], localdirname),env=git_env)
|
|
self._shellcmd("git remote remove origin", localdirname,env=git_env)
|
|
self._shellcmd("git remote add origin \"%s\"" % giturl, localdirname,env=git_env)
|
|
else:
|
|
logger.debug("localhostbecontroller: cloning %s in %s" % (giturl, localdirname))
|
|
self._shellcmd('git clone "%s" "%s"' % (giturl, localdirname),env=git_env)
|
|
|
|
# branch magic name "HEAD" will inhibit checkout
|
|
if commit != "HEAD":
|
|
logger.debug("localhostbecontroller: checking out commit %s to %s " % (commit, localdirname))
|
|
ref = commit if re.match('^[a-fA-F0-9]+$', commit) else 'origin/%s' % commit
|
|
self._shellcmd('git fetch && git reset --hard "%s"' % ref, localdirname,env=git_env)
|
|
|
|
# take the localdirname as poky dir if we can find the oe-init-build-env
|
|
if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")):
|
|
logger.debug("localhostbecontroller: selected poky dir name %s" % localdirname)
|
|
self.pokydirname = localdirname
|
|
|
|
# make sure we have a working bitbake
|
|
if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')):
|
|
logger.debug("localhostbecontroller: checking bitbake into the poky dirname %s " % self.pokydirname)
|
|
self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbake.commit, bitbake.giturl, os.path.join(self.pokydirname, 'bitbake')),env=git_env)
|
|
|
|
# verify our repositories
|
|
for name, dirpath in gitrepos[(giturl, commit)]:
|
|
localdirpath = os.path.join(localdirname, dirpath)
|
|
logger.debug("localhostbecontroller: localdirpath expected '%s'" % localdirpath)
|
|
if not os.path.exists(localdirpath):
|
|
raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit))
|
|
|
|
if name != "bitbake":
|
|
layerlist.append(localdirpath.rstrip("/"))
|
|
|
|
self.setCloneStatus(bitbake,'complete',clone_total,clone_count)
|
|
logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist))
|
|
|
|
# Resolve self.pokydirname if not resolved yet, consider the scenario
|
|
# where all layers are local, that's the else clause
|
|
if self.pokydirname is None:
|
|
if os.path.exists(os.path.join(self.be.sourcedir, "oe-init-build-env")):
|
|
logger.debug("localhostbecontroller: selected poky dir name %s" % self.be.sourcedir)
|
|
self.pokydirname = self.be.sourcedir
|
|
else:
|
|
# Alternatively, scan local layers for relative "oe-init-build-env" location
|
|
for layer in layers:
|
|
if os.path.exists(os.path.join(layer.layer_version.layer.local_source_dir,"..","oe-init-build-env")):
|
|
logger.debug("localhostbecontroller, setting pokydirname to %s" % (layer.layer_version.layer.local_source_dir))
|
|
self.pokydirname = os.path.join(layer.layer_version.layer.local_source_dir,"..")
|
|
break
|
|
else:
|
|
logger.error("pokydirname is not set, you will run into trouble!")
|
|
|
|
# 5. create custom layer and add custom recipes to it
|
|
for target in targets:
|
|
try:
|
|
customrecipe = CustomImageRecipe.objects.get(
|
|
name=target.target,
|
|
project=bitbake.req.project)
|
|
|
|
custom_layer_path = self.setup_custom_image_recipe(
|
|
customrecipe, layers)
|
|
|
|
if os.path.isdir(custom_layer_path):
|
|
layerlist.append(custom_layer_path)
|
|
|
|
except CustomImageRecipe.DoesNotExist:
|
|
continue # not a custom recipe, skip
|
|
|
|
layerlist.extend(nongitlayerlist)
|
|
logger.debug("\n\nset layers gives this list %s" % pformat(layerlist))
|
|
self.islayerset = True
|
|
return layerlist
|
|
|
|
def setup_custom_image_recipe(self, customrecipe, layers):
|
|
""" Set up toaster-custom-images layer and recipe files """
|
|
layerpath = os.path.join(self.be.builddir,
|
|
CustomImageRecipe.LAYER_NAME)
|
|
|
|
# create directory structure
|
|
for name in ("conf", "recipes"):
|
|
path = os.path.join(layerpath, name)
|
|
if not os.path.isdir(path):
|
|
os.makedirs(path)
|
|
|
|
# create layer.conf
|
|
config = os.path.join(layerpath, "conf", "layer.conf")
|
|
if not os.path.isfile(config):
|
|
with open(config, "w") as conf:
|
|
conf.write('BBPATH .= ":${LAYERDIR}"\nBBFILES += "${LAYERDIR}/recipes/*.bb"\n')
|
|
|
|
# Update the Layer_Version dirpath that has our base_recipe in
|
|
# to be able to read the base recipe to then generate the
|
|
# custom recipe.
|
|
br_layer_base_recipe = layers.get(
|
|
layer_version=customrecipe.base_recipe.layer_version)
|
|
|
|
# If the layer is one that we've cloned we know where it lives
|
|
if br_layer_base_recipe.giturl and br_layer_base_recipe.commit:
|
|
layer_path = self.getGitCloneDirectory(
|
|
br_layer_base_recipe.giturl,
|
|
br_layer_base_recipe.commit)
|
|
# Otherwise it's a local layer
|
|
elif br_layer_base_recipe.local_source_dir:
|
|
layer_path = br_layer_base_recipe.local_source_dir
|
|
else:
|
|
logger.error("Unable to workout the dir path for the custom"
|
|
" image recipe")
|
|
|
|
br_layer_base_dirpath = os.path.join(
|
|
self.be.sourcedir,
|
|
layer_path,
|
|
customrecipe.base_recipe.layer_version.dirpath)
|
|
|
|
customrecipe.base_recipe.layer_version.dirpath = br_layer_base_dirpath
|
|
|
|
customrecipe.base_recipe.layer_version.save()
|
|
|
|
# create recipe
|
|
recipe_path = os.path.join(layerpath, "recipes", "%s.bb" %
|
|
customrecipe.name)
|
|
with open(recipe_path, "w") as recipef:
|
|
recipef.write(customrecipe.generate_recipe_file_contents())
|
|
|
|
# Update the layer and recipe objects
|
|
customrecipe.layer_version.dirpath = layerpath
|
|
customrecipe.layer_version.layer.local_source_dir = layerpath
|
|
customrecipe.layer_version.layer.save()
|
|
customrecipe.layer_version.save()
|
|
|
|
customrecipe.file_path = recipe_path
|
|
customrecipe.save()
|
|
|
|
return layerpath
|
|
|
|
|
|
def readServerLogFile(self):
|
|
return open(os.path.join(self.be.builddir, "toaster_server.log"), "r").read()
|
|
|
|
|
|
def triggerBuild(self, bitbake, layers, variables, targets, brbe):
|
|
layers = self.setLayers(bitbake, layers, targets)
|
|
|
|
# init build environment from the clone
|
|
builddir = '%s-toaster-%d' % (self.be.builddir, bitbake.req.project.id)
|
|
oe_init = os.path.join(self.pokydirname, 'oe-init-build-env')
|
|
# init build environment
|
|
try:
|
|
custom_script = ToasterSetting.objects.get(name="CUSTOM_BUILD_INIT_SCRIPT").value
|
|
custom_script = custom_script.replace("%BUILDDIR%" ,builddir)
|
|
self._shellcmd("bash -c 'source %s'" % (custom_script))
|
|
except ToasterSetting.DoesNotExist:
|
|
self._shellcmd("bash -c 'source %s %s'" % (oe_init, builddir),
|
|
self.be.sourcedir)
|
|
|
|
# update bblayers.conf
|
|
bblconfpath = os.path.join(builddir, "conf/toaster-bblayers.conf")
|
|
with open(bblconfpath, 'w') as bblayers:
|
|
bblayers.write('# line added by toaster build control\n'
|
|
'BBLAYERS = "%s"' % ' '.join(layers))
|
|
|
|
# write configuration file
|
|
confpath = os.path.join(builddir, 'conf/toaster.conf')
|
|
with open(confpath, 'w') as conf:
|
|
for var in variables:
|
|
conf.write('%s="%s"\n' % (var.name, var.value))
|
|
conf.write('INHERIT+="toaster buildhistory"')
|
|
|
|
# clean the Toaster to build environment
|
|
env_clean = 'unset BBPATH;' # clean BBPATH for <= YP-2.4.0
|
|
|
|
# run bitbake server from the clone
|
|
bitbake = os.path.join(self.pokydirname, 'bitbake', 'bin', 'bitbake')
|
|
toasterlayers = os.path.join(builddir,"conf/toaster-bblayers.conf")
|
|
self._shellcmd('%s bash -c \"source %s %s; BITBAKE_UI="knotty" %s --read %s --read %s '
|
|
'--server-only -B 0.0.0.0:0\"' % (env_clean, oe_init,
|
|
builddir, bitbake, confpath, toasterlayers), self.be.sourcedir)
|
|
|
|
# read port number from bitbake.lock
|
|
self.be.bbport = -1
|
|
bblock = os.path.join(builddir, 'bitbake.lock')
|
|
# allow 10 seconds for bb lock file to appear but also be populated
|
|
for lock_check in range(10):
|
|
if not os.path.exists(bblock):
|
|
logger.debug("localhostbecontroller: waiting for bblock file to appear")
|
|
time.sleep(1)
|
|
continue
|
|
if 10 < os.stat(bblock).st_size:
|
|
break
|
|
logger.debug("localhostbecontroller: waiting for bblock content to appear")
|
|
time.sleep(1)
|
|
else:
|
|
raise BuildSetupException("Cannot find bitbake server lock file '%s'. Aborting." % bblock)
|
|
|
|
with open(bblock) as fplock:
|
|
for line in fplock:
|
|
if ":" in line:
|
|
self.be.bbport = line.split(":")[-1].strip()
|
|
logger.debug("localhostbecontroller: bitbake port %s", self.be.bbport)
|
|
break
|
|
|
|
if -1 == self.be.bbport:
|
|
raise BuildSetupException("localhostbecontroller: can't read bitbake port from %s" % bblock)
|
|
|
|
self.be.bbaddress = "localhost"
|
|
self.be.bbstate = BuildEnvironment.SERVER_STARTED
|
|
self.be.lock = BuildEnvironment.LOCK_RUNNING
|
|
self.be.save()
|
|
|
|
bbtargets = ''
|
|
for target in targets:
|
|
task = target.task
|
|
if task:
|
|
if not task.startswith('do_'):
|
|
task = 'do_' + task
|
|
task = ':%s' % task
|
|
bbtargets += '%s%s ' % (target.target, task)
|
|
|
|
# run build with local bitbake. stop the server after the build.
|
|
log = os.path.join(builddir, 'toaster_ui.log')
|
|
local_bitbake = os.path.join(os.path.dirname(os.getenv('BBBASEDIR')),
|
|
'bitbake')
|
|
self._shellcmd(['%s bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:%s" '
|
|
'%s %s -u toasterui --read %s --read %s --token="" >>%s 2>&1;'
|
|
'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:%s %s -m)&\"' \
|
|
% (env_clean, brbe, self.be.bbport, local_bitbake, bbtargets, confpath, toasterlayers, log,
|
|
self.be.bbport, bitbake,)],
|
|
builddir, nowait=True)
|
|
|
|
logger.debug('localhostbecontroller: Build launched, exiting. '
|
|
'Follow build logs at %s' % log)
|