mirrors/poky
1
0
Fork 0
mirror of https://git.yoctoproject.org/poky synced 2026-02-09 02:03:04 +01:00
Code Issues Packages Projects Releases Wiki Activity
Files
09febe09dc7481dcd08dc3cb033ea85e0e75abff
poky/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
Michael Wood c86a0d290f bitbake: toaster: Fix build execution regression 
Make sure the error value is initialised and update the string match
that we have to identify bitbake's current startup status.

Patch contribution from Eduard Bartosh

(Bitbake rev: 915ba08a8a3013e9787e564f2ffd8698c948f433)

Signed-off-by: Michael Wood <michael.g.wood@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2015-07-02 22:56:39 +01:00

342 lines
15 KiB
Python
Raw Blame History

#
# ex:ts=4:sw=4:sts=4:et
# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
#
# BitBake Toaster Implementation
#
# Copyright (C) 2014 Intel Corporation
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import os
import sys
import re
from django.db import transaction
from django.db.models import Q
from bldcontrol.models import BuildEnvironment, BRLayer, BRVariable, BRTarget, BRBitbake
import subprocess
from toastermain import settings
from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException
import logging
logger = logging.getLogger("toaster")
from pprint import pprint, pformat
class LocalhostBEController(BuildEnvironmentController):
""" Implementation of the BuildEnvironmentController for the localhost;
this controller manages the default build directory,
the server setup and system start and stop for the localhost-type build environment
"""
def __init__(self, be):
super(LocalhostBEController, self).__init__(be)
self.dburl = settings.getDATABASE_URL()
self.pokydirname = None
self.islayerset = False
def _shellcmd(self, command, cwd = None):
if cwd is None:
cwd = self.be.sourcedir
#logger.debug("lbc_shellcmmd: (%s) %s" % (cwd, command))
p = subprocess.Popen(command, cwd = cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
(out,err) = p.communicate()
p.wait()
if p.returncode:
if len(err) == 0:
err = "command: %s \n%s" % (command, out)
else:
err = "command: %s \n%s" % (command, err)
#logger.warn("localhostbecontroller: shellcmd error %s" % err)
raise ShellCmdException(err)
else:
#logger.debug("localhostbecontroller: shellcmd success")
return out
def _createdirpath(self, path):
from os.path import dirname as DN
if path == "":
raise Exception("Invalid path creation specified.")
if not os.path.exists(DN(path)):
self._createdirpath(DN(path))
if not os.path.exists(path):
os.mkdir(path, 0755)
def _setupBE(self):
assert self.pokydirname and os.path.exists(self.pokydirname)
self._createdirpath(self.be.builddir)
self._shellcmd("bash -c \"source %s/oe-init-build-env %s\"" % (self.pokydirname, self.be.builddir))
# delete the templateconf.cfg; it may come from an unsupported layer configuration
os.remove(os.path.join(self.be.builddir, "conf/templateconf.cfg"))
def writeConfFile(self, file_name, variable_list = None, raw = None):
filepath = os.path.join(self.be.builddir, file_name)
with open(filepath, "w") as conffile:
if variable_list is not None:
for i in variable_list:
conffile.write("%s=\"%s\"\n" % (i.name, i.value))
if raw is not None:
conffile.write(raw)
def startBBServer(self):
assert self.pokydirname and os.path.exists(self.pokydirname)
assert self.islayerset
# find our own toasterui listener/bitbake
from toaster.bldcontrol.management.commands.loadconf import _reduce_canon_path
own_bitbake = _reduce_canon_path(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../../bin/bitbake"))
assert os.path.exists(own_bitbake) and os.path.isfile(own_bitbake)
logger.debug("localhostbecontroller: running the listener at %s" % own_bitbake)
toaster_ui_log_filepath = os.path.join(self.be.builddir, "toaster_ui.log")
# get the file length; we need to detect the _last_ start of the toaster UI, not the first
toaster_ui_log_filelength = 0
if os.path.exists(toaster_ui_log_filepath):
with open(toaster_ui_log_filepath, "w") as f:
f.seek(0, 2) # jump to the end
toaster_ui_log_filelength = f.tell()
cmd = "bash -c \"source %s/oe-init-build-env %s 2>&1 >toaster_server.log && bitbake --read %s/conf/toaster-pre.conf --postread %s/conf/toaster.conf --server-only -t xmlrpc -B 0.0.0.0:0 2>&1 >>toaster_server.log \"" % (self.pokydirname, self.be.builddir, self.be.builddir, self.be.builddir)
port = "-1"
logger.debug("localhostbecontroller: starting builder \n%s\n" % cmd)
cmdoutput = self._shellcmd(cmd)
with open(self.be.builddir + "/toaster_server.log", "r") as f:
for i in f.readlines():
if i.startswith("Bitbake server address"):
port = i.split(" ")[-1]
logger.debug("localhostbecontroller: Found bitbake server port %s" % port)
cmd = "bash -c \"source %s/oe-init-build-env-memres -1 %s && DATABASE_URL=%s %s --observe-only -u toasterui --remote-server=0.0.0.0:-1 -t xmlrpc\"" % (self.pokydirname, self.be.builddir, self.dburl, own_bitbake)
with open(toaster_ui_log_filepath, "a+") as f:
p = subprocess.Popen(cmd, cwd = self.be.builddir, shell=True, stdout=f, stderr=f)
def _toaster_ui_started(filepath, filepos = 0):
if not os.path.exists(filepath):
return False
with open(filepath, "r") as f:
f.seek(filepos)
for line in f:
if line.startswith("Bitbake server started on demand"):
return True
return False
retries = 0
started = False
while not started and retries < 10:
started = _toaster_ui_started(toaster_ui_log_filepath, toaster_ui_log_filelength)
import time
logger.debug("localhostbecontroller: Waiting bitbake server to start")
time.sleep(0.5)
retries += 1
if not started:
toaster_ui_log = open(os.path.join(self.be.builddir, "toaster_ui.log"), "r").read()
toaster_server_log = open(os.path.join(self.be.builddir, "toaster_server.log"), "r").read()
raise BuildSetupException("localhostbecontroller: Bitbake server did not start in 5 seconds, aborting (Error: '%s' '%s')" % (toaster_ui_log, toaster_server_log))
logger.debug("localhostbecontroller: Started bitbake server")
while port == "-1":
# the port specification is "autodetect"; read the bitbake.lock file
with open("%s/bitbake.lock" % self.be.builddir, "r") as f:
for line in f.readlines():
if ":" in line:
port = line.split(":")[1].strip()
logger.debug("localhostbecontroller: Autodetected bitbake port %s", port)
break
assert self.be.sourcedir and os.path.exists(self.be.builddir)
self.be.bbaddress = "localhost"
self.be.bbport = port
self.be.bbstate = BuildEnvironment.SERVER_STARTED
self.be.save()
def stopBBServer(self):
assert self.pokydirname and os.path.exists(self.pokydirname)
assert self.islayerset
self._shellcmd("bash -c \"source %s/oe-init-build-env %s && %s source toaster stop\"" %
(self.pokydirname, self.be.builddir, (lambda: "" if self.be.bbtoken is None else "BBTOKEN=%s" % self.be.bbtoken)()))
self.be.bbstate = BuildEnvironment.SERVER_STOPPED
self.be.save()
logger.debug("localhostbecontroller: Stopped bitbake server")
def getGitCloneDirectory(self, url, branch):
""" Utility that returns the last component of a git path as directory
"""
import re
components = re.split(r'[:\.\/]', url)
base = components[-2] if components[-1] == "git" else components[-1]
if branch != "HEAD":
return "_%s_%s.toaster_cloned" % (base, branch)
# word of attention; this is a localhost-specific issue; only on the localhost we expect to have "HEAD" releases
# which _ALWAYS_ means the current poky checkout
from os.path import dirname as DN
local_checkout_path = DN(DN(DN(DN(DN(os.path.abspath(__file__))))))
#logger.debug("localhostbecontroller: using HEAD checkout in %s" % local_checkout_path)
return local_checkout_path
def setLayers(self, bitbakes, layers):
""" a word of attention: by convention, the first layer for any build will be poky! """
assert self.be.sourcedir is not None
assert len(bitbakes) == 1
# set layers in the layersource
# 1. get a list of repos with branches, and map dirpaths for each layer
gitrepos = {}
gitrepos[(bitbakes[0].giturl, bitbakes[0].commit)] = []
gitrepos[(bitbakes[0].giturl, bitbakes[0].commit)].append( ("bitbake", bitbakes[0].dirpath) )
for layer in layers:
# we don't process local URLs
if layer.giturl.startswith("file://"):
continue
if not (layer.giturl, layer.commit) in gitrepos:
gitrepos[(layer.giturl, layer.commit)] = []
gitrepos[(layer.giturl, layer.commit)].append( (layer.name, layer.dirpath) )
logger.debug("localhostbecontroller, our git repos are %s" % pformat(gitrepos))
# 2. find checked-out git repos in the sourcedir directory that may help faster cloning
cached_layers = {}
for ldir in os.listdir(self.be.sourcedir):
fldir = os.path.join(self.be.sourcedir, ldir)
if os.path.isdir(fldir):
try:
for line in self._shellcmd("git remote -v", fldir).split("\n"):
try:
remote = line.split("\t")[1].split(" ")[0]
if remote not in cached_layers:
cached_layers[remote] = fldir
except IndexError:
pass
except ShellCmdException:
# ignore any errors in collecting git remotes
pass
layerlist = []
# 3. checkout the repositories
for giturl, commit in gitrepos.keys():
localdirname = os.path.join(self.be.sourcedir, self.getGitCloneDirectory(giturl, commit))
logger.debug("localhostbecontroller: giturl %s:%s checking out in current directory %s" % (giturl, commit, localdirname))
# make sure our directory is a git repository
if os.path.exists(localdirname):
localremotes = self._shellcmd("git remote -v", localdirname)
if not giturl in localremotes:
raise BuildSetupException("Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something." % (localdirname, ", ".join(localremotes.split("\n")), giturl))
else:
if giturl in cached_layers:
logger.debug("localhostbecontroller git-copying %s to %s" % (cached_layers[giturl], localdirname))
self._shellcmd("git clone \"%s\" \"%s\"" % (cached_layers[giturl], localdirname))
self._shellcmd("git remote remove origin", localdirname)
self._shellcmd("git remote add origin \"%s\"" % giturl, localdirname)
else:
logger.debug("localhostbecontroller: cloning %s:%s in %s" % (giturl, commit, localdirname))
self._shellcmd("git clone \"%s\" --single-branch --branch \"%s\" \"%s\"" % (giturl, commit, localdirname))
# branch magic name "HEAD" will inhibit checkout
if commit != "HEAD":
logger.debug("localhostbecontroller: checking out commit %s to %s " % (commit, localdirname))
self._shellcmd("git fetch --all && git checkout \"%s\" && git rebase \"origin/%s\"" % (commit, commit) , localdirname)
# take the localdirname as poky dir if we can find the oe-init-build-env
if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")):
logger.debug("localhostbecontroller: selected poky dir name %s" % localdirname)
self.pokydirname = localdirname
# make sure we have a working bitbake
if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')):
logger.debug("localhostbecontroller: checking bitbake into the poky dirname %s " % self.pokydirname)
self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, 'bitbake')))
# verify our repositories
for name, dirpath in gitrepos[(giturl, commit)]:
localdirpath = os.path.join(localdirname, dirpath)
logger.debug("localhostbecontroller: localdirpath expected '%s'" % localdirpath)
if not os.path.exists(localdirpath):
raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit))
if name != "bitbake":
layerlist.append(localdirpath.rstrip("/"))
logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist))
# 4. configure the build environment, so we have a conf/bblayers.conf
assert self.pokydirname is not None
self._setupBE()
# 5. update the bblayers.conf
bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf")
if not os.path.exists(bblayerconf):
raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf)
BuildEnvironmentController._updateBBLayers(bblayerconf, layerlist)
self.islayerset = True
return True
def readServerLogFile(self):
return open(os.path.join(self.be.builddir, "toaster_server.log"), "r").read()
def release(self):
assert self.be.sourcedir and os.path.exists(self.be.builddir)
import shutil
shutil.rmtree(os.path.join(self.be.sourcedir, "build"))
assert not os.path.exists(self.be.builddir)
def triggerBuild(self, bitbake, layers, variables, targets):
# set up the buid environment with the needed layers
self.setLayers(bitbake, layers)
self.writeConfFile("conf/toaster-pre.conf", variables)
self.writeConfFile("conf/toaster.conf", raw = "INHERIT+=\"toaster buildhistory\"")
# get the bb server running with the build req id and build env id
bbctrl = self.getBBController()
# trigger the build command
task = reduce(lambda x, y: x if len(y)== 0 else y, map(lambda y: y.task, targets))
if len(task) == 0:
task = None
bbctrl.build(list(map(lambda x:x.target, targets)), task)
logger.debug("localhostbecontroller: Build launched, exiting. Follow build logs at %s/toaster_ui.log" % self.be.builddir)
# disconnect from the server
bbctrl.disconnect()
Reference in New Issue View Git Blame Copy Permalink