mirror of
https://git.yoctoproject.org/poky
synced 2026-04-02 17:02:21 +02:00
bitbake: toaster: properly set layers when running a build
This patch enables the localhost build controller to properly set the layers before the build runs. It creates the checkout directories under BuildEnvironment sourcedir directory, and runs the build in the buildir directory. Build launch errors are tracked in the newly added BRError table. These are different from build errors, in the sense that the build can't start due to these errors. (Bitbake rev: 1868d5635b517e0fe1b874674ea7a78910b26e2e) Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Alexandru DAMIAN
committed by
Richard Purdie
Richard Purdie
parent
6e71c276b5
commit
1b9175af3f
@@ -126,6 +126,8 @@ class BuildEnvironmentController(object):
|
||||
def setLayers(self,ls):
|
||||
""" Sets the layer variables in the config file, after validating local layer paths.
|
||||
The layer paths must be in a list of BRLayer object
|
||||
|
||||
a word of attention: by convention, the first layer for any build will be poky!
|
||||
"""
|
||||
raise Exception("Must override setLayers")
|
||||
|
||||
@@ -165,25 +167,31 @@ class BuildEnvironmentController(object):
|
||||
class ShellCmdException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class BuildSetupException(Exception):
|
||||
pass
|
||||
|
||||
class LocalhostBEController(BuildEnvironmentController):
|
||||
""" Implementation of the BuildEnvironmentController for the localhost;
|
||||
this controller manages the default build directory,
|
||||
the server setup and system start and stop for the localhost-type build environment
|
||||
|
||||
"""
|
||||
from os.path import dirname as DN
|
||||
|
||||
def __init__(self, be):
|
||||
super(LocalhostBEController, self).__init__(be)
|
||||
from os.path import dirname as DN
|
||||
self.dburl = settings.getDATABASE_URL()
|
||||
self.pokydirname = None
|
||||
|
||||
def _shellcmd(self, command):
|
||||
p = subprocess.Popen(command, cwd=self.be.sourcedir, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
def _shellcmd(self, command, cwd = None):
|
||||
if cwd is None:
|
||||
cwd = self.be.sourcedir
|
||||
|
||||
p = subprocess.Popen(command, cwd = cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
(out,err) = p.communicate()
|
||||
if p.returncode:
|
||||
if len(err) == 0:
|
||||
err = "command: %s" % command
|
||||
err = "command: %s \n%s" % (command, out)
|
||||
else:
|
||||
err = "command: %s \n%s" % (command, err)
|
||||
raise ShellCmdException(err)
|
||||
@@ -191,22 +199,20 @@ class LocalhostBEController(BuildEnvironmentController):
|
||||
return out
|
||||
|
||||
def _createdirpath(self, path):
|
||||
from os.path import dirname as DN
|
||||
if not os.path.exists(DN(path)):
|
||||
self._createdirpath(DN(path))
|
||||
if not os.path.exists(path):
|
||||
os.mkdir(path, 0755)
|
||||
|
||||
def _startBE(self):
|
||||
assert self.be.sourcedir and os.path.exists(self.be.sourcedir)
|
||||
assert self.pokydirname and os.path.exists(self.pokydirname)
|
||||
self._createdirpath(self.be.builddir)
|
||||
self._shellcmd("bash -c \"source %s/oe-init-build-env %s\"" % (self.be.sourcedir, self.be.builddir))
|
||||
self._shellcmd("bash -c \"source %s/oe-init-build-env %s\"" % (self.pokydirname, self.be.builddir))
|
||||
|
||||
def startBBServer(self):
|
||||
assert self.be.sourcedir and os.path.exists(self.be.sourcedir)
|
||||
|
||||
self._startBE()
|
||||
|
||||
print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb && sleep 1\"" % (self.be.sourcedir, self.be.builddir, self.dburl))
|
||||
assert self.pokydirname and os.path.exists(self.pokydirname)
|
||||
print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb && sleep 1\"" % (self.pokydirname, self.be.builddir, self.dburl))
|
||||
# FIXME unfortunate sleep 1 - we need to make sure that bbserver is started and the toaster ui is connected
|
||||
# but since they start async without any return, we just wait a bit
|
||||
print "Started server"
|
||||
@@ -225,10 +231,82 @@ class LocalhostBEController(BuildEnvironmentController):
|
||||
print "Stopped server"
|
||||
|
||||
def setLayers(self, layers):
|
||||
""" a word of attention: by convention, the first layer for any build will be poky! """
|
||||
|
||||
assert self.be.sourcedir is not None
|
||||
layerconf = os.path.join(self.be.builddir, "conf/bblayers.conf")
|
||||
if not os.path.exists(layerconf):
|
||||
raise Exception("BE is not consistent: bblayers.conf file missing at ", layerconf)
|
||||
# set layers in the layersource
|
||||
|
||||
# 1. get a list of repos, and map dirpaths for each layer
|
||||
gitrepos = {}
|
||||
for layer in layers:
|
||||
if not layer.giturl in gitrepos:
|
||||
gitrepos[layer.giturl] = []
|
||||
gitrepos[layer.giturl].append( (layer.name, layer.dirpath, layer.commit))
|
||||
for giturl in gitrepos.keys():
|
||||
commitid = gitrepos[giturl][0][2]
|
||||
for e in gitrepos[giturl]:
|
||||
if commitid != e[2]:
|
||||
raise BuildSetupException("More than one commit per git url, unsupported configuration")
|
||||
|
||||
def _getgitdirectoryname(url):
|
||||
import re
|
||||
components = re.split(r'[\.\/]', url)
|
||||
return components[-2] if components[-1] == "git" else components[-1]
|
||||
|
||||
layerlist = []
|
||||
|
||||
# 2. checkout the repositories
|
||||
for giturl in gitrepos.keys():
|
||||
localdirname = os.path.join(self.be.sourcedir, _getgitdirectoryname(giturl))
|
||||
print "DEBUG: giturl checking out in current directory", localdirname
|
||||
|
||||
# make sure our directory is a git repository
|
||||
if os.path.exists(localdirname):
|
||||
if not giturl in self._shellcmd("git remote -v", localdirname):
|
||||
raise BuildSetupException("Existing git repository at %s, but with different remotes (not '%s'). Aborting." % (localdirname, giturl))
|
||||
else:
|
||||
self._shellcmd("git clone \"%s\" \"%s\"" % (giturl, localdirname))
|
||||
# checkout the needed commit
|
||||
commit = gitrepos[giturl][0][2]
|
||||
self._shellcmd("git fetch --all && git checkout \"%s\"" % commit , localdirname)
|
||||
print "DEBUG: checked out commit ", commit, "to", localdirname
|
||||
|
||||
# if this is the first checkout, take the localdirname as poky dir
|
||||
if self.pokydirname is None:
|
||||
print "DEBUG: selected poky dir name", localdirname
|
||||
self.pokydirname = localdirname
|
||||
|
||||
# verify our repositories
|
||||
for name, dirpath, commit in gitrepos[giturl]:
|
||||
localdirpath = os.path.join(localdirname, dirpath)
|
||||
if not os.path.exists(localdirpath):
|
||||
raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit))
|
||||
|
||||
layerlist.append(localdirpath)
|
||||
|
||||
print "DEBUG: current layer list ", layerlist
|
||||
|
||||
# 3. configure the build environment, so we have a conf/bblayers.conf
|
||||
assert self.pokydirname is not None
|
||||
self._startBE()
|
||||
|
||||
# 4. update the bblayers.conf
|
||||
bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf")
|
||||
if not os.path.exists(bblayerconf):
|
||||
raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf)
|
||||
|
||||
conflines = open(bblayerconf, "r").readlines()
|
||||
|
||||
bblayerconffile = open(bblayerconf, "w")
|
||||
for i in xrange(len(conflines)):
|
||||
if conflines[i].startswith("# line added by toaster"):
|
||||
i += 2
|
||||
else:
|
||||
bblayerconffile.write(conflines[i])
|
||||
|
||||
bblayerconffile.write("\n# line added by toaster build control\nBBLAYERS = \"" + " ".join(layerlist) + "\"")
|
||||
bblayerconffile.close()
|
||||
|
||||
return True
|
||||
|
||||
def release(self):
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
from django.core.management.base import NoArgsCommand, CommandError
|
||||
from django.db import transaction
|
||||
from orm.models import Build
|
||||
from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException
|
||||
from bldcontrol.models import BuildRequest, BuildEnvironment
|
||||
from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException, BuildSetupException
|
||||
from bldcontrol.models import BuildRequest, BuildEnvironment, BRError
|
||||
import os
|
||||
|
||||
class Command(NoArgsCommand):
|
||||
@@ -25,6 +25,7 @@ class Command(NoArgsCommand):
|
||||
return br
|
||||
|
||||
def schedule(self):
|
||||
import traceback
|
||||
try:
|
||||
br = None
|
||||
try:
|
||||
@@ -63,15 +64,19 @@ class Command(NoArgsCommand):
|
||||
|
||||
# cleanup to be performed by toaster when the deed is done
|
||||
|
||||
except ShellCmdException as e:
|
||||
import traceback
|
||||
print " EE Error executing shell command\n", e
|
||||
traceback.format_exc(e)
|
||||
|
||||
except Exception as e:
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
raise e
|
||||
print " EE Error executing shell command\n", e
|
||||
traceback.print_exc(e)
|
||||
BRError.objects.create(req = br,
|
||||
errtype = str(type(e)),
|
||||
errmsg = str(e),
|
||||
traceback = traceback.format_exc(e))
|
||||
br.state = BuildRequest.REQ_FAILED
|
||||
br.save()
|
||||
bec.be.lock = BuildEnvironment.LOCK_FREE
|
||||
bec.be.save()
|
||||
|
||||
|
||||
def cleanup(self):
|
||||
from django.utils import timezone
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
from south.utils import datetime_utils as datetime
|
||||
from south.db import db
|
||||
from south.v2 import SchemaMigration
|
||||
from django.db import models
|
||||
|
||||
|
||||
class Migration(SchemaMigration):
|
||||
|
||||
def forwards(self, orm):
|
||||
# Adding model 'BRError'
|
||||
db.create_table(u'bldcontrol_brerror', (
|
||||
(u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
|
||||
('req', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['bldcontrol.BuildRequest'])),
|
||||
('errtype', self.gf('django.db.models.fields.CharField')(max_length=100)),
|
||||
('errmsg', self.gf('django.db.models.fields.TextField')()),
|
||||
('traceback', self.gf('django.db.models.fields.TextField')()),
|
||||
))
|
||||
db.send_create_signal(u'bldcontrol', ['BRError'])
|
||||
|
||||
|
||||
def backwards(self, orm):
|
||||
# Deleting model 'BRError'
|
||||
db.delete_table(u'bldcontrol_brerror')
|
||||
|
||||
|
||||
models = {
|
||||
u'bldcontrol.brerror': {
|
||||
'Meta': {'object_name': 'BRError'},
|
||||
'errmsg': ('django.db.models.fields.TextField', [], {}),
|
||||
'errtype': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
|
||||
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}),
|
||||
'traceback': ('django.db.models.fields.TextField', [], {})
|
||||
},
|
||||
u'bldcontrol.brlayer': {
|
||||
'Meta': {'object_name': 'BRLayer'},
|
||||
'commit': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
|
||||
'dirpath': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
|
||||
'giturl': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
|
||||
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
|
||||
'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"})
|
||||
},
|
||||
u'bldcontrol.brtarget': {
|
||||
'Meta': {'object_name': 'BRTarget'},
|
||||
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}),
|
||||
'target': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
|
||||
'task': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True'})
|
||||
},
|
||||
u'bldcontrol.brvariable': {
|
||||
'Meta': {'object_name': 'BRVariable'},
|
||||
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
|
||||
'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}),
|
||||
'value': ('django.db.models.fields.TextField', [], {'blank': 'True'})
|
||||
},
|
||||
u'bldcontrol.buildenvironment': {
|
||||
'Meta': {'object_name': 'BuildEnvironment'},
|
||||
'address': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
|
||||
'bbaddress': ('django.db.models.fields.CharField', [], {'max_length': '254', 'blank': 'True'}),
|
||||
'bbport': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
|
||||
'bbstate': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'bbtoken': ('django.db.models.fields.CharField', [], {'max_length': '126', 'blank': 'True'}),
|
||||
'betype': ('django.db.models.fields.IntegerField', [], {}),
|
||||
'builddir': ('django.db.models.fields.CharField', [], {'max_length': '512', 'blank': 'True'}),
|
||||
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
|
||||
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'lock': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'sourcedir': ('django.db.models.fields.CharField', [], {'max_length': '512', 'blank': 'True'}),
|
||||
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
|
||||
},
|
||||
u'bldcontrol.buildrequest': {
|
||||
'Meta': {'object_name': 'BuildRequest'},
|
||||
'build': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Build']", 'null': 'True'}),
|
||||
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
|
||||
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'project': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Project']"}),
|
||||
'state': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
|
||||
},
|
||||
u'orm.build': {
|
||||
'Meta': {'object_name': 'Build'},
|
||||
'bitbake_version': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
|
||||
'build_name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
|
||||
'completed_on': ('django.db.models.fields.DateTimeField', [], {}),
|
||||
'cooker_log_path': ('django.db.models.fields.CharField', [], {'max_length': '500'}),
|
||||
'distro': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
|
||||
'distro_version': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
|
||||
'errors_no': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'machine': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
|
||||
'outcome': ('django.db.models.fields.IntegerField', [], {'default': '2'}),
|
||||
'project': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Project']", 'null': 'True'}),
|
||||
'started_on': ('django.db.models.fields.DateTimeField', [], {}),
|
||||
'timespent': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
|
||||
'warnings_no': ('django.db.models.fields.IntegerField', [], {'default': '0'})
|
||||
},
|
||||
u'orm.project': {
|
||||
'Meta': {'object_name': 'Project'},
|
||||
'branch': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
|
||||
'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
|
||||
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
|
||||
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
|
||||
'short_description': ('django.db.models.fields.CharField', [], {'max_length': '50', 'blank': 'True'}),
|
||||
'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
|
||||
'user_id': ('django.db.models.fields.IntegerField', [], {'null': 'True'})
|
||||
}
|
||||
}
|
||||
|
||||
complete_apps = ['bldcontrol']
|
||||
@@ -48,12 +48,14 @@ class BuildRequest(models.Model):
|
||||
REQ_QUEUED = 1
|
||||
REQ_INPROGRESS = 2
|
||||
REQ_COMPLETED = 3
|
||||
REQ_FAILED = 4
|
||||
|
||||
REQUEST_STATE = (
|
||||
(REQ_CREATED, "created"),
|
||||
(REQ_QUEUED, "queued"),
|
||||
(REQ_INPROGRESS, "in progress"),
|
||||
(REQ_COMPLETED, "completed"),
|
||||
(REQ_FAILED, "failed"),
|
||||
)
|
||||
|
||||
project = models.ForeignKey(Project)
|
||||
@@ -84,4 +86,8 @@ class BRTarget(models.Model):
|
||||
target = models.CharField(max_length=100)
|
||||
task = models.CharField(max_length=100, null=True)
|
||||
|
||||
|
||||
class BRError(models.Model):
|
||||
req = models.ForeignKey(BuildRequest)
|
||||
errtype = models.CharField(max_length=100)
|
||||
errmsg = models.TextField()
|
||||
traceback = models.TextField()
|
||||
|
||||
Reference in New Issue
Block a user