diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py index 8bdc9cc0a7..a5b22379aa 100644 --- a/bitbake/lib/bb/ui/buildinfohelper.py +++ b/bitbake/lib/bb/ui/buildinfohelper.py @@ -37,7 +37,7 @@ os.environ["DJANGO_SETTINGS_MODULE"] =\ django.setup() from orm.models import Build, Task, Recipe, Layer_Version, Layer, Target, LogMessage, HelpText -from orm.models import Target_Image_File, BuildArtifact +from orm.models import Target_Image_File, BuildArtifact, TargetArtifactFile from orm.models import Variable, VariableHistory from orm.models import Package, Package_File, Target_Installed_Package, Target_File from orm.models import Task_Dependency, Package_Dependency @@ -121,6 +121,13 @@ class ORMWrapper(object): return vars(self)[dictname][key] + def get_similar_target_with_image_files(self, target): + """ + Get a Target object "similar" to target; i.e. with the same target + name ('core-image-minimal' etc.) and machine. + """ + return target.get_similar_target_with_image_files() + def _timestamp_to_datetime(self, secs): """ Convert timestamp in seconds to Python datetime @@ -678,27 +685,32 @@ class ORMWrapper(object): file_name = file_name, file_size = file_size) - def save_artifact_information_no_dedupe(self, build_obj, file_name, file_size): + def save_target_artifact_file(self, target_obj, file_name, file_size): """ - Save artifact information without checking for duplicate paths; - this is used when we are saving data about an artifact which was - generated by a previous build but which is also relevant to this build, - e.g. a bzImage file. + Save artifact file information for a Target target_obj. + + Note that this doesn't include SDK artifacts, only images and + related files (e.g. bzImage). """ - BuildArtifact.objects.create(build=build_obj, file_name=file_name, - file_size=file_size) + TargetArtifactFile.objects.create(target=target_obj, + file_name=file_name, file_size=file_size) def save_artifact_information(self, build_obj, file_name, file_size): - # we skip the image files from other builds - if Target_Image_File.objects.filter(file_name = file_name).count() > 0: - return - + """ + TODO this is currently used to save SDK artifacts to the database, + but will be replaced in future once SDK artifacts are associated with + Target objects (as they eventually should be) + """ # do not update artifacts found in other builds if BuildArtifact.objects.filter(file_name = file_name).count() > 0: return - self.save_artifact_information_no_dedupe(self, build_obj, file_name, - file_size) + # do not update artifact if already a target artifact file for that path + if TargetArtifactFile.objects.filter(file_name = file_name).count() > 0: + return + + BuildArtifact.objects.create(build=build_obj, file_name=file_name, + file_size=file_size) def create_logmessage(self, log_information): assert 'build' in log_information @@ -1496,7 +1508,7 @@ class BuildInfoHelper(object): self.orm_wrapper.create_logmessage(log_information) - def _get_files_from_image_license(self, image_license_manifest_path): + def _get_filenames_from_image_license(self, image_license_manifest_path): """ Find the FILES line in the image_license.manifest file, which has the basenames of the bzImage and modules files @@ -1567,19 +1579,20 @@ class BuildInfoHelper(object): OR - 2. There are no files for the target, so copy them from a - previous build with the same target + machine. + 2. There are no new files for the target (they were already produced by + a previous build), so copy them from the most recent previous build with + the same target, task and machine. """ deploy_dir_image = \ self.server.runCommand(['getVariable', 'DEPLOY_DIR_IMAGE'])[0] # if there's no DEPLOY_DIR_IMAGE, there aren't going to be - # any build artifacts, so we can return immediately + # any image artifacts, so we can return immediately if not deploy_dir_image: return buildname = self.server.runCommand(['getVariable', 'BUILDNAME'])[0] - machine = self.server.runCommand(['getVariable', 'MACHINE'])[0] + machine = self.server.runCommand(['getVariable', 'MACHINE'])[0] image_name = self.server.runCommand(['getVariable', 'IMAGE_NAME'])[0] # location of the image_license.manifest files for this build; @@ -1597,7 +1610,10 @@ class BuildInfoHelper(object): image_file_extensions_unique = set(image_file_extensions.split(' ')) targets = self.internal_state['targets'] + + # filter out anything which isn't an image target image_targets = [target for target in targets if target.is_image] + for target in image_targets: # this is set to True if we find at least one file relating to # this target; if this remains False after the scan, we copy the @@ -1625,16 +1641,17 @@ class BuildInfoHelper(object): if os.path.isfile(image_license_manifest_path): has_files = True - basenames = self._get_files_from_image_license( + basenames = self._get_filenames_from_image_license( image_license_manifest_path) for basename in basenames: artifact_path = os.path.join(deploy_dir_image, basename) artifact_size = os.stat(artifact_path).st_size - self.orm_wrapper.save_artifact_information_no_dedupe( - self.internal_state['build'], artifact_path, - artifact_size) + # note that the artifact will only be saved against this + # build if it hasn't been already + self.orm_wrapper.save_target_artifact_file(target, + artifact_path, artifact_size) # store the license manifest path on the target # (this file is also created any time an image file is created) @@ -1648,7 +1665,10 @@ class BuildInfoHelper(object): # (via real_image_name); note that we don't have to set # has_files = True, as searching for the license manifest file # will already have set it to true if at least one image file was - # produced + # produced; note that the real_image_name includes BUILDNAME, which + # in turn includes a timestamp; so if no files were produced for + # this timestamp (i.e. the build reused existing image files already + # in the directory), no files will be recorded against this target image_files = self._get_image_files(deploy_dir_image, real_image_name, image_file_extensions_unique) @@ -1657,11 +1677,18 @@ class BuildInfoHelper(object): target, image_file['path'], image_file['size']) if not has_files: - # TODO copy artifact and image files from the - # most-recently-built Target with the same target + machine - # as this Target; also copy the license manifest path, - # as that is treated differently - pass + # copy image files and build artifacts from the + # most-recently-built Target with the + # same target + machine as this Target; also copy the license + # manifest path, as that is not treated as an artifact and needs + # to be set separately + most_recent = \ + self.orm_wrapper.get_similar_target_with_image_files(target) + + if most_recent: + logger.info('image artifacts for target %s cloned from ' \ + 'target %s' % (target.pk, most_recent.pk)) + target.clone_artifacts_from(most_recent) def close(self, errorcode): if self.brbe is not None: diff --git a/bitbake/lib/toaster/orm/migrations/0008_targetartifactfile.py b/bitbake/lib/toaster/orm/migrations/0008_targetartifactfile.py new file mode 100644 index 0000000000..9f1d9bf4ec --- /dev/null +++ b/bitbake/lib/toaster/orm/migrations/0008_targetartifactfile.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('orm', '0007_auto_20160523_1446'), + ] + + operations = [ + migrations.CreateModel( + name='TargetArtifactFile', + fields=[ + ('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')), + ('file_name', models.FilePathField()), + ('file_size', models.IntegerField()), + ('target', models.ForeignKey(to='orm.Target')), + ], + ), + ] diff --git a/bitbake/lib/toaster/orm/models.py b/bitbake/lib/toaster/orm/models.py index 40cdb9e5c5..9edbef3396 100644 --- a/bitbake/lib/toaster/orm/models.py +++ b/bitbake/lib/toaster/orm/models.py @@ -22,7 +22,7 @@ from __future__ import unicode_literals from django.db import models, IntegrityError -from django.db.models import F, Q, Avg, Max, Sum +from django.db.models import F, Q, Avg, Max, Sum, Count from django.utils import timezone from django.utils.encoding import force_bytes @@ -438,7 +438,9 @@ class Build(models.Model): def get_image_file_extensions(self): """ - Get list of file name extensions for images produced by this build + Get list of file name extensions for images produced by this build; + note that this is the actual list of extensions stored on Target objects + for this build, and not the value of IMAGE_FSTYPES. """ extensions = [] @@ -458,6 +460,15 @@ class Build(models.Model): return ', '.join(extensions) + def get_image_fstypes(self): + """ + Get the IMAGE_FSTYPES variable value for this build as a de-duplicated + list of image file suffixes. + """ + image_fstypes = Variable.objects.get( + build=self, variable_name='IMAGE_FSTYPES').variable_value + return list(set(re.split(r' {1,}', image_fstypes))) + def get_sorted_target_list(self): tgts = Target.objects.filter(build_id = self.id).order_by( 'target' ); return( tgts ); @@ -612,6 +623,114 @@ class Target(models.Model): def __unicode__(self): return self.target + def get_similar_targets(self): + """ + Get targets for the same machine, task and target name + (e.g. 'core-image-minimal') from a successful build for this project + (but excluding this target). + + Note that we look for targets built by this project because projects + can have different configurations from each other, and put their + artifacts in different directories. + """ + query = ~Q(pk=self.pk) & \ + Q(target=self.target) & \ + Q(build__machine=self.build.machine) & \ + Q(build__outcome=Build.SUCCEEDED) & \ + Q(build__project=self.build.project) + + return Target.objects.filter(query) + + def get_similar_target_with_image_files(self): + """ + Get the most recent similar target with Target_Image_Files associated + with it, for the purpose of cloning those files onto this target. + """ + similar_target = None + + candidates = self.get_similar_targets() + if candidates.count() < 1: + return similar_target + + task_subquery = Q(task=self.task) + + # we can look for a 'build' task if this task is a 'populate_sdk_ext' + # task, as it will have created images; and vice versa; note that + # 'build' targets can have their task set to ''; + # also note that 'populate_sdk' does not produce image files + image_tasks = [ + '', # aka 'build' + 'build', + 'populate_sdk_ext' + ] + if self.task in image_tasks: + task_subquery = Q(task__in=image_tasks) + + query = task_subquery & Q(num_files__gt=0) + + # annotate with the count of files, to exclude any targets which + # don't have associated files + candidates = candidates.annotate( + num_files=Count('target_image_file')) + + candidates = candidates.filter(query) + + if candidates.count() > 0: + candidates.order_by('build__completed_on') + similar_target = candidates.last() + + return similar_target + + def clone_artifacts_from(self, target): + """ + Make clones of the BuildArtifacts, Target_Image_Files and + TargetArtifactFile objects associated with Target target, then + associate them with this target. + + Note that for Target_Image_Files, we only want files from the previous + build whose suffix matches one of the suffixes defined in this + target's build's IMAGE_FSTYPES configuration variable. This prevents the + Target_Image_File object for an ext4 image being associated with a + target for a project which didn't produce an ext4 image (for example). + + Also sets the license_manifest_path of this target to the same path + as that of target being cloned from, as the license manifest path is + also a build artifact but is treated differently. + """ + + image_fstypes = self.build.get_image_fstypes() + + # filter out any image files whose suffixes aren't in the + # IMAGE_FSTYPES suffixes variable for this target's build + image_files = [target_image_file \ + for target_image_file in target.target_image_file_set.all() \ + if target_image_file.suffix in image_fstypes] + + for image_file in image_files: + image_file.pk = None + image_file.target = self + image_file.save() + + artifact_files = target.targetartifactfile_set.all() + for artifact_file in artifact_files: + artifact_file.pk = None + artifact_file.target = self + artifact_file.save() + + self.license_manifest_path = target.license_manifest_path + self.save() + +# an Artifact is anything that results from a target being built, and may +# be of interest to the user, and is not an image file +class TargetArtifactFile(models.Model): + target = models.ForeignKey(Target) + file_name = models.FilePathField() + file_size = models.IntegerField() + + @property + def basename(self): + return os.path.basename(self.file_name) + class Target_Image_File(models.Model): # valid suffixes for image files produced by a build SUFFIXES = { diff --git a/bitbake/lib/toaster/toastergui/templates/builddashboard.html b/bitbake/lib/toaster/toastergui/templates/builddashboard.html index dcda2a891f..f6d62b9951 100644 --- a/bitbake/lib/toaster/toastergui/templates/builddashboard.html +++ b/bitbake/lib/toaster/toastergui/templates/builddashboard.html @@ -74,7 +74,7 @@ {% for target in targets %} {% if target.target.is_image %}
-

{{target.target}}

+

{{target.target.target}}

Packages included
{{target.npkg}}
@@ -124,6 +124,19 @@ {% endfor %} +
+ Kernel artifacts +
+
+
    + {% for artifact in target.target_artifacts|dictsort:"basename" %} +
  • + {{artifact.basename}} + ({{artifact.file_size|filtered_filesizeformat}}) +
    • + {% endfor %} +
+
{% endif %}
diff --git a/bitbake/lib/toaster/toastergui/views.py b/bitbake/lib/toaster/toastergui/views.py index ad85fafb4d..0ec88d9a77 100755 --- a/bitbake/lib/toaster/toastergui/views.py +++ b/bitbake/lib/toaster/toastergui/views.py @@ -31,6 +31,7 @@ from django.shortcuts import render, redirect, get_object_or_404 from orm.models import Build, Target, Task, Layer, Layer_Version, Recipe, LogMessage, Variable from orm.models import Task_Dependency, Recipe_Dependency, Package, Package_File, Package_Dependency from orm.models import Target_Installed_Package, Target_File, Target_Image_File, BuildArtifact, CustomImagePackage +from orm.models import TargetArtifactFile from orm.models import BitbakeVersion, CustomImageRecipe from bldcontrol import bbcontroller from django.views.decorators.cache import cache_control @@ -509,6 +510,8 @@ def builddashboard( request, build_id ): targetHasNoImages = True elem[ 'imageFiles' ] = imageFiles elem[ 'targetHasNoImages' ] = targetHasNoImages + elem['target_artifacts'] = t.targetartifactfile_set.all() + targets.append( elem ) ## @@ -2294,6 +2297,10 @@ if True: elif artifact_type == "buildartifact": file_name = BuildArtifact.objects.get(build = build, pk = artifact_id).file_name + elif artifact_type == "targetartifactfile": + target = TargetArtifactFile.objects.get(pk=artifact_id) + file_name = target.file_name + elif artifact_type == "licensemanifest": file_name = Target.objects.get(build = build, pk = artifact_id).license_manifest_path