mirror of
https://git.yoctoproject.org/poky
synced 2026-01-29 21:08:42 +01:00
2f4f8ef0fc
Record some additional information about images - the uncompressed size of the final image as well as the values of various variables that may have influenced its contents. This is recorded in a machine-readable "image-info.txt" file similar in structure to the package history files. Also add some code to analyse changes to these values. (Most of the variable values aren't monitored directly but will be used as contextual information when they change at the same time as the content of the image changing.) (From OE-Core rev: 459ed6759a307b389f6ec1874136ec9aa0749120) Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
248 lines
9.6 KiB
Python
248 lines
9.6 KiB
Python
# Report significant differences in the buildhistory repository since a specific revision
|
|
#
|
|
# Copyright (C) 2012 Intel Corporation
|
|
# Author: Paul Eggleton <paul.eggleton@linux.intel.com>
|
|
#
|
|
# Note: requires GitPython 0.3.1+
|
|
#
|
|
# You can use this from the command line by running scripts/buildhistory-diff
|
|
#
|
|
|
|
import sys
|
|
import os.path
|
|
import difflib
|
|
import git
|
|
|
|
|
|
# How to display fields
|
|
list_fields = ['DEPENDS', 'RDEPENDS', 'RRECOMMENDS', 'PACKAGES', 'FILES', 'FILELIST', 'USER_CLASSES', 'IMAGE_CLASSES', 'IMAGE_FEATURES', 'IMAGE_LINGUAS', 'IMAGE_INSTALL', 'BAD_RECOMMENDATIONS']
|
|
numeric_fields = ['PKGSIZE', 'IMAGESIZE']
|
|
# Fields to monitor
|
|
monitor_fields = ['RDEPENDS', 'RRECOMMENDS', 'PACKAGES', 'FILELIST', 'PKGSIZE', 'IMAGESIZE']
|
|
# Percentage change to alert for numeric fields
|
|
monitor_numeric_threshold = 20
|
|
# Image files to monitor (note that image-info.txt is handled separately)
|
|
img_monitor_files = ['installed-package-names.txt', 'files-in-image.txt']
|
|
|
|
class ChangeRecord:
|
|
def __init__(self, path, fieldname, oldvalue, newvalue):
|
|
self.path = path
|
|
self.fieldname = fieldname
|
|
self.oldvalue = oldvalue
|
|
self.newvalue = newvalue
|
|
self.filechanges = None
|
|
|
|
def __str__(self):
|
|
if self.fieldname in list_fields:
|
|
aitems = self.oldvalue.split(' ')
|
|
bitems = self.newvalue.split(' ')
|
|
removed = list(set(aitems) - set(bitems))
|
|
added = list(set(bitems) - set(aitems))
|
|
return '%s: %s:%s%s' % (self.path, self.fieldname, ' removed "%s"' % ' '.join(removed) if removed else '', ' added "%s"' % ' '.join(added) if added else '')
|
|
elif self.fieldname in numeric_fields:
|
|
aval = int(self.oldvalue)
|
|
bval = int(self.newvalue)
|
|
percentchg = ((bval - aval) / float(aval)) * 100
|
|
return '%s: %s changed from %d to %d (%s%d%%)' % (self.path, self.fieldname, aval, bval, '+' if percentchg > 0 else '', percentchg)
|
|
elif self.fieldname in img_monitor_files:
|
|
out = 'Changes to %s (%s):\n ' % (self.path, self.fieldname)
|
|
if self.filechanges:
|
|
out += '\n '.join(['%s' % i for i in self.filechanges])
|
|
else:
|
|
alines = self.oldvalue.splitlines()
|
|
blines = self.newvalue.splitlines()
|
|
diff = difflib.unified_diff(alines, blines, self.fieldname, self.fieldname, lineterm='')
|
|
out += '\n '.join(list(diff))
|
|
out += '\n --'
|
|
return out
|
|
else:
|
|
return '%s: %s changed from "%s" to "%s"' % (self.path, self.self.fieldname, self.oldvalue, self.newvalue)
|
|
|
|
|
|
class FileChange:
|
|
changetype_add = 'A'
|
|
changetype_remove = 'R'
|
|
changetype_type = 'T'
|
|
changetype_perms = 'P'
|
|
changetype_ownergroup = 'O'
|
|
changetype_link = 'L'
|
|
|
|
def __init__(self, path, changetype, oldvalue = None, newvalue = None):
|
|
self.path = path
|
|
self.changetype = changetype
|
|
self.oldvalue = oldvalue
|
|
self.newvalue = newvalue
|
|
|
|
def _ftype_str(self, ftype):
|
|
if ftype == '-':
|
|
return 'file'
|
|
elif ftype == 'd':
|
|
return 'directory'
|
|
elif ftype == 'l':
|
|
return 'symlink'
|
|
elif ftype == 'c':
|
|
return 'char device'
|
|
elif ftype == 'b':
|
|
return 'block device'
|
|
elif ftype == 'p':
|
|
return 'fifo'
|
|
elif ftype == 's':
|
|
return 'socket'
|
|
else:
|
|
return 'unknown (%s)' % ftype
|
|
|
|
def __str__(self):
|
|
if self.changetype == self.changetype_add:
|
|
return '%s was added' % self.path
|
|
elif self.changetype == self.changetype_remove:
|
|
return '%s was removed' % self.path
|
|
elif self.changetype == self.changetype_type:
|
|
return '%s changed type from %s to %s' % (self.path, self._ftype_str(self.oldvalue), self._ftype_str(self.newvalue))
|
|
elif self.changetype == self.changetype_perms:
|
|
return '%s changed permissions from %s to %s' % (self.path, self.oldvalue, self.newvalue)
|
|
elif self.changetype == self.changetype_ownergroup:
|
|
return '%s changed owner/group from %s to %s' % (self.path, self.oldvalue, self.newvalue)
|
|
elif self.changetype == self.changetype_link:
|
|
return '%s changed symlink target from %s to %s' % (self.path, self.oldvalue, self.newvalue)
|
|
else:
|
|
return '%s changed (unknown)' % self.path
|
|
|
|
|
|
def blob_to_dict(blob):
|
|
alines = blob.data_stream.read().splitlines()
|
|
adict = {}
|
|
for line in alines:
|
|
splitv = [i.strip() for i in line.split('=',1)]
|
|
if splitv.count > 1:
|
|
adict[splitv[0]] = splitv[1]
|
|
return adict
|
|
|
|
|
|
def file_list_to_dict(lines):
|
|
adict = {}
|
|
for line in lines:
|
|
# Leave the last few fields intact so we handle file names containing spaces
|
|
splitv = line.split(None,4)
|
|
# Grab the path and remove the leading .
|
|
path = splitv[4][1:].strip()
|
|
# Handle symlinks
|
|
if(' -> ' in path):
|
|
target = path.split(' -> ')[1]
|
|
path = path.split(' -> ')[0]
|
|
adict[path] = splitv[0:3] + [target]
|
|
else:
|
|
adict[path] = splitv[0:3]
|
|
return adict
|
|
|
|
|
|
def compare_file_lists(alines, blines):
|
|
adict = file_list_to_dict(alines)
|
|
bdict = file_list_to_dict(blines)
|
|
filechanges = []
|
|
for path, splitv in adict.iteritems():
|
|
newsplitv = bdict.pop(path, None)
|
|
if newsplitv:
|
|
# Check type
|
|
oldvalue = splitv[0][0]
|
|
newvalue = newsplitv[0][0]
|
|
if oldvalue != newvalue:
|
|
filechanges.append(FileChange(path, FileChange.changetype_type, oldvalue, newvalue))
|
|
# Check permissions
|
|
oldvalue = splitv[0][1:]
|
|
newvalue = newsplitv[0][1:]
|
|
if oldvalue != newvalue:
|
|
filechanges.append(FileChange(path, FileChange.changetype_perms, oldvalue, newvalue))
|
|
# Check owner/group
|
|
oldvalue = '%s/%s' % (splitv[1], splitv[2])
|
|
newvalue = '%s/%s' % (newsplitv[1], newsplitv[2])
|
|
if oldvalue != newvalue:
|
|
filechanges.append(FileChange(path, FileChange.changetype_ownergroup, oldvalue, newvalue))
|
|
# Check symlink target
|
|
if newsplitv[0][0] == 'l':
|
|
if splitv.count > 3:
|
|
oldvalue = splitv[3]
|
|
else:
|
|
oldvalue = None
|
|
newvalue = newsplitv[3]
|
|
if oldvalue != newvalue:
|
|
filechanges.append(FileChange(path, FileChange.changetype_link, oldvalue, newvalue))
|
|
else:
|
|
filechanges.append(FileChange(path, FileChange.changetype_remove))
|
|
|
|
# Whatever is left over has been added
|
|
for path in bdict:
|
|
filechanges.append(FileChange(path, FileChange.changetype_add))
|
|
|
|
return filechanges
|
|
|
|
|
|
def compare_lists(alines, blines):
|
|
removed = list(set(alines) - set(blines))
|
|
added = list(set(blines) - set(alines))
|
|
|
|
filechanges = []
|
|
for pkg in removed:
|
|
filechanges.append(FileChange(pkg, FileChange.changetype_remove))
|
|
for pkg in added:
|
|
filechanges.append(FileChange(pkg, FileChange.changetype_add))
|
|
|
|
return filechanges
|
|
|
|
|
|
def compare_dict_blobs(path, ablob, bblob, report_all):
|
|
adict = blob_to_dict(ablob)
|
|
bdict = blob_to_dict(bblob)
|
|
|
|
changes = []
|
|
for key in adict:
|
|
if report_all or key in monitor_fields:
|
|
if adict[key] != bdict[key]:
|
|
if (not report_all) and key in numeric_fields:
|
|
aval = int(adict[key])
|
|
bval = int(bdict[key])
|
|
percentchg = ((bval - aval) / float(aval)) * 100
|
|
if percentchg < monitor_numeric_threshold:
|
|
continue
|
|
chg = ChangeRecord(path, key, adict[key], bdict[key])
|
|
changes.append(chg)
|
|
return changes
|
|
|
|
|
|
def process_changes(repopath, revision1, revision2 = 'HEAD', report_all = False):
|
|
repo = git.Repo(repopath)
|
|
assert repo.bare == False
|
|
commit = repo.commit(revision1)
|
|
diff = commit.diff(revision2)
|
|
|
|
changes = []
|
|
for d in diff.iter_change_type('M'):
|
|
path = os.path.dirname(d.a_blob.path)
|
|
if path.startswith('packages/'):
|
|
changes.extend(compare_dict_blobs(path, d.a_blob, d.b_blob, report_all))
|
|
elif path.startswith('images/'):
|
|
filename = os.path.basename(d.a_blob.path)
|
|
if filename in img_monitor_files:
|
|
if filename == 'files-in-image.txt':
|
|
alines = d.a_blob.data_stream.read().splitlines()
|
|
blines = d.b_blob.data_stream.read().splitlines()
|
|
filechanges = compare_file_lists(alines,blines)
|
|
if filechanges:
|
|
chg = ChangeRecord(path, filename, None, None)
|
|
chg.filechanges = filechanges
|
|
changes.append(chg)
|
|
elif filename == 'installed-package-names.txt':
|
|
alines = d.a_blob.data_stream.read().splitlines()
|
|
blines = d.b_blob.data_stream.read().splitlines()
|
|
filechanges = compare_lists(alines,blines)
|
|
if filechanges:
|
|
chg = ChangeRecord(path, filename, None, None)
|
|
chg.filechanges = filechanges
|
|
changes.append(chg)
|
|
else:
|
|
chg = ChangeRecord(path, filename, d.a_blob.data_stream.read(), d.b_blob.data_stream.read())
|
|
changes.append(chg)
|
|
elif filename == 'image-info.txt':
|
|
changes.extend(compare_dict_blobs(path, d.a_blob, d.b_blob, report_all))
|
|
|
|
return changes
|