mirror of
https://git.yoctoproject.org/poky
synced 2026-02-20 08:29:42 +01:00
scripts/buildstats-diff: move code to lib/buildstats.py
Move over code from buildstats-diff to new scripts/lib/buildstats.py module in order to share code related to buildstats processing. Also, refactor the code, introducing new classes to make the code readable, maintainable and easier to debug. [YOCTO #11381] (From OE-Core rev: 8a2cd9afc95919737d8e75234e78bbc52e1494a1) Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com> Signed-off-by: Ross Burton <ross.burton@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Markus Lehtonen
committed by
Richard Purdie
Richard Purdie
parent
873707489f
commit
b5fb3dd904
@@ -15,15 +15,18 @@
|
||||
#
|
||||
import argparse
|
||||
import glob
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from collections import namedtuple
|
||||
from operator import attrgetter
|
||||
|
||||
# Import oe libs
|
||||
scripts_path = os.path.dirname(os.path.realpath(__file__))
|
||||
sys.path.append(os.path.join(scripts_path, 'lib'))
|
||||
from buildstats import BuildStats, diff_buildstats, taskdiff_fields
|
||||
|
||||
|
||||
# Setup logging
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
|
||||
log = logging.getLogger()
|
||||
@@ -34,196 +37,16 @@ class ScriptError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
taskdiff_fields = ('pkg', 'pkg_op', 'task', 'task_op', 'value1', 'value2',
|
||||
'absdiff', 'reldiff')
|
||||
TaskDiff = namedtuple('TaskDiff', ' '.join(taskdiff_fields))
|
||||
|
||||
|
||||
class BSTask(dict):
|
||||
def __init__(self, *args, **kwargs):
|
||||
self['start_time'] = None
|
||||
self['elapsed_time'] = None
|
||||
self['status'] = None
|
||||
self['iostat'] = {}
|
||||
self['rusage'] = {}
|
||||
self['child_rusage'] = {}
|
||||
super(BSTask, self).__init__(*args, **kwargs)
|
||||
|
||||
@property
|
||||
def cputime(self):
|
||||
"""Sum of user and system time taken by the task"""
|
||||
rusage = self['rusage']['ru_stime'] + self['rusage']['ru_utime']
|
||||
if self['child_rusage']:
|
||||
# Child rusage may have been optimized out
|
||||
return rusage + self['child_rusage']['ru_stime'] + self['child_rusage']['ru_utime']
|
||||
else:
|
||||
return rusage
|
||||
|
||||
@property
|
||||
def walltime(self):
|
||||
"""Elapsed wall clock time"""
|
||||
return self['elapsed_time']
|
||||
|
||||
@property
|
||||
def read_bytes(self):
|
||||
"""Bytes read from the block layer"""
|
||||
return self['iostat']['read_bytes']
|
||||
|
||||
@property
|
||||
def write_bytes(self):
|
||||
"""Bytes written to the block layer"""
|
||||
return self['iostat']['write_bytes']
|
||||
|
||||
@property
|
||||
def read_ops(self):
|
||||
"""Number of read operations on the block layer"""
|
||||
if self['child_rusage']:
|
||||
# Child rusage may have been optimized out
|
||||
return self['rusage']['ru_inblock'] + self['child_rusage']['ru_inblock']
|
||||
else:
|
||||
return self['rusage']['ru_inblock']
|
||||
|
||||
@property
|
||||
def write_ops(self):
|
||||
"""Number of write operations on the block layer"""
|
||||
if self['child_rusage']:
|
||||
# Child rusage may have been optimized out
|
||||
return self['rusage']['ru_oublock'] + self['child_rusage']['ru_oublock']
|
||||
else:
|
||||
return self['rusage']['ru_oublock']
|
||||
|
||||
|
||||
def read_buildstats_file(buildstat_file):
|
||||
"""Convert buildstat text file into dict/json"""
|
||||
bs_task = BSTask()
|
||||
log.debug("Reading task buildstats from %s", buildstat_file)
|
||||
end_time = None
|
||||
with open(buildstat_file) as fobj:
|
||||
for line in fobj.readlines():
|
||||
key, val = line.split(':', 1)
|
||||
val = val.strip()
|
||||
if key == 'Started':
|
||||
start_time = float(val)
|
||||
bs_task['start_time'] = start_time
|
||||
elif key == 'Ended':
|
||||
end_time = float(val)
|
||||
elif key.startswith('IO '):
|
||||
split = key.split()
|
||||
bs_task['iostat'][split[1]] = int(val)
|
||||
elif key.find('rusage') >= 0:
|
||||
split = key.split()
|
||||
ru_key = split[-1]
|
||||
if ru_key in ('ru_stime', 'ru_utime'):
|
||||
val = float(val)
|
||||
else:
|
||||
val = int(val)
|
||||
ru_type = 'rusage' if split[0] == 'rusage' else \
|
||||
'child_rusage'
|
||||
bs_task[ru_type][ru_key] = val
|
||||
elif key == 'Status':
|
||||
bs_task['status'] = val
|
||||
if end_time is not None and start_time is not None:
|
||||
bs_task['elapsed_time'] = end_time - start_time
|
||||
else:
|
||||
raise ScriptError("{} looks like a invalid buildstats file".format(buildstat_file))
|
||||
return bs_task
|
||||
|
||||
|
||||
def read_buildstats_dir(bs_dir):
|
||||
"""Read buildstats directory"""
|
||||
def split_nevr(nevr):
|
||||
"""Split name and version information from recipe "nevr" string"""
|
||||
n_e_v, revision = nevr.rsplit('-', 1)
|
||||
match = re.match(r'^(?P<name>\S+)-((?P<epoch>[0-9]{1,5})_)?(?P<version>[0-9]\S*)$',
|
||||
n_e_v)
|
||||
if not match:
|
||||
# If we're not able to parse a version starting with a number, just
|
||||
# take the part after last dash
|
||||
match = re.match(r'^(?P<name>\S+)-((?P<epoch>[0-9]{1,5})_)?(?P<version>[^-]+)$',
|
||||
n_e_v)
|
||||
name = match.group('name')
|
||||
version = match.group('version')
|
||||
epoch = match.group('epoch')
|
||||
return name, epoch, version, revision
|
||||
|
||||
if not os.path.isfile(os.path.join(bs_dir, 'build_stats')):
|
||||
raise ScriptError("{} does not look like a buildstats directory".format(bs_dir))
|
||||
|
||||
log.debug("Reading buildstats directory %s", bs_dir)
|
||||
|
||||
buildstats = {}
|
||||
subdirs = os.listdir(bs_dir)
|
||||
for dirname in subdirs:
|
||||
recipe_dir = os.path.join(bs_dir, dirname)
|
||||
if not os.path.isdir(recipe_dir):
|
||||
continue
|
||||
name, epoch, version, revision = split_nevr(dirname)
|
||||
recipe_bs = {'nevr': dirname,
|
||||
'name': name,
|
||||
'epoch': epoch,
|
||||
'version': version,
|
||||
'revision': revision,
|
||||
'tasks': {}}
|
||||
for task in os.listdir(recipe_dir):
|
||||
recipe_bs['tasks'][task] = [read_buildstats_file(
|
||||
os.path.join(recipe_dir, task))]
|
||||
if name in buildstats:
|
||||
raise ScriptError("Cannot handle multiple versions of the same "
|
||||
"package ({})".format(name))
|
||||
buildstats[name] = recipe_bs
|
||||
|
||||
return buildstats
|
||||
|
||||
|
||||
def bs_append(dst, src):
|
||||
"""Append data from another buildstats"""
|
||||
if set(dst.keys()) != set(src.keys()):
|
||||
raise ScriptError("Refusing to join buildstats, set of packages is "
|
||||
"different")
|
||||
for pkg, data in dst.items():
|
||||
if data['nevr'] != src[pkg]['nevr']:
|
||||
raise ScriptError("Refusing to join buildstats, package version "
|
||||
"differs: {} vs. {}".format(data['nevr'], src[pkg]['nevr']))
|
||||
if set(data['tasks'].keys()) != set(src[pkg]['tasks'].keys()):
|
||||
raise ScriptError("Refusing to join buildstats, set of tasks "
|
||||
"in {} differ".format(pkg))
|
||||
for taskname, taskdata in data['tasks'].items():
|
||||
taskdata.extend(src[pkg]['tasks'][taskname])
|
||||
|
||||
|
||||
def read_buildstats_json(path):
|
||||
"""Read buildstats from JSON file"""
|
||||
buildstats = {}
|
||||
with open(path) as fobj:
|
||||
bs_json = json.load(fobj)
|
||||
for recipe_bs in bs_json:
|
||||
if recipe_bs['name'] in buildstats:
|
||||
raise ScriptError("Cannot handle multiple versions of the same "
|
||||
"package ({})".format(recipe_bs['name']))
|
||||
|
||||
if recipe_bs['epoch'] is None:
|
||||
recipe_bs['nevr'] = "{}-{}-{}".format(recipe_bs['name'], recipe_bs['version'], recipe_bs['revision'])
|
||||
else:
|
||||
recipe_bs['nevr'] = "{}-{}_{}-{}".format(recipe_bs['name'], recipe_bs['epoch'], recipe_bs['version'], recipe_bs['revision'])
|
||||
|
||||
for task, data in recipe_bs['tasks'].copy().items():
|
||||
recipe_bs['tasks'][task] = [BSTask(data)]
|
||||
|
||||
buildstats[recipe_bs['name']] = recipe_bs
|
||||
|
||||
return buildstats
|
||||
|
||||
|
||||
def read_buildstats(path, multi):
|
||||
"""Read buildstats"""
|
||||
if not os.path.exists(path):
|
||||
raise ScriptError("No such file or directory: {}".format(path))
|
||||
|
||||
if os.path.isfile(path):
|
||||
return read_buildstats_json(path)
|
||||
return BuildStats.from_file_json(path)
|
||||
|
||||
if os.path.isfile(os.path.join(path, 'build_stats')):
|
||||
return read_buildstats_dir(path)
|
||||
return BuildStats.from_dir(path)
|
||||
|
||||
# Handle a non-buildstat directory
|
||||
subpaths = sorted(glob.glob(path + '/*'))
|
||||
@@ -238,17 +61,16 @@ def read_buildstats(path, multi):
|
||||
bs = None
|
||||
for subpath in subpaths:
|
||||
if os.path.isfile(subpath):
|
||||
tmpbs = read_buildstats_json(subpath)
|
||||
_bs = BuildStats.from_file_json(subpath)
|
||||
else:
|
||||
tmpbs = read_buildstats_dir(subpath)
|
||||
if not bs:
|
||||
bs = tmpbs
|
||||
_bs = BuildStats.from_dir(subpath)
|
||||
if bs is None:
|
||||
bs = _bs
|
||||
else:
|
||||
log.debug("Joining buildstats")
|
||||
bs_append(bs, tmpbs)
|
||||
|
||||
bs.aggregate(_bs)
|
||||
if not bs:
|
||||
raise ScriptError("No buildstats found under {}".format(path))
|
||||
|
||||
return bs
|
||||
|
||||
|
||||
@@ -266,11 +88,11 @@ def print_ver_diff(bs1, bs2):
|
||||
common_pkgs = pkgs2.intersection(pkgs1)
|
||||
if common_pkgs:
|
||||
for pkg in common_pkgs:
|
||||
if bs1[pkg]['epoch'] != bs2[pkg]['epoch']:
|
||||
if bs1[pkg].epoch != bs2[pkg].epoch:
|
||||
echanged.append(pkg)
|
||||
elif bs1[pkg]['version'] != bs2[pkg]['version']:
|
||||
elif bs1[pkg].version != bs2[pkg].version:
|
||||
vchanged.append(pkg)
|
||||
elif bs1[pkg]['revision'] != bs2[pkg]['revision']:
|
||||
elif bs1[pkg].revision != bs2[pkg].revision:
|
||||
rchanged.append(pkg)
|
||||
else:
|
||||
unchanged.append(pkg)
|
||||
@@ -288,37 +110,37 @@ def print_ver_diff(bs1, bs2):
|
||||
print("\nNEW PACKAGES:")
|
||||
print("-------------")
|
||||
for pkg in sorted(new_pkgs):
|
||||
print(fmt_str.format(pkg, bs2[pkg]['nevr'], maxlen=maxlen))
|
||||
print(fmt_str.format(pkg, bs2[pkg].nevr, maxlen=maxlen))
|
||||
|
||||
if deleted_pkgs:
|
||||
print("\nDELETED PACKAGES:")
|
||||
print("-----------------")
|
||||
for pkg in sorted(deleted_pkgs):
|
||||
print(fmt_str.format(pkg, bs1[pkg]['nevr'], maxlen=maxlen))
|
||||
print(fmt_str.format(pkg, bs1[pkg].nevr, maxlen=maxlen))
|
||||
|
||||
fmt_str = " {0:{maxlen}} {1:<20} ({2})"
|
||||
if rchanged:
|
||||
print("\nREVISION CHANGED:")
|
||||
print("-----------------")
|
||||
for pkg in sorted(rchanged):
|
||||
field1 = "{} -> {}".format(pkg, bs1[pkg]['revision'], bs2[pkg]['revision'])
|
||||
field2 = "{} -> {}".format(bs1[pkg]['nevr'], bs2[pkg]['nevr'])
|
||||
field1 = "{} -> {}".format(pkg, bs1[pkg].revision, bs2[pkg].revision)
|
||||
field2 = "{} -> {}".format(bs1[pkg].nevr, bs2[pkg].nevr)
|
||||
print(fmt_str.format(pkg, field1, field2, maxlen=maxlen))
|
||||
|
||||
if vchanged:
|
||||
print("\nVERSION CHANGED:")
|
||||
print("----------------")
|
||||
for pkg in sorted(vchanged):
|
||||
field1 = "{} -> {}".format(bs1[pkg]['version'], bs2[pkg]['version'])
|
||||
field2 = "{} -> {}".format(bs1[pkg]['nevr'], bs2[pkg]['nevr'])
|
||||
field1 = "{} -> {}".format(bs1[pkg].version, bs2[pkg].version)
|
||||
field2 = "{} -> {}".format(bs1[pkg].nevr, bs2[pkg].nevr)
|
||||
print(fmt_str.format(pkg, field1, field2, maxlen=maxlen))
|
||||
|
||||
if echanged:
|
||||
print("\nEPOCH CHANGED:")
|
||||
print("--------------")
|
||||
for pkg in sorted(echanged):
|
||||
field1 = "{} -> {}".format(bs1[pkg]['epoch'], bs2[pkg]['epoch'])
|
||||
field2 = "{} -> {}".format(bs1[pkg]['nevr'], bs2[pkg]['nevr'])
|
||||
field1 = "{} -> {}".format(bs1[pkg].epoch, bs2[pkg].epoch)
|
||||
field2 = "{} -> {}".format(bs1[pkg].nevr, bs2[pkg].nevr)
|
||||
print(fmt_str.format(pkg, field1, field2, maxlen=maxlen))
|
||||
|
||||
|
||||
@@ -359,12 +181,10 @@ def print_task_diff(bs1, bs2, val_type, min_val=0, min_absdiff=0, sort_by=('absd
|
||||
"""Get cumulative sum of all tasks"""
|
||||
total = 0.0
|
||||
for recipe_data in buildstats.values():
|
||||
for bs_task in recipe_data['tasks'].values():
|
||||
total += sum([getattr(b, val_type) for b in bs_task]) / len(bs_task)
|
||||
for bs_task in recipe_data.tasks.values():
|
||||
total += getattr(bs_task, val_type)
|
||||
return total
|
||||
|
||||
tasks_diff = []
|
||||
|
||||
if min_val:
|
||||
print("Ignoring tasks less than {} ({})".format(
|
||||
val_to_str(min_val, True), val_to_str(min_val)))
|
||||
@@ -373,49 +193,7 @@ def print_task_diff(bs1, bs2, val_type, min_val=0, min_absdiff=0, sort_by=('absd
|
||||
val_to_str(min_absdiff, True), val_to_str(min_absdiff)))
|
||||
|
||||
# Prepare the data
|
||||
pkgs = set(bs1.keys()).union(set(bs2.keys()))
|
||||
for pkg in pkgs:
|
||||
tasks1 = bs1[pkg]['tasks'] if pkg in bs1 else {}
|
||||
tasks2 = bs2[pkg]['tasks'] if pkg in bs2 else {}
|
||||
if not tasks1:
|
||||
pkg_op = '+ '
|
||||
elif not tasks2:
|
||||
pkg_op = '- '
|
||||
else:
|
||||
pkg_op = ' '
|
||||
|
||||
for task in set(tasks1.keys()).union(set(tasks2.keys())):
|
||||
task_op = ' '
|
||||
if task in tasks1:
|
||||
# Average over all values
|
||||
val1 = [getattr(b, val_type) for b in bs1[pkg]['tasks'][task]]
|
||||
val1 = sum(val1) / len(val1)
|
||||
else:
|
||||
task_op = '+ '
|
||||
val1 = 0
|
||||
if task in tasks2:
|
||||
# Average over all values
|
||||
val2 = [getattr(b, val_type) for b in bs2[pkg]['tasks'][task]]
|
||||
val2 = sum(val2) / len(val2)
|
||||
else:
|
||||
val2 = 0
|
||||
task_op = '- '
|
||||
|
||||
if val1 == 0:
|
||||
reldiff = float('inf')
|
||||
else:
|
||||
reldiff = 100 * (val2 - val1) / val1
|
||||
|
||||
if max(val1, val2) < min_val:
|
||||
log.debug("Filtering out %s:%s (%s)", pkg, task,
|
||||
val_to_str(max(val1, val2)))
|
||||
continue
|
||||
if abs(val2 - val1) < min_absdiff:
|
||||
log.debug("Filtering out %s:%s (difference of %s)", pkg, task,
|
||||
val_to_str(val2-val1))
|
||||
continue
|
||||
tasks_diff.append(TaskDiff(pkg, pkg_op, task, task_op, val1, val2,
|
||||
val2-val1, reldiff))
|
||||
tasks_diff = diff_buildstats(bs1, bs2, val_type, min_val, min_absdiff)
|
||||
|
||||
# Sort our list
|
||||
for field in reversed(sort_by):
|
||||
|
||||
301
scripts/lib/buildstats.py
Normal file
301
scripts/lib/buildstats.py
Normal file
@@ -0,0 +1,301 @@
|
||||
#
|
||||
# Copyright (c) 2017, Intel Corporation.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify it
|
||||
# under the terms and conditions of the GNU General Public License,
|
||||
# version 2, as published by the Free Software Foundation.
|
||||
#
|
||||
# This program is distributed in the hope it will be useful, but WITHOUT
|
||||
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
# more details.
|
||||
#
|
||||
"""Functionality for analyzing buildstats"""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from collections import namedtuple
|
||||
from statistics import mean
|
||||
|
||||
|
||||
log = logging.getLogger()
|
||||
|
||||
|
||||
taskdiff_fields = ('pkg', 'pkg_op', 'task', 'task_op', 'value1', 'value2',
|
||||
'absdiff', 'reldiff')
|
||||
TaskDiff = namedtuple('TaskDiff', ' '.join(taskdiff_fields))
|
||||
|
||||
|
||||
class BSError(Exception):
|
||||
"""Error handling of buildstats"""
|
||||
pass
|
||||
|
||||
|
||||
class BSTask(dict):
|
||||
def __init__(self, *args, **kwargs):
|
||||
self['start_time'] = None
|
||||
self['elapsed_time'] = None
|
||||
self['status'] = None
|
||||
self['iostat'] = {}
|
||||
self['rusage'] = {}
|
||||
self['child_rusage'] = {}
|
||||
super(BSTask, self).__init__(*args, **kwargs)
|
||||
|
||||
@property
|
||||
def cputime(self):
|
||||
"""Sum of user and system time taken by the task"""
|
||||
rusage = self['rusage']['ru_stime'] + self['rusage']['ru_utime']
|
||||
if self['child_rusage']:
|
||||
# Child rusage may have been optimized out
|
||||
return rusage + self['child_rusage']['ru_stime'] + self['child_rusage']['ru_utime']
|
||||
else:
|
||||
return rusage
|
||||
|
||||
@property
|
||||
def walltime(self):
|
||||
"""Elapsed wall clock time"""
|
||||
return self['elapsed_time']
|
||||
|
||||
@property
|
||||
def read_bytes(self):
|
||||
"""Bytes read from the block layer"""
|
||||
return self['iostat']['read_bytes']
|
||||
|
||||
@property
|
||||
def write_bytes(self):
|
||||
"""Bytes written to the block layer"""
|
||||
return self['iostat']['write_bytes']
|
||||
|
||||
@property
|
||||
def read_ops(self):
|
||||
"""Number of read operations on the block layer"""
|
||||
if self['child_rusage']:
|
||||
# Child rusage may have been optimized out
|
||||
return self['rusage']['ru_inblock'] + self['child_rusage']['ru_inblock']
|
||||
else:
|
||||
return self['rusage']['ru_inblock']
|
||||
|
||||
@property
|
||||
def write_ops(self):
|
||||
"""Number of write operations on the block layer"""
|
||||
if self['child_rusage']:
|
||||
# Child rusage may have been optimized out
|
||||
return self['rusage']['ru_oublock'] + self['child_rusage']['ru_oublock']
|
||||
else:
|
||||
return self['rusage']['ru_oublock']
|
||||
|
||||
@classmethod
|
||||
def from_file(cls, buildstat_file):
|
||||
"""Read buildstat text file"""
|
||||
bs_task = cls()
|
||||
log.debug("Reading task buildstats from %s", buildstat_file)
|
||||
end_time = None
|
||||
with open(buildstat_file) as fobj:
|
||||
for line in fobj.readlines():
|
||||
key, val = line.split(':', 1)
|
||||
val = val.strip()
|
||||
if key == 'Started':
|
||||
start_time = float(val)
|
||||
bs_task['start_time'] = start_time
|
||||
elif key == 'Ended':
|
||||
end_time = float(val)
|
||||
elif key.startswith('IO '):
|
||||
split = key.split()
|
||||
bs_task['iostat'][split[1]] = int(val)
|
||||
elif key.find('rusage') >= 0:
|
||||
split = key.split()
|
||||
ru_key = split[-1]
|
||||
if ru_key in ('ru_stime', 'ru_utime'):
|
||||
val = float(val)
|
||||
else:
|
||||
val = int(val)
|
||||
ru_type = 'rusage' if split[0] == 'rusage' else \
|
||||
'child_rusage'
|
||||
bs_task[ru_type][ru_key] = val
|
||||
elif key == 'Status':
|
||||
bs_task['status'] = val
|
||||
if end_time is not None and start_time is not None:
|
||||
bs_task['elapsed_time'] = end_time - start_time
|
||||
else:
|
||||
raise BSError("{} looks like a invalid buildstats file".format(buildstat_file))
|
||||
return bs_task
|
||||
|
||||
|
||||
class BSTaskAggregate(object):
|
||||
"""Class representing multiple runs of the same task"""
|
||||
properties = ('cputime', 'walltime', 'read_bytes', 'write_bytes',
|
||||
'read_ops', 'write_ops')
|
||||
|
||||
def __init__(self, tasks=None):
|
||||
self._tasks = tasks or []
|
||||
self._properties = {}
|
||||
|
||||
def __getattr__(self, name):
|
||||
if name in self.properties:
|
||||
if name not in self._properties:
|
||||
# Calculate properties on demand only. We only provide mean
|
||||
# value, so far
|
||||
self._properties[name] = mean([getattr(t, name) for t in self._tasks])
|
||||
return self._properties[name]
|
||||
else:
|
||||
raise AttributeError("'BSTaskAggregate' has no attribute '{}'".format(name))
|
||||
|
||||
def append(self, task):
|
||||
"""Append new task"""
|
||||
# Reset pre-calculated properties
|
||||
assert isinstance(task, BSTask), "Type is '{}' instead of 'BSTask'".format(type(task))
|
||||
self._properties = {}
|
||||
self._tasks.append(task)
|
||||
|
||||
|
||||
class BSRecipe(object):
|
||||
"""Class representing buildstats of one recipe"""
|
||||
def __init__(self, name, epoch, version, revision):
|
||||
self.name = name
|
||||
self.epoch = epoch
|
||||
self.version = version
|
||||
self.revision = revision
|
||||
if epoch is None:
|
||||
self.nevr = "{}-{}-{}".format(name, version, revision)
|
||||
else:
|
||||
self.nevr = "{}-{}_{}-{}".format(name, epoch, version, revision)
|
||||
self.tasks = {}
|
||||
|
||||
def aggregate(self, bsrecipe):
|
||||
"""Aggregate data of another recipe buildstats"""
|
||||
if self.nevr != bsrecipe.nevr:
|
||||
raise ValueError("Refusing to aggregate buildstats, recipe version "
|
||||
"differs: {} vs. {}".format(self.nevr, bsrecipe.nevr))
|
||||
if set(self.tasks.keys()) != set(bsrecipe.tasks.keys()):
|
||||
raise ValueError("Refusing to aggregate buildstats, set of tasks "
|
||||
"in {} differ".format(self.name))
|
||||
|
||||
for taskname, taskdata in bsrecipe.tasks.items():
|
||||
if not isinstance(self.tasks[taskname], BSTaskAggregate):
|
||||
self.tasks[taskname] = BSTaskAggregate([self.tasks[taskname]])
|
||||
self.tasks[taskname].append(taskdata)
|
||||
|
||||
|
||||
class BuildStats(dict):
|
||||
"""Class representing buildstats of one build"""
|
||||
|
||||
@classmethod
|
||||
def from_json(cls, bs_json):
|
||||
"""Create new BuildStats object from JSON object"""
|
||||
buildstats = cls()
|
||||
for recipe in bs_json:
|
||||
if recipe['name'] in buildstats:
|
||||
raise BSError("Cannot handle multiple versions of the same "
|
||||
"package ({})".format(recipe['name']))
|
||||
bsrecipe = BSRecipe(recipe['name'], recipe['epoch'],
|
||||
recipe['version'], recipe['revision'])
|
||||
for task, data in recipe['tasks'].items():
|
||||
bsrecipe.tasks[task] = BSTask(data)
|
||||
|
||||
buildstats[recipe['name']] = bsrecipe
|
||||
|
||||
return buildstats
|
||||
|
||||
@staticmethod
|
||||
def from_file_json(path):
|
||||
"""Load buildstats from a JSON file"""
|
||||
with open(path) as fobj:
|
||||
bs_json = json.load(fobj)
|
||||
return BuildStats.from_json(bs_json)
|
||||
|
||||
|
||||
@staticmethod
|
||||
def split_nevr(nevr):
|
||||
"""Split name and version information from recipe "nevr" string"""
|
||||
n_e_v, revision = nevr.rsplit('-', 1)
|
||||
match = re.match(r'^(?P<name>\S+)-((?P<epoch>[0-9]{1,5})_)?(?P<version>[0-9]\S*)$',
|
||||
n_e_v)
|
||||
if not match:
|
||||
# If we're not able to parse a version starting with a number, just
|
||||
# take the part after last dash
|
||||
match = re.match(r'^(?P<name>\S+)-((?P<epoch>[0-9]{1,5})_)?(?P<version>[^-]+)$',
|
||||
n_e_v)
|
||||
name = match.group('name')
|
||||
version = match.group('version')
|
||||
epoch = match.group('epoch')
|
||||
return name, epoch, version, revision
|
||||
|
||||
@classmethod
|
||||
def from_dir(cls, path):
|
||||
"""Load buildstats from a buildstats directory"""
|
||||
if not os.path.isfile(os.path.join(path, 'build_stats')):
|
||||
raise BSError("{} does not look like a buildstats directory".format(path))
|
||||
|
||||
log.debug("Reading buildstats directory %s", path)
|
||||
|
||||
buildstats = cls()
|
||||
subdirs = os.listdir(path)
|
||||
for dirname in subdirs:
|
||||
recipe_dir = os.path.join(path, dirname)
|
||||
if not os.path.isdir(recipe_dir):
|
||||
continue
|
||||
name, epoch, version, revision = cls.split_nevr(dirname)
|
||||
bsrecipe = BSRecipe(name, epoch, version, revision)
|
||||
for task in os.listdir(recipe_dir):
|
||||
bsrecipe.tasks[task] = BSTask.from_file(
|
||||
os.path.join(recipe_dir, task))
|
||||
if name in buildstats:
|
||||
raise BSError("Cannot handle multiple versions of the same "
|
||||
"package ({})".format(name))
|
||||
buildstats[name] = bsrecipe
|
||||
|
||||
return buildstats
|
||||
|
||||
def aggregate(self, buildstats):
|
||||
"""Aggregate other buildstats into this"""
|
||||
if set(self.keys()) != set(buildstats.keys()):
|
||||
raise ValueError("Refusing to aggregate buildstats, set of "
|
||||
"recipes is different")
|
||||
for pkg, data in buildstats.items():
|
||||
self[pkg].aggregate(data)
|
||||
|
||||
|
||||
def diff_buildstats(bs1, bs2, stat_attr, min_val=None, min_absdiff=None):
|
||||
"""Compare the tasks of two buildstats"""
|
||||
tasks_diff = []
|
||||
pkgs = set(bs1.keys()).union(set(bs2.keys()))
|
||||
for pkg in pkgs:
|
||||
tasks1 = bs1[pkg].tasks if pkg in bs1 else {}
|
||||
tasks2 = bs2[pkg].tasks if pkg in bs2 else {}
|
||||
if not tasks1:
|
||||
pkg_op = '+'
|
||||
elif not tasks2:
|
||||
pkg_op = '-'
|
||||
else:
|
||||
pkg_op = ' '
|
||||
|
||||
for task in set(tasks1.keys()).union(set(tasks2.keys())):
|
||||
task_op = ' '
|
||||
if task in tasks1:
|
||||
val1 = getattr(bs1[pkg].tasks[task], stat_attr)
|
||||
else:
|
||||
task_op = '+'
|
||||
val1 = 0
|
||||
if task in tasks2:
|
||||
val2 = getattr(bs2[pkg].tasks[task], stat_attr)
|
||||
else:
|
||||
val2 = 0
|
||||
task_op = '-'
|
||||
|
||||
if val1 == 0:
|
||||
reldiff = float('inf')
|
||||
else:
|
||||
reldiff = 100 * (val2 - val1) / val1
|
||||
|
||||
if min_val and max(val1, val2) < min_val:
|
||||
log.debug("Filtering out %s:%s (%s)", pkg, task,
|
||||
max(val1, val2))
|
||||
continue
|
||||
if min_absdiff and abs(val2 - val1) < min_absdiff:
|
||||
log.debug("Filtering out %s:%s (difference of %s)", pkg, task,
|
||||
val2-val1)
|
||||
continue
|
||||
tasks_diff.append(TaskDiff(pkg, pkg_op, task, task_op, val1, val2,
|
||||
val2-val1, reldiff))
|
||||
return tasks_diff
|
||||
Reference in New Issue
Block a user