mirrors/poky
1
0
Fork 0
mirror of https://git.yoctoproject.org/poky synced 2026-09-24 22:36:21 +02:00
Code Issues Packages Projects Releases Wiki Activity

scripts/buildstats-diff: introduce --diff-attr

Browse Source 
A new command line option for choosing which "attribute" of the
buildstats to compare. At first, the already supported 'cputime' is the
only available option. But, refactoring done in this patch should make
it easy to add new attribute types.

(From OE-Core rev: 0782825138731b3f1e6a8e05d723c1d5cd60c90c)

Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
Signed-off-by: Ross Burton <ross.burton@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Markus Lehtonen
2016-09-29 17:28:02 +03:00
committed by Richard Purdie
parent e05309194a
commit 01a2b5823b
1 changed files with 84 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

140
scripts/buildstats-diff
View File

@@ -48,8 +48,7 @@ TIMEZONES = {'UTC': TimeZone(0, 'UTC'),
'EET': TimeZone(7200, 'EET'), 'EET': TimeZone(7200, 'EET'),
'EEST': TimeZone(10800, 'EEST')} 'EEST': TimeZone(10800, 'EEST')}
taskdiff_fields = ('pkg', 'pkg_op', 'task', 'task_op', 'value1', 'value2',
taskdiff_fields = ('pkg', 'pkg_op', 'task', 'task_op', 'cputime1', 'cputime2',
'absdiff', 'reldiff') 'absdiff', 'reldiff')
TaskDiff = namedtuple('TaskDiff', ' '.join(taskdiff_fields)) TaskDiff = namedtuple('TaskDiff', ' '.join(taskdiff_fields))
@@ -272,16 +271,46 @@ def print_ver_diff(bs1, bs2):
print(fmt_str.format(pkg, field1, field2, maxlen=maxlen)) print(fmt_str.format(pkg, field1, field2, maxlen=maxlen))
def print_task_diff(bs1, bs2, val_type, min_val=0, min_absdiff=0, sort_by=('absdiff',)):
def print_task_diff(bs1, bs2, min_val=0, min_absdiff=0, sort_by=('absdiff',)):
"""Diff task execution times""" """Diff task execution times"""
def val_to_str(val, human_readable=False):
"""Convert raw value to printable string"""
def hms_time(secs):
"""Get time in human-readable HH:MM:SS format"""
h = int(secs / 3600)
m = int((secs % 3600) / 60)
s = secs % 60
if h == 0:
return "{:02d}:{:04.1f}".format(m, s)
else:
return "{:d}:{:02d}:{:04.1f}".format(h, m, s)
if val_type == 'cputime':
if human_readable:
return hms_time(val)
else:
return "{:.1f}s".format(val)
else:
return str(val)
def sum_vals(buildstats):
"""Get cumulative sum of all tasks"""
total = 0.0
for recipe_data in buildstats.values():
for bs_task in recipe_data['tasks'].values():
total += getattr(bs_task, val_type)
return total
tasks_diff = [] tasks_diff = []
if min_val: if min_val:
print("Ignoring tasks shorter than {}s".format(min_val)) print("Ignoring tasks less than {} ({})".format(
val_to_str(min_val, True), val_to_str(min_val)))
if min_absdiff: if min_absdiff:
print("Ignoring time differences shorter than {}s".format(min_absdiff)) print("Ignoring differences less than {} ({})".format(
val_to_str(min_absdiff, True), val_to_str(min_absdiff)))
# Prepare the data
pkgs = set(bs1.keys()).union(set(bs2.keys())) pkgs = set(bs1.keys()).union(set(bs2.keys()))
for pkg in pkgs: for pkg in pkgs:
tasks1 = bs1[pkg]['tasks'] if pkg in bs1 else {} tasks1 = bs1[pkg]['tasks'] if pkg in bs1 else {}
@@ -294,25 +323,27 @@ def print_task_diff(bs1, bs2, min_val=0, min_absdiff=0, sort_by=('absdiff',)):
pkg_op = ' ' pkg_op = ' '
for task in set(tasks1.keys()).union(set(tasks2.keys())): for task in set(tasks1.keys()).union(set(tasks2.keys())):
t1 = bs1[pkg]['tasks'][task].cputime if task in tasks1 else 0 val1 = getattr(bs1[pkg]['tasks'][task], val_type) if task in tasks1 else 0
t2 = bs2[pkg]['tasks'][task].cputime if task in tasks2 else 0 val2 = getattr(bs2[pkg]['tasks'][task], val_type) if task in tasks2 else 0
task_op = ' ' task_op = ' '
if t1 == 0: if val1 == 0:
reldiff = float('inf') reldiff = float('inf')
task_op = '+ ' task_op = '+ '
else: else:
reldiff = 100 * (t2 - t1) / t1 reldiff = 100 * (val2 - val1) / val1
if t2 == 0: if val2 == 0:
task_op = '- ' task_op = '- '
cputime = max(t1, t2) if max(val1, val2) < min_val:
if cputime < min_val: log.debug("Filtering out %s:%s (%s)", pkg, task,
log.debug("Filtering out %s:%s (%0.1fs)", pkg, task, cputime) val_to_str(max(val1, val2)))
continue continue
if abs(t2-t1) < min_absdiff: if abs(val2 - val1) < min_absdiff:
log.debug("Filtering out %s:%s (difference of %0.1fs)", pkg, task, t2-t1) log.debug("Filtering out %s:%s (difference of %s)", pkg, task,
val_to_str(val2-val1))
continue continue
tasks_diff.append(TaskDiff(pkg, pkg_op, task, task_op, t1, t2, t2-t1, reldiff)) tasks_diff.append(TaskDiff(pkg, pkg_op, task, task_op, val1, val2,
val2-val1, reldiff))
# Sort our list # Sort our list
for field in reversed(sort_by): for field in reversed(sort_by):
@@ -323,17 +354,18 @@ def print_task_diff(bs1, bs2, min_val=0, min_absdiff=0, sort_by=('absdiff',)):
reverse = False reverse = False
tasks_diff = sorted(tasks_diff, key=attrgetter(field), reverse=reverse) tasks_diff = sorted(tasks_diff, key=attrgetter(field), reverse=reverse)
linedata = [(' ', 'PKG', ' ', 'TASK', 'ABSDIFF', 'RELDIFF', 'CPUTIME1', 'CPUTIME2')] linedata = [(' ', 'PKG', ' ', 'TASK', 'ABSDIFF', 'RELDIFF',
val_type.upper() + '1', val_type.upper() + '2')]
field_lens = dict([('len_{}'.format(i), len(f)) for i, f in enumerate(linedata[0])]) field_lens = dict([('len_{}'.format(i), len(f)) for i, f in enumerate(linedata[0])])
# Prepare fields in string format and measure field lengths # Prepare fields in string format and measure field lengths
for diff in tasks_diff: for diff in tasks_diff:
task_prefix = diff.task_op if diff.pkg_op == ' ' else ' ' task_prefix = diff.task_op if diff.pkg_op == ' ' else ' '
linedata.append((diff.pkg_op, diff.pkg, task_prefix, diff.task, linedata.append((diff.pkg_op, diff.pkg, task_prefix, diff.task,
'{:+.1f}'.format(diff.absdiff), val_to_str(diff.absdiff),
'{:+.1f}%'.format(diff.reldiff), '{:+.1f}%'.format(diff.reldiff),
'{:.1f}s'.format(diff.cputime1), val_to_str(diff.value1),
'{:.1f}s'.format(diff.cputime2))) val_to_str(diff.value2)))
for i, field in enumerate(linedata[-1]): for i, field in enumerate(linedata[-1]):
key = 'len_{}'.format(i) key = 'len_{}'.format(i)
if len(field) > field_lens[key]: if len(field) > field_lens[key]:
@@ -345,33 +377,14 @@ def print_task_diff(bs1, bs2, min_val=0, min_absdiff=0, sort_by=('absdiff',)):
print("{:{len_0}}{:{len_1}} {:{len_2}}{:{len_3}} {:>{len_4}} {:>{len_5}} {:>{len_6}} -> {:{len_7}}".format( print("{:{len_0}}{:{len_1}} {:{len_2}}{:{len_3}} {:>{len_4}} {:>{len_5}} {:>{len_6}} -> {:{len_7}}".format(
*fields, **field_lens)) *fields, **field_lens))
# Print summary of the diffs
def print_timediff_summary(bs1, bs2): total1 = sum_vals(bs1)
"""Print summary of the timediffs""" total2 = sum_vals(bs2)
def total_cputime(buildstats): print("\nCumulative {}:".format(val_type))
sum = 0.0 print (" {} {:+.1f}% {} ({}) -> {} ({})".format(
for recipe_data in buildstats.values(): val_to_str(total2 - total1), 100 * (total2-total1) / total1,
for bs_task in recipe_data['tasks'].values(): val_to_str(total1, True), val_to_str(total1),
sum += bs_task.cputime val_to_str(total2, True), val_to_str(total2)))
return sum
def hms_time(secs):
"""Get time in human-readable HH:MM:SS format"""
h = int(secs / 3600)
m = int((secs % 3600) / 60)
s = secs % 60
if h == 0:
return "{:02d}:{:04.1f}".format(m, s)
else:
return "{:d}:{:02d}:{:04.1f}".format(h, m, s)
total1 = total_cputime(bs1)
total2 = total_cputime(bs2)
print("\nCumulative CPU Time:")
print (" {:+.1f}s {:+.1f}% {} ({:.1f}s) -> {} ({:.1f}s)".format(
total2 - total1, 100 * (total2-total1) / total1,
hms_time(total1), total1, hms_time(total2), total2))
def parse_args(argv): def parse_args(argv):
@@ -382,15 +395,21 @@ Script for comparing buildstats of two separate builds."""
formatter_class=argparse.ArgumentDefaultsHelpFormatter, formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description=description) description=description)
min_val_defaults = {'cputime': 3.0}
min_absdiff_defaults = {'cputime': 1.0}
parser.add_argument('--debug', '-d', action='store_true', parser.add_argument('--debug', '-d', action='store_true',
help="Verbose logging") help="Verbose logging")
parser.add_argument('--ver-diff', action='store_true', parser.add_argument('--ver-diff', action='store_true',
help="Show package version differences and exit") help="Show package version differences and exit")
parser.add_argument('--min-val', default=3.0, type=float, parser.add_argument('--diff-attr', default='cputime', choices=('cputime',),
help="Filter out tasks shorter than MIN_VAL seconds") help="Buildstat attribute which to compare")
parser.add_argument('--min-absdiff', default=1.0, type=float, parser.add_argument('--min-val', default=min_val_defaults, type=float,
help="Filter out tasks whose difference in cputime is " help="Filter out tasks less than MIN_VAL. "
"less that MIN_ABSDIFF seconds") "Default depends on --diff-attr.")
parser.add_argument('--min-absdiff', default=min_absdiff_defaults, type=float,
help="Filter out tasks whose difference is less than "
"MIN_ABSDIFF, Default depends on --diff-attr.")
parser.add_argument('--sort-by', default='absdiff', parser.add_argument('--sort-by', default='absdiff',
help="Comma-separated list of field sort order. " help="Comma-separated list of field sort order. "
"Prepend the field name with '-' for reversed sort. " "Prepend the field name with '-' for reversed sort. "
@@ -398,7 +417,16 @@ Script for comparing buildstats of two separate builds."""
parser.add_argument('buildstats1', metavar='BUILDSTATS1', help="'Left' buildstat") parser.add_argument('buildstats1', metavar='BUILDSTATS1', help="'Left' buildstat")
parser.add_argument('buildstats2', metavar='BUILDSTATS2', help="'Right' buildstat") parser.add_argument('buildstats2', metavar='BUILDSTATS2', help="'Right' buildstat")
return parser.parse_args(argv) args = parser.parse_args(argv)
# Handle defaults for the filter arguments
if args.min_val is min_val_defaults:
args.min_val = min_val_defaults[args.diff_attr]
if args.min_absdiff is min_absdiff_defaults:
args.min_absdiff = min_absdiff_defaults[args.diff_attr]
return args
def main(argv=None): def main(argv=None):
"""Script entry point""" """Script entry point"""
@@ -422,8 +450,8 @@ def main(argv=None):
if args.ver_diff: if args.ver_diff:
print_ver_diff(bs1, bs2) print_ver_diff(bs1, bs2)
else: else:
print_task_diff(bs1, bs2, args.min_val, args.min_absdiff, sort_by) print_task_diff(bs1, bs2, args.diff_attr, args.min_val,
print_timediff_summary(bs1, bs2) args.min_absdiff, sort_by)
return 0 return 0