perfcmp

#!/usr/bin/env python
"""Compare 2 perf runs.

To use, specify the local directories that contain
the run information::

    $ ./perfcmp /results/2016-01-01-1111/ /results/2016-01-01-2222/

"""
import os
import json
import argparse

from colorama import Fore, Style
from tabulate import tabulate


class RunComparison(object):

    MEMORY_FIELDS = ['average_memory', 'max_memory']
    TIME_FIELDS = ['total_time']
    # Fields that aren't memory or time fields, they require
    # no special formatting.
    OTHER_FIELDS = ['average_cpu']

    def __init__(self, old_summary, new_summary):
        self.old_summary = old_summary
        self.new_summary = new_summary

    def iter_field_names(self):
        for field in self.TIME_FIELDS + self.MEMORY_FIELDS + self.OTHER_FIELDS:
            yield field

    def old(self, field):
        value = self.old_summary[field]
        return self._format(field, value)

    def old_suffix(self, field):
        value = self.old_summary[field]
        return self._format_suffix(field, value)

    def new_suffix(self, field):
        value = self.new_summary[field]
        return self._format_suffix(field, value)

    def _format_suffix(self, field, value):
        if field in self.TIME_FIELDS:
            return 'sec'
        elif field in self.OTHER_FIELDS:
            return ''
        else:
            # The suffix depends on the actual value.
            return self._human_readable_size(value)[1]

    def old_stddev(self, field):
        real_field = 'std_dev_%s' % field
        return self.old(real_field)

    def new(self, field):
        value = self.new_summary[field]
        return self._format(field, value)

    def new_stddev(self, field):
        real_field = 'std_dev_%s' % field
        return self.new(real_field)

    def _format(self, field, value):
        if field.startswith('std_dev_'):
            field = field[len('std_dev_'):]
        if field in self.MEMORY_FIELDS:
            return self._human_readable_size(value)[0]
        elif field in self.TIME_FIELDS:
            return '%-3.2f' % value
        else:
            return '%.2f' % value

    def _human_readable_size(self, value):
        hummanize_suffixes = ('KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB')
        base = 1024
        bytes_int = float(value)

        if bytes_int == 1:
            return '1 Byte'
        elif bytes_int < base:
            return '%d Bytes' % bytes_int

        for i, suffix in enumerate(hummanize_suffixes):
            unit = base ** (i+2)
            if round((bytes_int / unit) * base) < base:
                return ['%.2f' % (base * bytes_int / unit), suffix]

    def diff_percent(self, field):
        diff_percent = (
            (self.new_summary[field] - self.old_summary[field]) /
            float(self.old_summary[field])) * 100
        return diff_percent


def compare_runs(old_dir, new_dir):
    for dirname in os.listdir(old_dir):
        old_run_dir = os.path.join(old_dir, dirname)
        new_run_dir = os.path.join(new_dir, dirname)
        if not os.path.isdir(old_run_dir):
            continue
        old_summary = get_summary(old_run_dir)
        new_summary = get_summary(new_run_dir)
        comp = RunComparison(old_summary, new_summary)
        header = [Style.BRIGHT + dirname + Style.RESET_ALL,
                  Style.BRIGHT + 'old' + Style.RESET_ALL,
                  # Numeric suffix (MiB, GiB, sec).
                  '',
                  'std_dev',
                  Style.BRIGHT + 'new' + Style.RESET_ALL,
                  # Numeric suffix (MiB, GiB, sec).
                  '',
                  'std_dev',
                  Style.BRIGHT + 'delta' + Style.RESET_ALL]
        rows = []
        for field in comp.iter_field_names():
            row = [field, comp.old(field), comp.old_suffix(field),
                   comp.old_stddev(field), comp.new(field),
                   comp.new_suffix(field), comp.new_stddev(field)]
            diff_percent = comp.diff_percent(field)
            diff_percent_str = '%.2f%%' % diff_percent
            if diff_percent < 0:
                diff_percent_str = (
                    Fore.GREEN + diff_percent_str + Style.RESET_ALL)
            else:
                diff_percent_str = (
                    Fore.RED + diff_percent_str + Style.RESET_ALL)
            row.append(diff_percent_str)
            rows.append(row)
        print(tabulate(rows, headers=header, tablefmt='plain'))
        print('')


def get_summary(benchmark_dir):
    summary_json = os.path.join(benchmark_dir, 'summary.json')
    with open(summary_json) as f:
        return json.load(f)


def main():
    parser = argparse.ArgumentParser(description='__doc__')
    parser.add_argument('oldrunid', help='Path to old run idir')
    parser.add_argument('newrunid', help='Local to new run dir')
    args = parser.parse_args()
    compare_runs(args.oldrunid, args.newrunid)


if __name__ == '__main__':
    main()