compare.py

#!/usr/bin/env python
"""Tool to filter, organize, compare and display benchmarking results. Usefull
for smaller datasets. It works great with a few dozen runs it is not designed to
deal with hundreds.
Requires the pandas library to be installed."""
from __future__ import print_function

import pandas as pd
from scipy import stats
import sys
import os.path
import re
import numbers
import argparse

GEOMEAN_ROW = "Geomean difference"


def read_lit_json(filename):
    import json

    jsondata = json.load(open(filename))
    columns = []
    columnindexes = {}
    names = set()
    info_columns = ["hash"]
    # Pass1: Figure out metrics (= the column index)
    if "tests" not in jsondata:
        print("%s: Could not find toplevel 'tests' key")
        sys.exit(1)
    for test in jsondata["tests"]:
        name = test.get("name")
        if name is None:
            sys.stderr.write("Error: Found unnamed test\n" % name)
            sys.exit(1)
        if name in names:
            sys.stderr.write("Error: Multiple tests with name '%s'\n" % name)
            sys.exit(1)
        if "metrics" not in test:
            print("Warning: '%s' has no metrics, skipping!" % test["name"])
            continue
        names.add(name)
        for name in test["metrics"].keys():
            if name not in columnindexes:
                columnindexes[name] = len(columns)
                columns.append(name)
        for name in test.keys():
            if name not in columnindexes and name in info_columns:
                columnindexes[name] = len(columns)
                columns.append(name)

    # Pass2 actual data construction
    nan = float("NaN")
    data = []
    testnames = []
    for test in jsondata["tests"]:
        if "metrics" not in test:
            continue
        name = test["name"]
        if "shortname" in test:
            name = test["shortname"]
        testnames.append(name)

        datarow = [nan] * len(columns)
        for (metricname, value) in test["metrics"].items():
            datarow[columnindexes[metricname]] = value
        for (name, value) in test.items():
            index = columnindexes.get(name)
            if index is not None:
                datarow[index] = test[name]
        data.append(datarow)
    index = pd.Index(testnames, name="Program")
    return pd.DataFrame(data=data, index=index, columns=columns)


def read_report_simple_csv(filename):
    return pd.read_csv(filename, na_values=["*"], index_col=0, header=0)


def read(name):
    if name.endswith(".json"):
        return read_lit_json(name)
    if name.endswith(".csv"):
        return read_report_simple_csv(name)
    raise Exception("Cannot determine file format")


def readmulti(filenames):
    # Read datasets
    datasetnames = []
    datasets = []
    prev_index = None
    for filename in filenames:
        data = read(filename)
        name = os.path.basename(filename)
        # drop .json/.csv suffix; TODO: Should we rather do this in the printing
        # logic?
        for ext in [".csv", ".json"]:
            if name.endswith(ext):
                name = name[: -len(ext)]
        datasets.append(data)
        suffix = ""
        count = 0
        while True:
            if name + suffix not in datasetnames:
                break
            suffix = str(count)
            count += 1

        datasetnames.append(name + suffix)
        # Warn if index names are different
        if prev_index is not None and prev_index.name != data.index.name:
            sys.stderr.write(
                "Warning: Mismatched index names: '%s' vs '%s'\n"
                % (prev_index.name, data.index.name)
            )
        prev_index = data.index
    # Merge datasets
    d = pd.concat(datasets, axis=0, names=["run"], keys=datasetnames)
    return d


def get_values(values):
    # Create data view without diff column.
    if "diff" in values.columns:
        values = values[[c for c in values.columns if c != "diff"]]
    has_two_runs = len(values.columns) == 2
    if has_two_runs:
        return (values.iloc[:, 0], values.iloc[:, 1])
    else:
        return (values.min(axis=1), values.max(axis=1))


def add_diff_column(metric, values, absolute_diff=False):
    values0, values1 = get_values(values[metric])
    values0.fillna(0.0, inplace=True)
    values1.fillna(0.0, inplace=True)
    # Quotient or absolute difference?
    if absolute_diff:
        values[(metric, "diff")] = values1 - values0
    else:
        values[(metric, "diff")] = (values1 / values0) - 1.0
    return values


def add_geomean_row(metrics, data, dataout):
    """
    Normalize values1 over values0, compute geomean difference and add a
    summary row to dataout.
    """
    gm = pd.DataFrame(index=[GEOMEAN_ROW], columns=dataout.columns, dtype="float64")
    for metric in metrics:
        values0, values1 = get_values(data[metric])
        # Avoid infinite values in the diff and instead use NaN, as otherwise
        # the computation of the geometric mean will fail.
        values0 = values0.replace({0: float("NaN")})
        relative = values1 / values0
        gm_diff = stats.gmean(relative.dropna()) - 1.0
        gm[(metric, "diff")] = gm_diff
    gm.Program = GEOMEAN_ROW
    return pd.concat([dataout, gm])


def filter_failed(data, key="Exec"):
    return data.loc[data[key] == "pass"]


def filter_short(data, threshold, key="Exec_Time"):
    return data.loc[data[key] >= threshold]


def filter_same_hash(data, key="hash"):
    assert key in data.columns
    assert data.index.get_level_values(0).nunique() > 1

    return data.groupby(level=1).filter(lambda x: x[key].nunique() != 1)


def filter_blacklist(data, blacklist):
    return data.loc[~(data.index.get_level_values(1).isin(blacklist))]


def print_filter_stats(reason, before, after):
    n_before = len(before.groupby(level=1))
    n_after = len(after.groupby(level=1))
    n_filtered = n_before - n_after
    if n_filtered != 0:
        print("%s: %s (filtered out)" % (reason, n_filtered))


# Truncate a string to a maximum length by keeping a prefix, a suffix and ...
# in the middle
def truncate(string, prefix_len, suffix_len):
    return re.sub(
        "^(.{%d}).*(.{%d})$" % (prefix_len, suffix_len), r"\g<1>...\g<2>", string
    )


# Search for common prefixes and suffixes in a list of names and return
# a (prefix,suffix) tuple that specifies how many characters can be dropped
# for the prefix/suffix. The numbers will be small enough that no name will
# become shorter than min_len characters.
def determine_common_prefix_suffix(names, min_len=8):
    if len(names) <= 1:
        return (0, 0)
    name0 = names.iloc[0]
    prefix = name0
    prefix_len = len(name0)
    suffix = name0
    suffix_len = len(name0)
    shortest_name = len(name0)
    for name in names:
        if len(name) < shortest_name:
            shortest_name = len(name)
        while prefix_len > 0 and name[:prefix_len] != prefix:
            prefix_len -= 1
            prefix = name0[:prefix_len]
        while suffix_len > 0 and name[-suffix_len:] != suffix:
            suffix_len -= 1
            suffix = name0[-suffix_len:]

    if suffix[0] != "." and suffix[0] != "_":
        suffix_len = 0
    suffix_len = max(0, min(shortest_name - prefix_len - min_len, suffix_len))
    prefix_len = max(0, min(shortest_name - suffix_len, prefix_len))
    return (prefix_len, suffix_len)


def format_relative_diff(value):
    if not isinstance(value, numbers.Integral):
        return "%4.1f%%" % (value * 100.0)
    else:
        return "%-5d" % value


def print_result(
    d,
    limit_output=True,
    shorten_names=True,
    minimal_names=False,
    show_diff_column=True,
    sortkey="diff",
    sort_by_abs=True,
    absolute_diff=False,
):
    metrics = d.columns.levels[0]
    if sort_by_abs:
        d = d.sort_values(by=(metrics[0], sortkey), key=pd.Series.abs, ascending=False)
    else:
        d = d.sort_values(by=(metrics[0], sortkey), ascending=False)

    # Ensure that the columns are grouped by metric (rather than having the
    # diffs at the end of the line).
    d = d.reindex(columns=d.columns.levels[0], level=0)

    if not show_diff_column:
        # Remove all diff columns (using level=1 since level 0 is the metric).
        d.drop(labels="diff", level=1, axis=1, inplace=True)
    dataout = d
    if limit_output:
        # Take 15 topmost elements
        dataout = dataout.head(15)

    formatters = dict()
    if not absolute_diff:
        for m in metrics:
            formatters[(m, "diff")] = format_relative_diff
    # Turn index into a column so we can format it...
    formatted_program = dataout.index.to_series()
    if shorten_names:

        def format_name(name, common_prefix, common_suffix):
            name = name[common_prefix:]
            if common_suffix > 0:
                name = name[:-common_suffix]
            return "%-45s" % truncate(name, 10, 30)

        def strip_name_fully(name):
            name = name.split("/")[-1]
            if name.endswith(".test"):
                name = name[:-5]
            return name

        # The to_string formatters argument appears to be ignored for
        # dtype=object, so transform the program column manually.
        if minimal_names:
            formatted_program = formatted_program.map(strip_name_fully)
        else:
            drop_prefix, drop_suffix = determine_common_prefix_suffix(formatted_program)
            formatted_program = formatted_program.map(
                lambda name: format_name(name, drop_prefix, drop_suffix)
            )
    dataout.insert(0, "Program", formatted_program)
    # Add the geometric mean row after we have formatted the program names
    # as it will otherwise interfere with common prefix/suffix computation.
    if show_diff_column and not absolute_diff:
        # geometric mean only makes sense for relative differences.
        dataout = add_geomean_row(metrics, d, dataout)

    def float_format(x):
        if x == "":
            return ""
        return "%6.2f" % (x,)

    pd.set_option("display.max_colwidth", 0)
    pd.set_option("display.width", 0)
    # Print an empty value instead of NaN (for the geomean row).
    out = dataout.to_string(
        index=False,
        justify="left",
        na_rep="",
        float_format=float_format,
        formatters=formatters,
    )
    print(out)
    print(d.describe())


def main():
    parser = argparse.ArgumentParser(prog="compare.py")
    parser.add_argument("-a", "--all", action="store_true")
    parser.add_argument("-f", "--full", action="store_true")
    parser.add_argument("-m", "--metric", action="append", dest="metrics", default=[])
    parser.add_argument(
        "--nodiff", action="store_false", dest="show_diff", default=None
    )
    parser.add_argument("--diff", action="store_true", dest="show_diff")
    parser.add_argument(
        "--absolute-diff",
        action="store_true",
        help="Use an absolute instead of a relative difference",
    )
    parser.add_argument(
        "--filter-short",
        nargs="?",
        dest="filter_short",
        default=None,
        help="Filter benchmarks with execution times less than N seconds (default 1.0s)",
    )
    parser.add_argument(
        "--no-filter-failed", action="store_false", dest="filter_failed", default=True
    )
    parser.add_argument(
        "--filter-hash", action="store_true", dest="filter_hash", default=False
    )
    parser.add_argument("--filter-blacklist", dest="filter_blacklist", default=None)
    parser.add_argument(
        "--merge-average",
        action="store_const",
        dest="merge_function",
        const=pd.DataFrame.mean,
        default=pd.DataFrame.min,
    )
    parser.add_argument(
        "--merge-min",
        action="store_const",
        dest="merge_function",
        const=pd.DataFrame.min,
    )
    parser.add_argument(
        "--merge-max",
        action="store_const",
        dest="merge_function",
        const=pd.DataFrame.max,
    )
    parser.add_argument(
        "--lhs-name", default="lhs", help="Name used to describe left side in 'vs' mode"
    )
    parser.add_argument(
        "--rhs-name",
        default="rhs",
        help="Name used to describe right side in 'vs' mode",
    )
    parser.add_argument(
        "files",
        metavar="FILE",
        nargs="+",
        help="To compare two groups of results, put 'vs' between them",
    )
    parser.add_argument(
        "--minimal-names", action="store_true", dest="minimal_names", default=False
    )
    parser.add_argument(
        "--no-abs-sort",
        action="store_true",
        dest="no_abs_sort",
        default=False,
        help="Don't use abs() when sorting results",
    )
    config = parser.parse_args()

    if config.show_diff is None:
        config.show_diff = len(config.files) > 1

    # If only --filter-short is provided, i.e. its optional argument is
    # omitted, we default to threshold of 1 second to filter out apps and
    # results with a execution time less than that.
    filter_short_threshold = 1.0

    # If the optional argument to --filter-short is omitted, we need to take
    # care of this case and command line:
    #     --filter-short FILE [FILE ...]
    # I.e., we need to recognise that FILE is not the optional argument to
    # --filter-short. The way we do this, is to try converting the option value
    # to a float, and if that fails, we insert it back into the files list (in
    # the first position).
    if config.filter_short is not None:
        try:
            filter_short_threshold = float(config.filter_short)
        except:
            config.files.insert(0, config.filter_short)

    # Read inputs
    files = config.files
    if "vs" in files:
        split = files.index("vs")
        lhs = files[0:split]
        rhs = files[split + 1 :]

        # Filter minimum of lhs and rhs
        lhs_d = readmulti(lhs)
        lhs_merged = lhs_d.groupby(level=1).apply(config.merge_function)
        rhs_d = readmulti(rhs)
        rhs_merged = rhs_d.groupby(level=1).apply(config.merge_function)

        # Combine to new dataframe
        data = pd.concat(
            [lhs_merged, rhs_merged],
            names=["l/r"],
            keys=[config.lhs_name, config.rhs_name],
        )
    else:
        data = readmulti(files)

    # Decide which metric to display / what is our "main" metric
    metrics = config.metrics
    if len(metrics) == 0:
        defaults = ["Exec_Time", "exec_time", "Value", "Runtime"]
        for defkey in defaults:
            if defkey in data.columns:
                metrics = [defkey]
                break
    if len(metrics) == 0:
        sys.stderr.write("No default metric found and none specified\n")
        sys.stderr.write("Available metrics:\n")
        for column in data.columns:
            sys.stderr.write("\t%s\n" % column)
        sys.exit(1)
    for metric in metrics:
        problem = False
        if metric not in data.columns:
            sys.stderr.write("Unknown metric '%s'\n" % metric)
            problem = True
        if problem:
            sys.exit(1)

    # Filter data
    proggroup = data.groupby(level=1)
    initial_size = len(proggroup.indices)
    print("Tests: %s" % (initial_size,))
    if config.filter_failed and hasattr(data, "Exec"):
        newdata = filter_failed(data)
        print_filter_stats("Failed", data, newdata)
        newdata = newdata.drop("Exec", 1)
        data = newdata
    if config.filter_short:
        newdata = filter_short(data, filter_short_threshold, metric)
        print_filter_stats("Short Running", data, newdata)
        data = newdata
    if (
        config.filter_hash
        and "hash" in data.columns
        and data.index.get_level_values(0).nunique() > 1
    ):
        newdata = filter_same_hash(data)
        print_filter_stats("Same hash", data, newdata)
        data = newdata
    if config.filter_blacklist:
        blacklist = open(config.filter_blacklist).readlines()
        blacklist = [line.strip() for line in blacklist]
        newdata = filter_blacklist(data, blacklist)
        print_filter_stats("In Blacklist", data, newdata)
        data = newdata
    final_size = len(data.groupby(level=1))
    if final_size != initial_size:
        print("Remaining: %d" % (final_size,))

    # Reduce / add columns
    print("Metric: %s" % (",".join(metrics),))
    if len(metrics) > 0:
        data = data[metrics]

    data = data.unstack(level=0)

    for metric in data.columns.levels[0]:
        data = add_diff_column(metric, data, absolute_diff=config.absolute_diff)

    sortkey = "diff"
    # TODO: should we still be sorting by diff even if the diff is hidden?
    if len(config.files) == 1:
        sortkey = data.columns.levels[1][0]

    # Print data
    print("")
    shorten_names = not config.full
    limit_output = (not config.all) and (not config.full)
    print_result(
        data,
        limit_output,
        shorten_names,
        config.minimal_names,
        config.show_diff,
        sortkey,
        config.no_abs_sort,
        config.absolute_diff,
    )


if __name__ == "__main__":
    main()