full_stack_tests_plot.py

#!/usr/bin/env python
# Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
#
# Use of this source code is governed by a BSD-style license
# that can be found in the LICENSE file in the root of the source
# tree. An additional intellectual property rights grant can be found
# in the file PATENTS.  All contributing project authors may
# be found in the AUTHORS file in the root of the source tree.
"""Generate graphs for data generated by loopback tests.

Usage examples:
  Show end to end time for a single full stack test.
  ./full_stack_tests_plot.py -df end_to_end -o 600 --frames 1000 vp9_data.txt

  Show simultaneously PSNR and encoded frame size for two different runs of
  full stack test. Averaged over a cycle of 200 frames. Used e.g. for
  screenshare slide test.
  ./full_stack_tests_plot.py -c 200 -df psnr -drf encoded_frame_size \\
                             before.txt after.txt

  Similar to the previous test, but multiple graphs.
  ./full_stack_tests_plot.py -c 200 -df psnr vp8.txt vp9.txt --next \\
                             -c 200 -df sender_time vp8.txt vp9.txt --next \\
                             -c 200 -df end_to_end vp8.txt vp9.txt
"""

import argparse
from collections import defaultdict
import itertools
import sys
import matplotlib.pyplot as plt
import numpy

# Fields
DROPPED = 0
INPUT_TIME = 1  # ms (timestamp)
SEND_TIME = 2  # ms (timestamp)
RECV_TIME = 3  # ms (timestamp)
RENDER_TIME = 4  # ms (timestamp)
ENCODED_FRAME_SIZE = 5  # bytes
PSNR = 6
SSIM = 7
ENCODE_TIME = 8  # ms (time interval)

TOTAL_RAW_FIELDS = 9

SENDER_TIME = TOTAL_RAW_FIELDS + 0
RECEIVER_TIME = TOTAL_RAW_FIELDS + 1
END_TO_END = TOTAL_RAW_FIELDS + 2
RENDERED_DELTA = TOTAL_RAW_FIELDS + 3

FIELD_MASK = 255

# Options
HIDE_DROPPED = 256
RIGHT_Y_AXIS = 512

# internal field id, field name, title
_FIELDS = [
    # Raw
    (DROPPED, "dropped", "dropped"),
    (INPUT_TIME, "input_time_ms", "input time"),
    (SEND_TIME, "send_time_ms", "send time"),
    (RECV_TIME, "recv_time_ms", "recv time"),
    (ENCODED_FRAME_SIZE, "encoded_frame_size", "encoded frame size"),
    (PSNR, "psnr", "PSNR"),
    (SSIM, "ssim", "SSIM"),
    (RENDER_TIME, "render_time_ms", "render time"),
    (ENCODE_TIME, "encode_time_ms", "encode time"),
    # Auto-generated
    (SENDER_TIME, "sender_time", "sender time"),
    (RECEIVER_TIME, "receiver_time", "receiver time"),
    (END_TO_END, "end_to_end", "end to end"),
    (RENDERED_DELTA, "rendered_delta", "rendered delta"),
]

NAME_TO_ID = {field[1]: field[0] for field in _FIELDS}
ID_TO_TITLE = {field[0]: field[2] for field in _FIELDS}


def FieldArgToId(arg):
    if arg == "none":
        return None
    if arg in NAME_TO_ID:
        return NAME_TO_ID[arg]
    if arg + "_ms" in NAME_TO_ID:
        return NAME_TO_ID[arg + "_ms"]
    raise Exception("Unrecognized field name \"{}\"".format(arg))


class PlotLine(object):
    """Data for a single graph line."""

    def __init__(self, label, values, flags):
        self.label = label
        self.values = values
        self.flags = flags


class Data(object):
    """Object representing one full stack test."""

    def __init__(self, filename):
        self.title = ""
        self.length = 0
        self.samples = defaultdict(list)

        self._ReadSamples(filename)

    def _ReadSamples(self, filename):
        """Reads graph data from the given file."""
        f = open(filename)
        it = iter(f)

        self.title = it.next().strip()
        self.length = int(it.next())
        field_names = [name.strip() for name in it.next().split()]
        field_ids = [NAME_TO_ID[name] for name in field_names]

        for field_id in field_ids:
            self.samples[field_id] = [0.0] * self.length

        for sample_id in xrange(self.length):
            for col, value in enumerate(it.next().split()):
                self.samples[field_ids[col]][sample_id] = float(value)

        self._SubtractFirstInputTime()
        self._GenerateAdditionalData()

        f.close()

    def _SubtractFirstInputTime(self):
        offset = self.samples[INPUT_TIME][0]
        for field in [INPUT_TIME, SEND_TIME, RECV_TIME, RENDER_TIME]:
            if field in self.samples:
                self.samples[field] = [x - offset for x in self.samples[field]]

    def _GenerateAdditionalData(self):
        """Calculates sender time, receiver time etc. from the raw data."""
        s = self.samples
        last_render_time = 0
        for field_id in [
                SENDER_TIME, RECEIVER_TIME, END_TO_END, RENDERED_DELTA
        ]:
            s[field_id] = [0] * self.length

        for k in range(self.length):
            s[SENDER_TIME][k] = s[SEND_TIME][k] - s[INPUT_TIME][k]

            decoded_time = s[RENDER_TIME][k]
            s[RECEIVER_TIME][k] = decoded_time - s[RECV_TIME][k]
            s[END_TO_END][k] = decoded_time - s[INPUT_TIME][k]
            if not s[DROPPED][k]:
                if k > 0:
                    s[RENDERED_DELTA][k] = decoded_time - last_render_time
                last_render_time = decoded_time

    def _Hide(self, values):
        """
    Replaces values for dropped frames with None.
    These values are then skipped by the Plot() method.
    """

        return [
            None if self.samples[DROPPED][k] else values[k]
            for k in range(len(values))
        ]

    def AddSamples(self, config, target_lines_list):
        """Creates graph lines from the current data set with given config."""
        for field in config.fields:
            # field is None means the user wants just to skip the color.
            if field is None:
                target_lines_list.append(None)
                continue

            field_id = field & FIELD_MASK
            values = self.samples[field_id]

            if field & HIDE_DROPPED:
                values = self._Hide(values)

            target_lines_list.append(
                PlotLine(self.title + " " + ID_TO_TITLE[field_id], values,
                         field & ~FIELD_MASK))


def AverageOverCycle(values, length):
    """
  Returns the list:
    [
        avg(values[0], values[length], ...),
        avg(values[1], values[length + 1], ...),
        ...
        avg(values[length - 1], values[2 * length - 1], ...),
    ]

  Skips None values when calculating the average value.
  """

    total = [0.0] * length
    count = [0] * length
    for k, val in enumerate(values):
        if val is not None:
            total[k % length] += val
            count[k % length] += 1

    result = [0.0] * length
    for k in range(length):
        result[k] = total[k] / count[k] if count[k] else None
    return result


class PlotConfig(object):
    """Object representing a single graph."""

    def __init__(self,
                 fields,
                 data_list,
                 cycle_length=None,
                 frames=None,
                 offset=0,
                 output_filename=None,
                 title="Graph"):
        self.fields = fields
        self.data_list = data_list
        self.cycle_length = cycle_length
        self.frames = frames
        self.offset = offset
        self.output_filename = output_filename
        self.title = title

    def Plot(self, ax1):
        lines = []
        for data in self.data_list:
            if not data:
                # Add None lines to skip the colors.
                lines.extend([None] * len(self.fields))
            else:
                data.AddSamples(self, lines)

        def _SliceValues(values):
            if self.offset:
                values = values[self.offset:]
            if self.frames:
                values = values[:self.frames]
            return values

        length = None
        for line in lines:
            if line is None:
                continue

            line.values = _SliceValues(line.values)
            if self.cycle_length:
                line.values = AverageOverCycle(line.values, self.cycle_length)

            if length is None:
                length = len(line.values)
            elif length != len(line.values):
                raise Exception("All arrays should have the same length!")

        ax1.set_xlabel("Frame", fontsize="large")
        if any(line.flags & RIGHT_Y_AXIS for line in lines if line):
            ax2 = ax1.twinx()
            ax2.set_xlabel("Frame", fontsize="large")
        else:
            ax2 = None

        # Have to implement color_cycle manually, due to two scales in a graph.
        color_cycle = ["b", "r", "g", "c", "m", "y", "k"]
        color_iter = itertools.cycle(color_cycle)

        for line in lines:
            if not line:
                color_iter.next()
                continue

            if self.cycle_length:
                x = numpy.array(range(self.cycle_length))
            else:
                x = numpy.array(
                    range(self.offset, self.offset + len(line.values)))
            y = numpy.array(line.values)
            ax = ax2 if line.flags & RIGHT_Y_AXIS else ax1
            ax.Plot(x,
                    y,
                    "o-",
                    label=line.label,
                    markersize=3.0,
                    linewidth=1.0,
                    color=color_iter.next())

        ax1.grid(True)
        if ax2:
            ax1.legend(loc="upper left", shadow=True, fontsize="large")
            ax2.legend(loc="upper right", shadow=True, fontsize="large")
        else:
            ax1.legend(loc="best", shadow=True, fontsize="large")


def LoadFiles(filenames):
    result = []
    for filename in filenames:
        if filename in LoadFiles.cache:
            result.append(LoadFiles.cache[filename])
        else:
            data = Data(filename)
            LoadFiles.cache[filename] = data
            result.append(data)
    return result


LoadFiles.cache = {}


def GetParser():
    class CustomAction(argparse.Action):
        def __call__(self, parser, namespace, values, option_string=None):
            if "ordered_args" not in namespace:
                namespace.ordered_args = []
            namespace.ordered_args.append((self.dest, values))

    parser = argparse.ArgumentParser(
        description=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter)

    parser.add_argument("-c",
                        "--cycle_length",
                        nargs=1,
                        action=CustomAction,
                        type=int,
                        help="Cycle length over which to average the values.")
    parser.add_argument(
        "-f",
        "--field",
        nargs=1,
        action=CustomAction,
        help="Name of the field to show. Use 'none' to skip a color.")
    parser.add_argument("-r",
                        "--right",
                        nargs=0,
                        action=CustomAction,
                        help="Use right Y axis for given field.")
    parser.add_argument("-d",
                        "--drop",
                        nargs=0,
                        action=CustomAction,
                        help="Hide values for dropped frames.")
    parser.add_argument("-o",
                        "--offset",
                        nargs=1,
                        action=CustomAction,
                        type=int,
                        help="Frame offset.")
    parser.add_argument("-n",
                        "--next",
                        nargs=0,
                        action=CustomAction,
                        help="Separator for multiple graphs.")
    parser.add_argument(
        "--frames",
        nargs=1,
        action=CustomAction,
        type=int,
        help="Frame count to show or take into account while averaging.")
    parser.add_argument("-t",
                        "--title",
                        nargs=1,
                        action=CustomAction,
                        help="Title of the graph.")
    parser.add_argument("-O",
                        "--output_filename",
                        nargs=1,
                        action=CustomAction,
                        help="Use to save the graph into a file. "
                        "Otherwise, a window will be shown.")
    parser.add_argument(
        "files",
        nargs="+",
        action=CustomAction,
        help="List of text-based files generated by loopback tests.")
    return parser


def _PlotConfigFromArgs(args, graph_num):
    # Pylint complains about using kwargs, so have to do it this way.
    cycle_length = None
    frames = None
    offset = 0
    output_filename = None
    title = "Graph"

    fields = []
    files = []
    mask = 0
    for key, values in args:
        if key == "cycle_length":
            cycle_length = values[0]
        elif key == "frames":
            frames = values[0]
        elif key == "offset":
            offset = values[0]
        elif key == "output_filename":
            output_filename = values[0]
        elif key == "title":
            title = values[0]
        elif key == "drop":
            mask |= HIDE_DROPPED
        elif key == "right":
            mask |= RIGHT_Y_AXIS
        elif key == "field":
            field_id = FieldArgToId(values[0])
            fields.append(field_id | mask if field_id is not None else None)
            mask = 0  # Reset mask after the field argument.
        elif key == "files":
            files.extend(values)

    if not files:
        raise Exception(
            "Missing file argument(s) for graph #{}".format(graph_num))
    if not fields:
        raise Exception(
            "Missing field argument(s) for graph #{}".format(graph_num))

    return PlotConfig(fields,
                      LoadFiles(files),
                      cycle_length=cycle_length,
                      frames=frames,
                      offset=offset,
                      output_filename=output_filename,
                      title=title)


def PlotConfigsFromArgs(args):
    """Generates plot configs for given command line arguments."""
    # The way it works:
    #   First we detect separators -n/--next and split arguments into groups, one
    #   for each plot. For each group, we partially parse it with
    #   argparse.ArgumentParser, modified to remember the order of arguments.
    #   Then we traverse the argument list and fill the PlotConfig.
    args = itertools.groupby(args, lambda x: x in ["-n", "--next"])
    prep_args = list(list(group) for match, group in args if not match)

    parser = GetParser()
    plot_configs = []
    for index, raw_args in enumerate(prep_args):
        graph_args = parser.parse_args(raw_args).ordered_args
        plot_configs.append(_PlotConfigFromArgs(graph_args, index))
    return plot_configs


def ShowOrSavePlots(plot_configs):
    for config in plot_configs:
        fig = plt.figure(figsize=(14.0, 10.0))
        ax = fig.add_subPlot(1, 1, 1)

        plt.title(config.title)
        config.Plot(ax)
        if config.output_filename:
            print "Saving to", config.output_filename
            fig.savefig(config.output_filename)
            plt.close(fig)

    plt.show()


if __name__ == "__main__":
    ShowOrSavePlots(PlotConfigsFromArgs(sys.argv[1:]))