process-stats-dir.py

#!/usr/bin/python
#
# ==-- process-stats-dir - summarize one or more Swift -stats-output-dirs --==#
#
# This source file is part of the Swift.org open source project
#
# Copyright (c) 2014-2017 Apple Inc. and the Swift project authors
# Licensed under Apache License v2.0 with Runtime Library Exception
#
# See https://swift.org/LICENSE.txt for license information
# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
#
# ==------------------------------------------------------------------------==#
#
# This file processes the contents of one or more directories generated by
# `swiftc -stats-output-dir` and emits summary data, traces etc. for analysis.

import argparse
import csv
import datetime
import json
import os
import platform
import random
import re
import sys
import time
import urllib
import urllib2


class JobStats:

    def __init__(self, jobkind, jobid, module, start_usec, dur_usec,
                 jobargs, stats):
        self.jobkind = jobkind
        self.jobid = jobid
        self.module = module
        self.start_usec = start_usec
        self.dur_usec = dur_usec
        self.jobargs = jobargs
        self.stats = stats

    def is_driver_job(self):
        return self.jobkind == 'driver'

    def is_frontend_job(self):
        return self.jobkind == 'frontend'

    def driver_jobs_ran(self):
        assert(self.is_driver_job())
        return self.stats.get("Driver.NumDriverJobsRun", 0)

    def driver_jobs_skipped(self):
        assert(self.is_driver_job())
        return self.stats.get("Driver.NumDriverJobsSkipped", 0)

    def driver_jobs_total(self):
        assert(self.is_driver_job())
        return self.driver_jobs_ran() + self.driver_jobs_skipped()

    def merged_with(self, other):
        merged_stats = {}
        for k, v in self.stats.items() + other.stats.items():
            merged_stats[k] = v + merged_stats.get(k, 0.0)
        merged_kind = self.jobkind
        if other.jobkind != merged_kind:
            merged_kind = "<merged>"
        merged_module = self.module
        if other.module != merged_module:
            merged_module = "<merged>"
        merged_start = min(self.start_usec, other.start_usec)
        merged_end = max(self.start_usec + self.dur_usec,
                         other.start_usec + other.dur_usec)
        merged_dur = merged_end - merged_start
        return JobStats(merged_kind, random.randint(0, 1000000000),
                        merged_module, merged_start, merged_dur,
                        self.jobargs + other.jobargs, merged_stats)

    def incrementality_percentage(self):
        assert(self.is_driver_job())
        ran = self.driver_jobs_ran()
        total = self.driver_jobs_total()
        return round((float(ran) / float(total)) * 100.0, 2)

    # Return a JSON-formattable object of the form preferred by google chrome's
    # 'catapult' trace-viewer.
    def to_catapult_trace_obj(self):
        return {"name": self.module,
                "cat": self.jobkind,
                "ph": "X",              # "X" == "complete event"
                "pid": self.jobid,
                "tid": 1,
                "ts": self.start_usec,
                "dur": self.dur_usec,
                "args": self.jobargs}

    def start_timestr(self):
        t = datetime.datetime.fromtimestamp(self.start_usec / 1000000.0)
        return t.strftime("%Y-%m-%d %H:%M:%S")

    def end_timestr(self):
        t = datetime.datetime.fromtimestamp((self.start_usec +
                                             self.dur_usec) / 1000000.0)
        return t.strftime("%Y-%m-%d %H:%M:%S")

    def pick_lnt_metric_suffix(self, metric_name):
        if "BytesOutput" in metric_name:
            return "code_size"
        if "RSS" in metric_name or "BytesAllocated" in metric_name:
            return "mem"
        return "compile"

    # Return a JSON-formattable object of the form preferred by LNT's
    # 'submit' format.
    def to_lnt_test_obj(self, args):
        run_info = {
            "run_order": str(args.lnt_order),
            "tag": str(args.lnt_tag),
        }
        run_info.update(dict(args.lnt_run_info))
        stats = self.stats
        return {
            "Machine":
            {
                "Name": args.lnt_machine,
                "Info": dict(args.lnt_machine_info)
            },
            "Run":
            {
                "Start Time": self.start_timestr(),
                "End Time": self.end_timestr(),
                "Info": run_info
            },
            "Tests":
            [
                {
                    "Data": [v],
                    "Info": {},
                    "Name": "%s.%s.%s.%s" % (args.lnt_tag, self.module,
                                             k, self.pick_lnt_metric_suffix(k))
                }
                for (k, v) in stats.items()
            ]
        }


# Return an array of JobStats objects
def load_stats_dir(path):
    jobstats = []
    fpat = r"^stats-(?P<start>\d+)-swift-(?P<kind>\w+)-(?P<pid>\d+).json$"
    for root, dirs, files in os.walk(path):
        for f in files:
            m = re.match(fpat, f)
            if m:
                # NB: "pid" in fpat is a random number, not unix pid.
                mg = m.groupdict()
                jobkind = mg['kind']
                jobid = int(mg['pid'])
                start_usec = int(mg['start'])

                j = json.load(open(os.path.join(root, f)))
                dur_usec = 1
                jobargs = None
                module = "module"
                patstr = (r"time\.swift-" + jobkind +
                          r"\.(?P<module>[^\.]+)(?P<filename>.*)\.wall$")
                pat = re.compile(patstr)
                stats = dict()
                for (k, v) in j.items():
                    if k.startswith("time."):
                        v = int(1000000.0 * float(v))
                    stats[k] = v
                    tm = re.match(pat, k)
                    if tm:
                        tmg = tm.groupdict()
                        dur_usec = v
                        module = tmg['module']
                        if 'filename' in tmg:
                            ff = tmg['filename']
                            if ff.startswith('.'):
                                ff = ff[1:]
                            jobargs = [ff]

                e = JobStats(jobkind=jobkind, jobid=jobid,
                             module=module, start_usec=start_usec,
                             dur_usec=dur_usec, jobargs=jobargs,
                             stats=stats)
                jobstats.append(e)
    return jobstats


# Passed args with 2-element remainder ["old", "new"], return a list of tuples
# of the form [(name, (oldstats, newstats))] where each name is a common subdir
# of each of "old" and "new", and the stats are those found in the respective
# dirs.
def load_paired_stats_dirs(args):
    assert(len(args.remainder) == 2)
    paired_stats = []
    (old, new) = args.remainder
    for p in sorted(os.listdir(old)):
        full_old = os.path.join(old, p)
        full_new = os.path.join(new, p)
        if not (os.path.exists(full_old) and os.path.isdir(full_old) and
                os.path.exists(full_new) and os.path.isdir(full_new)):
            continue
        old_stats = load_stats_dir(full_old)
        new_stats = load_stats_dir(full_new)
        if len(old_stats) == 0 or len(new_stats) == 0:
            continue
        paired_stats.append((p, (old_stats, new_stats)))
    return paired_stats


def write_catapult_trace(args):
    allstats = []
    for path in args.remainder:
        allstats += load_stats_dir(path)
    json.dump([s.to_catapult_trace_obj() for s in allstats], args.output)


def write_lnt_values(args):
    for d in args.remainder:
        stats = load_stats_dir(d)
        merged = merge_all_jobstats(stats)
        j = merged.to_lnt_test_obj(args)
        if args.lnt_submit is None:
            json.dump(j, args.output, indent=4)
        else:
            url = args.lnt_submit
            print "\nSubmitting to LNT server: " + url
            json_report = {'input_data': json.dumps(j), 'commit': '1'}
            data = urllib.urlencode(json_report)
            response_str = urllib2.urlopen(urllib2.Request(url, data))
            response = json.loads(response_str.read())
            print "### response:"
            print response
            if 'success' in response:
                print "Server response:\tSuccess"
            else:
                print "Server response:\tError"
                print "Error:\t", response['error']
                sys.exit(1)


def merge_all_jobstats(jobstats):
    m = None
    for j in jobstats:
        if m is None:
            m = j
        else:
            m = m.merged_with(j)
    return m


def show_paired_incrementality(args):
    fieldnames = ["old_pct", "old_skip",
                  "new_pct", "new_skip",
                  "delta_pct", "delta_skip",
                  "name"]
    out = csv.DictWriter(args.output, fieldnames, dialect='excel-tab')
    out.writeheader()

    for (name, (oldstats, newstats)) in load_paired_stats_dirs(args):
        olddriver = merge_all_jobstats([x for x in oldstats
                                        if x.is_driver_job()])
        newdriver = merge_all_jobstats([x for x in newstats
                                        if x.is_driver_job()])
        if olddriver is None or newdriver is None:
            continue
        oldpct = olddriver.incrementality_percentage()
        newpct = newdriver.incrementality_percentage()
        deltapct = newpct - oldpct
        oldskip = olddriver.driver_jobs_skipped()
        newskip = newdriver.driver_jobs_skipped()
        deltaskip = newskip - oldskip
        out.writerow(dict(name=name,
                          old_pct=oldpct, old_skip=oldskip,
                          new_pct=newpct, new_skip=newskip,
                          delta_pct=deltapct, delta_skip=deltaskip))


def show_incrementality(args):
    fieldnames = ["incrementality", "name"]
    out = csv.DictWriter(args.output, fieldnames, dialect='excel-tab')
    out.writeheader()

    for path in args.remainder:
        stats = load_stats_dir(path)
        for s in stats:
            if s.is_driver_job():
                pct = s.incrementality_percentage()
                out.writerow(dict(name=os.path.basename(path),
                                  incrementality=pct))


def compare_frontend_stats(args):
    assert(len(args.remainder) == 2)
    (olddir, newdir) = args.remainder

    regressions = 0
    fieldnames = ["old", "new", "delta_pct", "name"]
    out = csv.DictWriter(args.output, fieldnames, dialect='excel-tab')
    out.writeheader()

    old_stats = load_stats_dir(olddir)
    new_stats = load_stats_dir(newdir)
    old_merged = merge_all_jobstats([x for x in old_stats
                                     if x.is_frontend_job()])
    new_merged = merge_all_jobstats([x for x in new_stats
                                     if x.is_frontend_job()])
    if old_merged is None or new_merged is None:
        return regressions
    for stat_name in sorted(old_merged.stats.keys()):
        if stat_name in new_merged.stats:
            old = old_merged.stats[stat_name]
            new = new_merged.stats.get(stat_name, 0)
            if old == 0 or new == 0:
                continue
            delta = (new - old)
            delta_pct = round((float(delta) / float(old)) * 100.0, 2)
            if (stat_name.startswith("time.") and
               abs(delta) < args.delta_usec_thresh):
                continue
            if abs(delta_pct) < args.delta_pct_thresh:
                continue
            out.writerow(dict(name=stat_name, old=old, new=new,
                              delta_pct=delta_pct))
            if delta > 0:
                regressions += 1
    return regressions


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--verbose", action="store_true",
                        help="Report activity verbosely")
    parser.add_argument("--output", default="-",
                        type=argparse.FileType('wb', 0),
                        help="Write output to file")
    parser.add_argument("--paired", action="store_true",
                        help="Process two dirs-of-stats-dirs, pairwise")
    parser.add_argument("--delta-pct-thresh", type=float, default=0.01,
                        help="Percentage change required to report")
    parser.add_argument("--delta-usec-thresh", type=int, default=100000,
                        help="Absolute delta on times required to report")
    parser.add_argument("--lnt-machine", type=str, default=platform.node(),
                        help="Machine name for LNT submission")
    parser.add_argument("--lnt-run-info", action='append', default=[],
                        type=lambda kv: kv.split("="),
                        help="Extra key=value pairs for LNT run-info")
    parser.add_argument("--lnt-machine-info", action='append', default=[],
                        type=lambda kv: kv.split("="),
                        help="Extra key=value pairs for LNT machine-info")
    parser.add_argument("--lnt-order", type=str,
                        default=str(int(time.time())),
                        help="Order for LNT submission")
    parser.add_argument("--lnt-tag", type=str, default="swift-compile",
                        help="Tag for LNT submission")
    parser.add_argument("--lnt-submit", type=str, default=None,
                        help="URL to submit LNT data to (rather than print)")
    modes = parser.add_mutually_exclusive_group(required=True)
    modes.add_argument("--catapult", action="store_true",
                       help="emit a 'catapult'-compatible trace of events")
    modes.add_argument("--incrementality", action="store_true",
                       help="summarize the 'incrementality' of a build")
    modes.add_argument("--compare-frontend-stats", action="store_true",
                       help="Compare frontend stats from two stats-dirs")
    modes.add_argument("--lnt", action="store_true",
                       help="Emit an LNT-compatible test summary")
    parser.add_argument('remainder', nargs=argparse.REMAINDER,
                        help="stats-dirs to process")

    args = parser.parse_args()
    if len(args.remainder) == 0:
        parser.print_help()
        return 1
    if args.catapult:
        write_catapult_trace(args)
    elif args.compare_frontend_stats:
        return compare_frontend_stats(args)
    elif args.incrementality:
        if args.paired:
            show_paired_incrementality(args)
        else:
            show_incrementality(args)
    elif args.lnt:
        write_lnt_values(args)
    return None


sys.exit(main())