coverage-build-db

#!/usr/bin/env python3
# utils/coverage/coverage-build-db - Build sqlite3 database from profdata
#
# This source file is part of the Swift.org open source project
#
# Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
# Licensed under Apache License v2.0 with Runtime Library Exception
#
# See https://swift.org/LICENSE.txt for license information
# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors

import Queue
import argparse
import logging
import multiprocessing
import os
import re
import sqlite3
import sys

logging_format = '%(asctime)s %(levelname)s %(message)s'
logging.basicConfig(level=logging.DEBUG,
                    format=logging_format,
                    filename='/tmp/%s.log' % os.path.basename(__file__),
                    filemode='w')
console = logging.StreamHandler()
console.setLevel(logging.INFO)
formatter = logging.Formatter(logging_format)
console.setFormatter(formatter)
logging.getLogger().addHandler(console)


def parse_coverage_log(filepath):
    """Return parsed coverage intervals from `llvm-profdata show` output at
    `filepath`"""
    logging.info('Parsing: %s', filepath)
    test = os.path.basename(os.path.dirname(filepath)).rsplit('.', 1)[0]
    with open(filepath) as f:
        # Initial parse
        parsed = []
        for section in [section.split("\n")
                        for section in f.read().split("\n\n")]:
            if ".cpp:" in section[0] or ".h:" in section[0]:
                parsed_section = [section[0].split(":", 1) + [test]]
                for line in section[1:]:
                    linedata = linedata_re.match(line).group(1, 2)
                    parsed_section.append(linedata)
                parsed.append(parsed_section)
        # Build intervals
        parsed_interval = []
        for section in parsed:
            new_section = [section[0]]
            i = 1
            if len(section) > 2:
                while i < len(section) - 1:
                    while i < len(section) - 1 and \
                            section[i][0] in ['0', None]:
                        i += 1
                    if i == len(section) - 1:
                        if section[i][0] not in ['0', None]:
                            istart = int(section[i][1])
                            iend = istart
                            new_section.append((istart, iend))
                        break
                    istart = int(section[i][1])
                    iend = istart  # single line interval starting
                    while i < len(section) - 1 and \
                            int(section[i + 1][1]) == iend + 1 and \
                            section[i + 1][0] not in ['0', None]:
                        iend += 1
                        i += 1
                    new_section.append((istart, iend))
                    i += 1
            else:
                if section[i][0] not in ['0', None]:
                    istart = int(section[i][1])
                    iend = istart
                    new_section.append((istart, iend))
            parsed_interval.append(new_section)
        return parsed_interval


def worker(args):
    """Parse coverage log at path `args[0]` and place in queue `args[1]`"""
    args[1].put(parse_coverage_log(args[0]))


manager = multiprocessing.Manager()
result_queue = manager.Queue()
worker_queue = []

linedata_re = re.compile(r"^\s*([^\s]+?)?\|\s*(\d+)\|")
strings = dict()

pool = multiprocessing.Pool(3)


def insert_coverage_section_into_db(db_cursor, section):
    """Insert parsed intervals in `section` into database at `db_cursor`"""
    filename = section[0][0]
    function = section[0][1]
    test = section[0][2]
    logging.debug('Inserting section into database for file: %s', filename)
    if filename not in strings:
        db_cursor.execute("INSERT INTO strings (string) VALUES (?)",
                          (filename,))
        strings[filename] = db_cursor.lastrowid
    if test not in strings:
        db_cursor.execute("INSERT INTO strings (string) VALUES (?)", (test,))
        strings[test] = db_cursor.lastrowid
    if function not in strings:
        db_cursor.execute("INSERT INTO strings (string) VALUES (?)",
                          (function,))
        strings[function] = db_cursor.lastrowid
    for istart, iend in section[1:]:
        logging.debug('%s, %s, %s, %s, %s', function, filename, test, istart,
                      iend)
        db_cursor.execute("INSERT INTO coverage VALUES (?,?,?,?,?)",
                          (strings[function], strings[filename],
                           strings[test], istart, iend))


def main():
    parser = argparse.ArgumentParser(
        description='Build sqlite3 database from profdata')
    parser.add_argument('root_directory',
                        metavar='root-directory',
                        help='a root directory to recursively search for '
                             'coverage logs')
    parser.add_argument('--coverage-filename',
                        help='a coverage log file name to search for '
                             '(default: coverage.log.demangled)',
                        metavar='NAME',
                        default='coverage.log.demangled')
    parser.add_argument('--db-filepath',
                        help='the filepath of the coverage db to build '
                             '(default: coverage.db)',
                        metavar='PATH',
                        default='coverage.db')
    parser.add_argument('--log',
                        help='the level of information to log (default: info)',
                        metavar='LEVEL',
                        default='info',
                        choices=['info', 'debug', 'warning', 'error',
                                 'critical'])
    args = parser.parse_args()

    console.setLevel(level=args.log.upper())
    logging.debug(args)

    if not os.path.exists(args.root_directory):
        logging.critical('Directory not found: %s', args.root_directory)
        return 1

    if os.path.exists(args.db_filepath):
        logging.info('Deleting existing database: %s', args.db_filepath)
        os.unlink(args.db_filepath)

    logging.info('Creating database: %s', args.db_filepath)
    conn = sqlite3.connect(args.db_filepath)

    try:
        logging.debug('Creating coverage table')
        c = conn.cursor()
        c.execute('''CREATE TABLE strings
                     (id integer primary key autoincrement, string text)''')
        c.execute('''CREATE TABLE coverage
                     (function references strings(id),
                      src references strings(id),
                      test references strings(id),
                      istart integer, iend integer)''')
        conn.commit()

        for root, folders, files in os.walk(args.root_directory):
            for f in files:
                if f == args.coverage_filename:
                    filepath = os.path.join(root, f)
                    logging.debug('Adding file to queue: %s', filepath)
                    worker_queue.append((filepath, result_queue))

        logging.info('Finished adding %s paths to queue', len(worker_queue))
        pool.map_async(worker, worker_queue)

        while True:
            parsed = result_queue.get(timeout=60)
            logging.info('Inserting coverage data into db for %s function(s)',
                         len(parsed))
            for section in parsed:
                insert_coverage_section_into_db(c, section)
            conn.commit()
            result_queue.task_done()
    except Queue.Empty:
        logging.info('Queue exhausted. Shutting down...')
    except KeyboardInterrupt:
        logging.debug('Caught KeyboardInterrupt. Shutting down...')
    finally:
        logging.debug('Closing database')
        conn.commit()
        conn.close()
        logging.debug('Terminating pool')
        pool.terminate()

    return 0


if __name__ == '__main__':
    sys.exit(main())