diff --git a/src/runner.ts b/src/runner.ts
index b8a3dc004fb..1d6e7933438 100644
--- a/src/runner.ts
+++ b/src/runner.ts
@@ -96,9 +96,10 @@ export interface Options {
     rulesDirectory?: string | string[];
 
     /**
-     * That TSLint produces the correct output for the specified directory.
+     * Run the tests in the given directories to ensure a (custom) TSLint rule's output matches the expected output.
+     * When this property is `true` the `files` property is used to specify the directories from which the tests should be executed.
      */
-    test?: string;
+    test?: boolean;
 
     /**
      * Whether to enable type checking when linting a project.
@@ -122,7 +123,7 @@ export async function run(options: Options, logger: Logger): Promise<Status> {
         return await runWorker(options, logger);
     } catch (error) {
         if (error instanceof FatalError) {
-            logger.error(error.message);
+            logger.error(`${error.message}\n`);
             return Status.FatalError;
         }
         throw error;
@@ -142,7 +143,7 @@ async function runWorker(options: Options, logger: Logger): Promise<Status> {
     if (options.test) {
         const test = await import("./test");
         const results = test.runTests((options.files || []).map(trimSingleQuotes), options.rulesDirectory);
-        return test.consoleTestResultsHandler(results) ? Status.Ok : Status.FatalError;
+        return test.consoleTestResultsHandler(results, logger) ? Status.Ok : Status.FatalError;
     }
 
     if (options.config && !fs.existsSync(options.config)) {
@@ -151,20 +152,20 @@ async function runWorker(options: Options, logger: Logger): Promise<Status> {
 
     const { output, errorCount } = await runLinter(options, logger);
     if (output && output.trim()) {
-        logger.log(output);
+        logger.log(`${output}\n`);
     }
     return options.force || errorCount === 0 ? Status.Ok : Status.LintError;
 }
 
 async function runLinter(options: Options, logger: Logger): Promise<LintResult> {
-    const { files, program } = resolveFilesAndProgram(options);
+    const { files, program } = resolveFilesAndProgram(options, logger);
     // if type checking, run the type checker
     if (program && options.typeCheck) {
         const diagnostics = ts.getPreEmitDiagnostics(program);
         if (diagnostics.length !== 0) {
             const message = diagnostics.map((d) => showDiagnostic(d, program, options.outputAbsolutePaths)).join("\n");
             if (options.force) {
-                logger.error(message);
+                logger.error(`${message}\n`);
             } else {
                 throw new FatalError(message);
             }
@@ -173,12 +174,15 @@ async function runLinter(options: Options, logger: Logger): Promise<LintResult>
     return doLinting(options, files, program, logger);
 }
 
-function resolveFilesAndProgram({ files, project, exclude, outputAbsolutePaths }: Options): { files: string[]; program?: ts.Program } {
+function resolveFilesAndProgram(
+    { files, project, exclude, outputAbsolutePaths }: Options,
+    logger: Logger,
+): { files: string[]; program?: ts.Program } {
     // remove single quotes which break matching on Windows when glob is passed in single quotes
     exclude = exclude.map(trimSingleQuotes);
 
     if (project === undefined) {
-        return { files: resolveGlobs(files, exclude, outputAbsolutePaths) };
+        return { files: resolveGlobs(files, exclude, outputAbsolutePaths, logger) };
     }
 
     const projectPath = findTsconfig(project);
@@ -202,7 +206,7 @@ function resolveFilesAndProgram({ files, project, exclude, outputAbsolutePaths }
                 if (fs.existsSync(file)) {
                     throw new FatalError(`'${file}' is not included in project.`);
                 }
-                console.warn(`'${file}' does not exist. This will be an error in TSLint 6.`); // TODO make this an error in v6.0.0
+                logger.error(`'${file}' does not exist. This will be an error in TSLint 6.\n`); // TODO make this an error in v6.0.0
             }
         }
     }
@@ -217,15 +221,15 @@ function filterFiles(files: string[], patterns: string[], include: boolean): str
     return files.filter((file) => include === matcher.some((pattern) => pattern.match(file)));
 }
 
-function resolveGlobs(files: string[], ignore: string[], outputAbsolutePaths?: boolean): string[] {
+function resolveGlobs(files: string[], ignore: string[], outputAbsolutePaths: boolean | undefined, logger: Logger): string[] {
     const results = flatMap(
         files,
         (file) => glob.sync(trimSingleQuotes(file), { ignore, nodir: true }),
     );
     // warn if `files` contains non-existent files, that are not patters and not excluded by any of the exclude patterns
     for (const file of filterFiles(files, ignore, false)) {
-        if (!glob.hasMagic(file)) {
-            console.warn(`'${file}' does not exist. This will be an error in TSLint 6.`); // TODO make this an error in v6.0.0
+        if (!glob.hasMagic(file) && !results.some(createMinimatchFilter(file))) {
+            logger.error(`'${file}' does not exist. This will be an error in TSLint 6.\n`); // TODO make this an error in v6.0.0
         }
     }
     const cwd = process.cwd();
@@ -248,17 +252,17 @@ async function doLinting(
     let configFile: IConfigurationFile | undefined;
 
     for (const file of files) {
+        const folder = path.dirname(file);
+        if (lastFolder !== folder) {
+            configFile = findConfiguration(possibleConfigAbsolutePath, folder).results;
+            lastFolder = folder;
+        }
         if (isFileExcluded(file)) {
             continue;
         }
 
         const contents = program !== undefined ? program.getSourceFile(file).text : await tryReadFile(file, logger);
         if (contents !== undefined) {
-            const folder = path.dirname(file);
-            if (lastFolder !== folder) {
-                configFile = findConfiguration(possibleConfigAbsolutePath, folder).results;
-                lastFolder = folder;
-            }
             linter.lint(file, contents, configFile);
         }
     }
@@ -287,7 +291,7 @@ async function tryReadFile(filename: string, logger: Logger): Promise<string | u
             // MPEG transport streams use the '.ts' file extension. They use 0x47 as the frame
             // separator, repeating every 188 bytes. It is unlikely to find that pattern in
             // TypeScript source, so tslint ignores files with the specific pattern.
-            logger.error(`${filename}: ignoring MPEG transport stream`);
+            logger.error(`${filename}: ignoring MPEG transport stream\n`);
             return undefined;
         }
     } finally {
diff --git a/src/test.ts b/src/test.ts
index ded04e8845c..f6c3a6d79af 100644
--- a/src/test.ts
+++ b/src/test.ts
@@ -25,6 +25,7 @@ import * as ts from "typescript";
 
 import { Replacement } from "./language/rule/rule";
 import * as Linter from "./linter";
+import { Logger } from "./runner";
 import { denormalizeWinPath, mapDefined, readBufferWithDetectedEncoding } from "./utils";
 import { LintError } from "./verify/lintError";
 import * as parse from "./verify/parse";
@@ -195,11 +196,11 @@ export function runTest(testDirectory: string, rulesDirectory?: string | string[
     return results;
 }
 
-export function consoleTestResultsHandler(testResults: TestResult[]): boolean {
+export function consoleTestResultsHandler(testResults: TestResult[], logger: Logger): boolean {
     let didAllTestsPass = true;
 
     for (const testResult of testResults) {
-        if (!consoleTestResultHandler(testResult)) {
+        if (!consoleTestResultHandler(testResult, logger)) {
             didAllTestsPass = false;
         }
     }
@@ -207,7 +208,7 @@ export function consoleTestResultsHandler(testResults: TestResult[]): boolean {
     return didAllTestsPass;
 }
 
-export function consoleTestResultHandler(testResult: TestResult): boolean {
+export function consoleTestResultHandler(testResult: TestResult, logger: Logger): boolean {
     // needed to get colors to show up when passing through Grunt
     (chalk as any).enabled = true;
 
@@ -215,11 +216,10 @@ export function consoleTestResultHandler(testResult: TestResult): boolean {
 
     for (const fileName of Object.keys(testResult.results)) {
         const results = testResult.results[fileName];
-        process.stdout.write(`${fileName}:`);
+        logger.log(`${fileName}:`);
 
-        /* tslint:disable:no-console */
         if (results.skipped) {
-            console.log(chalk.yellow(` Skipped, requires typescript ${results.requirement}`));
+            logger.log(chalk.yellow(` Skipped, requires typescript ${results.requirement}\n`));
         } else {
             const markupDiffResults = diff.diffLines(results.markupFromMarkup, results.markupFromLinter);
             const fixesDiffResults = diff.diffLines(results.fixesFromLinter, results.fixesFromMarkup);
@@ -227,28 +227,26 @@ export function consoleTestResultHandler(testResult: TestResult): boolean {
             const didFixesTestPass = !fixesDiffResults.some((hunk) => hunk.added === true || hunk.removed === true);
 
             if (didMarkupTestPass && didFixesTestPass) {
-                console.log(chalk.green(" Passed"));
+                logger.log(chalk.green(" Passed\n"));
             } else {
-                console.log(chalk.red(" Failed!"));
+                logger.log(chalk.red(" Failed!\n"));
                 didAllTestsPass = false;
                 if (!didMarkupTestPass) {
-                    displayDiffResults(markupDiffResults, MARKUP_FILE_EXTENSION);
+                    displayDiffResults(markupDiffResults, MARKUP_FILE_EXTENSION, logger);
                 }
                 if (!didFixesTestPass) {
-                    displayDiffResults(fixesDiffResults, FIXES_FILE_EXTENSION);
+                    displayDiffResults(fixesDiffResults, FIXES_FILE_EXTENSION, logger);
                 }
             }
         }
-        /* tslint:enable:no-console */
     }
 
     return didAllTestsPass;
 }
 
-function displayDiffResults(diffResults: diff.IDiffResult[], extension: string) {
-    /* tslint:disable:no-console */
-    console.log(chalk.green(`Expected (from ${extension} file)`));
-    console.log(chalk.red("Actual (from TSLint)"));
+function displayDiffResults(diffResults: diff.IDiffResult[], extension: string, logger: Logger) {
+    logger.log(chalk.green(`Expected (from ${extension} file)\n`));
+    logger.log(chalk.red("Actual (from TSLint)\n"));
 
     for (const diffResult of diffResults) {
         let color = chalk.grey;
@@ -257,7 +255,6 @@ function displayDiffResults(diffResults: diff.IDiffResult[], extension: string)
         } else if (diffResult.removed) {
             color = chalk.red.underline;
         }
-        process.stdout.write(color(diffResult.value));
+        logger.log(color(diffResult.value));
     }
-    /* tslint:enable:no-console */
 }
diff --git a/src/tslint-cli.ts b/src/tslint-cli.ts
index 1ed3997c458..62fdc74dd50 100644
--- a/src/tslint-cli.ts
+++ b/src/tslint-cli.ts
@@ -38,7 +38,7 @@ interface Argv {
     formattersDir: string;
     format?: string;
     typeCheck?: boolean;
-    test?: string;
+    test?: boolean;
     version?: boolean;
 }
 
@@ -244,16 +244,9 @@ if (argv.typeCheck) {
     }
 }
 
-let log: (message: string) => void;
-if (argv.out != undefined) {
-    const outputStream = fs.createWriteStream(argv.out, {
-        flags: "w+",
-        mode: 420,
-    });
-    log = (message) => outputStream.write(`${message}\n`);
-} else {
-    log = console.log;
-}
+const outputStream: NodeJS.WritableStream = argv.out === undefined
+    ? process.stdout
+    : fs.createWriteStream(argv.out, {flags: "w+", mode: 420});
 
 run(
     {
@@ -273,8 +266,12 @@ run(
         typeCheck: argv.typeCheck,
     },
     {
-        log,
-        error: (m) => console.error(m),
+        log(m) {
+            outputStream.write(m);
+        },
+        error(m) {
+            process.stdout.write(m);
+        },
     })
     .then((rc) => {
         process.exitCode = rc;
diff --git a/test/executable/executableTests.ts b/test/executable/executableTests.ts
index a719bc95acc..c2e473a2b3e 100644
--- a/test/executable/executableTests.ts
+++ b/test/executable/executableTests.ts
@@ -19,6 +19,7 @@ import * as cp from "child_process";
 import * as fs from "fs";
 import * as os from "os";
 import * as path from "path";
+import { Logger, Options, run, Status } from "../../src/runner";
 import { denormalizeWinPath } from "../../src/utils";
 import { createTempFile } from "../utils";
 
@@ -27,6 +28,11 @@ const EXECUTABLE_DIR = path.resolve(process.cwd(), "test", "executable");
 const EXECUTABLE_PATH = path.resolve(EXECUTABLE_DIR, "npm-like-executable");
 const TEMP_JSON_PATH = path.resolve(EXECUTABLE_DIR, "tslint.json");
 
+const dummyLogger: Logger = {
+    log() { /* do nothing */ },
+    error() { /* do nothing */ },
+};
+
 describe("Executable", function(this: Mocha.ISuiteCallbackContext) {
     this.slow(3000);    // the executable is JIT-ed each time it runs; avoid showing slowness warnings
     this.timeout(4000);
@@ -71,117 +77,91 @@ describe("Executable", function(this: Mocha.ISuiteCallbackContext) {
             });
         });
 
-        it("warns if file does not exist", (done) => {
-            execCli(["foo/bar.ts"], (err, _stdout, stderr) => {
-                assert.isNull(err, "process should exit without error");
-
-                assert.include(stderr, "'foo/bar.ts' does not exist");
-                done();
-            });
+        it("warns if file does not exist", async () => {
+            const result = await execRunnerWithOutput({files: ["foo/bar.ts"]});
+            assert.strictEqual(result.status, Status.Ok, "process should exit without error");
+            assert.include(result.stderr, "'foo/bar.ts' does not exist");
         });
 
-        it("doesn't warn if non-existent file is excluded by --exclude", (done) => {
-            execCli(["foo/bar.js", "--exclude", "**/*.js"], (err, _stdout, stderr) => {
-                assert.isNull(err, "process should exit without error");
-
-                assert.notInclude(stderr, "does not exist");
-                done();
-            });
+        it("doesn't warn if non-existent file is excluded by --exclude", async () => {
+            const result = await execRunnerWithOutput({files: ["foo/bar.js"], exclude: ["**/*.js"]});
+            assert.strictEqual(result.status, Status.Ok, "process should exit without error");
+            assert.notInclude(result.stderr, "does not exist");
         });
 
-        it("doesn't warn if glob pattern doesn't match any file", (done) => {
-            execCli(["foobar/*.js"], (err, _stdout, stderr) => {
-                assert.isNull(err, "process should exit without error");
-
-                assert.notInclude(stderr, "does not exist");
-                done();
-            });
+        it("doesn't warn if glob pattern doesn't match any file", async () => {
+            const result = await execRunnerWithOutput({files: ["foobar/*.js"]});
+            assert.strictEqual(result.status, Status.Ok, "process should exit without error");
+            assert.notInclude(result.stderr, "does not exist");
         });
     });
 
     describe("Configuration file", () => {
-        it("exits with code 0 if relative path is passed without `./`", (done) => {
-            execCli(["-c", "test/config/tslint-almost-empty.json", "src/test.ts"], (err) => {
-                assert.isNull(err, "process should exit without an error");
-                done();
-            });
+        it("exits with code 0 if relative path is passed without `./`", async () => {
+            const status = await execRunner({config: "test/config/tslint-almost-empty.json", files: ["src/test.ts"]});
+            assert.equal(status, Status.Ok, "process should exit without an error");
         });
 
-        it("exits with code 0 if config file that extends relative config file", (done) => {
-            execCli(["-c", "test/config/tslint-extends-package-no-mod.json", "src/test.ts"], (err) => {
-                assert.isNull(err, "process should exit without an error");
-                done();
-            });
+        it("exits with code 0 if config file that extends relative config file", async () => {
+            const status = await execRunner({config: "test/config/tslint-extends-package-no-mod.json", files: ["src/test.ts"]});
+            assert.equal(status, Status.Ok, "process should exit without an error");
         });
 
-        it("exits with code 1 if json config file is invalid", (done) => {
-            execCli(["-c", "test/config/tslint-invalid.json", "src/test.ts"], (err, stdout, stderr) => {
-                assert.isNotNull(err, "process should exit with error");
-                assert.strictEqual(err.code, 1, "error code should be 1");
+        it("exits with code 1 if config file is invalid", async () => {
+            const result = await execRunnerWithOutput({config: "test/config/tslint-invalid.json", files: ["src/test.ts"]});
 
-                assert.include(stderr, "Failed to load", "stderr should contain notification about failing to load json config");
-                assert.strictEqual(stdout, "", "shouldn't contain any output in stdout");
-                done();
-            });
+            assert.equal(result.status, Status.FatalError, "process should exit with error");
+            assert.include(result.stderr, "Failed to load", "stderr should contain notification about failing to load json config");
+            assert.strictEqual(result.stdout, "", "shouldn't contain any output in stdout");
         });
 
-        it("exits with code 1 if yaml config file is invalid", (done) => {
-            execCli(["-c", "test/config/tslint-invalid.yaml", "src/test.ts"], (err, stdout, stderr) => {
-                assert.isNotNull(err, "process should exit with error");
-                assert.strictEqual(err.code, 1, "error code should be 1");
+        it("exits with code 1 if yaml config file is invalid", async () => {
+            const result = await execRunnerWithOutput({config: "test/config/tslint-invalid.yaml", files: ["src/test.ts"]});
+            assert.strictEqual(result.status, Status.FatalError, "error code should be 1");
 
-                assert.include(stderr, "Failed to load", "stderr should contain notification about failing to load yaml config");
-                assert.strictEqual(stdout, "", "shouldn't contain any output in stdout");
-                done();
-            });
+            assert.include(result.stderr, "Failed to load", "stderr should contain notification about failing to load yaml config");
+            assert.strictEqual(result.stdout, "", "shouldn't contain any output in stdout");
         });
 
-        it("mentions the root cause if a config file extends from an invalid file", (done) => {
-            execCli(["-c", "test/config/tslint-extends-invalid.json", "src/test.ts"], (err, stdout, stderr) => {
-                assert.isNotNull(err, "process should exit with error");
-                assert.strictEqual(err.code, 1, "error code should be 1");
+        it("mentions the root cause if a config file extends from an invalid file", async () => {
+            const result = await execRunnerWithOutput({config: "test/config/tslint-extends-invalid.json", files: ["src/test.ts"]});
 
-                assert.include(stderr, "Failed to load", "stderr should contain notification about failing to load json config");
-                assert.include(stderr, "tslint-invalid.json", "stderr should mention the problem file");
-                assert.strictEqual(stdout, "", "shouldn't contain any output in stdout");
-                done();
-            });
+            assert.equal(result.status, Status.FatalError, "process should exit with error");
+            assert.include(result.stderr, "Failed to load", "stderr should contain notification about failing to load json config");
+            assert.include(result.stderr, "tslint-invalid.json", "stderr should mention the problem file");
+            assert.strictEqual(result.stdout, "", "shouldn't contain any output in stdout");
         });
     });
 
     describe("Custom rules", () => {
-        it("exits with code 1 if nonexisting custom rules directory is passed", (done) => {
-            execCli(["-c", "./test/config/tslint-custom-rules.json", "-r", "./someRandomDir", "src/test.ts"], (err) => {
-                assert.isNotNull(err, "process should exit with error");
-                assert.strictEqual(err.code, 1, "error code should be 1");
-                done();
-            });
+        it("exits with code 1 if nonexisting custom rules directory is passed", async () => {
+            const status = await execRunner(
+                {config: "./test/config/tslint-custom-rules.json", rulesDirectory: "./someRandomDir", files: ["src/test.ts"]},
+            );
+            assert.equal(status, Status.FatalError, "error code should be 1");
         });
 
-        it("exits with code 2 if custom rules directory is passed and file contains lint errors", (done) => {
-            execCli(["-c", "./test/config/tslint-custom-rules.json", "-r", "./test/files/custom-rules", "src/test.ts"], (err) => {
-                assert.isNotNull(err, "process should exit with error");
-                assert.strictEqual(err.code, 2, "error code should be 2");
-                done();
-            });
+        it("exits with code 2 if custom rules directory is passed and file contains lint errors", async () => {
+            const status = await execRunner(
+                {config: "./test/config/tslint-custom-rules.json", rulesDirectory: "./test/files/custom-rules", files: ["src/test.ts"]},
+            );
+            assert.equal(status, Status.LintError, "error code should be 2");
         });
 
-        it("exits with code 0 if custom rules directory is passed and file contains lint warnings", (done) => {
-            execCli(
-                ["-c", "./test/config/tslint-extends-package-warning.json", "-r", "./test/files/custom-rules", "src/test.ts"],
-                (err) => {
-                    assert.isNull(err, "process should exit without an error");
-                    done();
+        it("exits with code 0 if custom rules directory is passed and file contains lint warnings", async () => {
+            const status = await execRunner(
+                {
+                    config: "./test/config/tslint-extends-package-warning.json",
+                    files: ["src/test.ts"],
+                    rulesDirectory: "./test/files/custom-rules",
                 },
             );
+            assert.equal(status, Status.Ok, "process should exit without an error");
         });
 
-        it("exits with code 2 if custom rules directory is specified in config file and file contains lint errors", (done) => {
-            execCli(["-c", "./test/config/tslint-custom-rules-with-dir.json", "src/test.ts"], (err) => {
-                assert.isNotNull(err, "process should exit with error");
-                assert.strictEqual(err.code, 2, "error code should be 2");
-                done();
-            });
+        it("exits with code 2 if custom rules directory is specified in config file and file contains lint errors", async () => {
+            const status = await execRunner({config: "./test/config/tslint-custom-rules-with-dir.json", files: ["src/test.ts"]});
+            assert.equal(status, Status.LintError, "error code should be 2");
         });
 
         it("are compiled just in time when using ts-node", (done) => {
@@ -204,55 +184,45 @@ describe("Executable", function(this: Mocha.ISuiteCallbackContext) {
     });
 
     describe("Config with excluded files", () => {
-        it("exits with code 2 if linter options doesn't exclude file with lint errors", (done) => {
-            execCli(["-c", "./test/files/config-exclude/tslint-exclude-one.json", "./test/files/config-exclude/included.ts"], (err) => {
-                assert.isNotNull(err, "process should exit with error");
-                assert.strictEqual(err.code, 2, "error code should be 2");
-                done();
-            });
+        it("exits with code 2 if linter options doesn't exclude file with lint errors", async () => {
+            const status = await execRunner(
+                {config: "./test/files/config-exclude/tslint-exclude-one.json", files: ["./test/files/config-exclude/included.ts"]},
+            );
+            assert.equal(status, Status.LintError, "error code should be 2");
         });
 
-        it("exits with code 0 if linter options exclude one file with lint errors", (done) => {
-            execCli(["-c", "./test/files/config-exclude/tslint-exclude-one.json", "./test/rules/config-exclude/excluded.ts"], (err) => {
-                assert.isNull(err, "process should exit without an error");
-                done();
-            });
+        it("exits with code 0 if linter options exclude one file with lint errors", async () => {
+            const status = await execRunner(
+                {config: "./test/files/config-exclude/tslint-exclude-one.json", files: ["./test/files/config-exclude/excluded.ts"]},
+            );
+            assert.equal(status, Status.Ok, "process should exit without an error");
         });
 
-        it("exits with code 0 if linter options excludes many files with lint errors", (done) => {
-            execCli(
-                [
-                    "-c",
-                    "./test/files/config-exclude/tslint-exclude-many.json",
-                    "./test/rules/config-exclude/excluded1.ts",
-                    "./test/rules/config-exclude/subdir/excluded2.ts"],
-                (err) => {
-                    assert.isNull(err, "process should exit without an error");
-                    done();
+        it("exits with code 0 if linter options excludes many files with lint errors", async () => {
+            const status = await execRunner(
+                {
+                    config: "./test/files/config-exclude/tslint-exclude-many.json",
+                    files: ["./test/rules/config-exclude/excluded1.ts", "./test/rules/config-exclude/subdir/excluded2.ts"],
                 },
             );
+            assert.strictEqual(status, Status.Ok, "process should exit without an error");
         });
 
-        it("excludes files relative to tslint.json", (done) => {
-            execCli(
-                ["-c", "./test/files/config-exclude/tslint-exclude-one.json", "./test/files/config-exclude/subdir/excluded.ts"],
-                (err) => {
-                    assert.isNotNull(err, "process should exit an error");
-                    assert.equal(err.code, 2, "exit code should be 2");
-                    done();
-                },
+        it("excludes files relative to tslint.json", async () => {
+            const status = await execRunner(
+                {config: "./test/files/config-exclude/tslint-exclude-one.json", files: ["./test/files/config-exclude/subdir/excluded.ts"]},
             );
+            assert.equal(status, Status.LintError, "exit code should be 2");
         });
 
-        it("excludes files relative to tslint.json they were declared in", (done) => {
-            execCli(
-                ["-c", "./test/files/config-exclude/subdir/tslint-extending.json", "./test/files/config-exclude/subdir/excluded.ts"],
-                (err) => {
-                    assert.isNotNull(err, "process should exit an error");
-                    assert.equal(err.code, 2, "exit code should be 2");
-                    done();
+        it("excludes files relative to tslint.json they were declared in", async () => {
+            const status = await execRunner(
+                {
+                    config: "./test/files/config-exclude/subdir/tslint-extending.json",
+                    files: ["./test/files/config-exclude/subdir/excluded.ts"],
                 },
             );
+            assert.equal(status, Status.LintError, "exit code should be 2");
         });
     });
 
@@ -271,314 +241,225 @@ describe("Executable", function(this: Mocha.ISuiteCallbackContext) {
     });
 
     describe("--fix flag", () => {
-        it("fixes multiple rules without overwriting each other", (done) => {
+        it("fixes multiple rules without overwriting each other", async () => {
             const tempFile = path.relative(process.cwd(), createTempFile("ts"));
-            fs.createReadStream("test/files/multiple-fixes-test/multiple-fixes.test.ts")
-                .pipe(fs.createWriteStream(tempFile))
-                .on("finish", () => {
-                    execCli(
-                        ["-c", "test/files/multiple-fixes-test/tslint.json", tempFile, "--fix"],
-                        (err, stdout) => {
-                            const content = fs.readFileSync(tempFile, "utf8");
-                            // compare against file name which will be returned by formatter (used in TypeScript)
-                            const denormalizedFileName = denormalizeWinPath(tempFile);
-                            fs.unlinkSync(tempFile);
-                            assert.strictEqual(content, "import * as y from \"a_long_module\";\nimport * as x from \"b\";\n");
-                            assert.isNull(err, "process should exit without an error");
-                            assert.strictEqual(stdout, `Fixed 2 error(s) in ${denormalizedFileName}`);
-                            done();
-                        });
-                });
+            fs.writeFileSync(tempFile, 'import * as x from "b"\nimport * as y from "a_long_module";\n');
+            const result = await execRunnerWithOutput(
+                {config: "test/files/multiple-fixes-test/tslint.json", files: [tempFile], fix: true},
+            );
+            const content = fs.readFileSync(tempFile, "utf8");
+            // compare against file name which will be returned by formatter (used in TypeScript)
+            const denormalizedFileName = denormalizeWinPath(tempFile);
+            fs.unlinkSync(tempFile);
+            assert.equal(result.status, Status.Ok, "process should exit without an error");
+            assert.strictEqual(content, "import * as y from \"a_long_module\";\nimport * as x from \"b\";\n");
+            assert.strictEqual(result.stdout.trim(), `Fixed 2 error(s) in ${denormalizedFileName}`);
         }).timeout(8000);
     });
 
     describe("--force flag", () => {
-        it("exits with code 0 if `--force` flag is passed", (done) => {
-            execCli(
-                ["-c", "./test/config/tslint-custom-rules.json", "-r", "./test/files/custom-rules", "--force", "src/test.ts"],
-                (err, stdout) => {
-                    assert.isNull(err, "process should exit without an error");
-                    assert.include(stdout, "failure", "errors should be reported");
-                    done();
-                });
+        it("exits with code 0 if `--force` flag is passed", async () => {
+            const result = await execRunnerWithOutput(
+                {
+                    config: "./test/config/tslint-custom-rules.json",
+                    files: ["src/test.ts"],
+                    force: true,
+                    rulesDirectory: "./test/files/custom-rules",
+                },
+            );
+            assert.equal(result.status, Status.Ok, "process should exit without an error");
+            assert.include(result.stdout, "failure", "errors should be reported");
         });
     });
 
     describe("--test flag", () => {
-        it("exits with code 0 if `--test` flag is used", (done) => {
-            execCli(["--test", "test/rules/no-eval"], (err) => {
-                assert.isNull(err, "process should exit without an error");
-                done();
-            });
+        it("exits with code 0 if `--test` flag is used", async () => {
+            const status = await execRunner({test: true, files: ["test/rules/no-eval"]});
+            assert.equal(status, Status.Ok, "process should exit without an error");
         });
 
-        it("exits with code 0 if `--test` flag is used with a wildcard", (done) => {
-            execCli(["--test", "test/rules/no-e*"], (err) => {
-                assert.isNull(err, "process should exit without an error");
-                done();
-            });
+        it("exits with code 0 if `--test` flag is used with a wildcard", async () => {
+            const status = await execRunner({test: true, files: ["test/rules/no-e*"]});
+            assert.equal(status, Status.Ok, "process should exit without an error");
         });
 
-        it("exits with code 1 if `--test` flag is used with incorrect rule", (done) => {
-            execCli(["--test", "test/files/incorrect-rule-test"], (err) => {
-                assert.isNotNull(err, "process should exit with error");
-                assert.strictEqual(err.code, 1, "error code should be 1");
-                done();
-            });
+        it("exits with code 1 if `--test` flag is used with incorrect rule", async () => {
+            const status = await execRunner({test: true, files: ["test/files/incorrect-rule-test"]});
+            assert.equal(status, Status.FatalError, "error code should be 1");
         });
 
-        it("exits with code 1 if `--test` flag is used with incorrect rule in a wildcard", (done) => {
-            execCli(["--test", "test/files/incorrect-rule-*"], (err) => {
-                assert.isNotNull(err, "process should exit with error");
-                assert.strictEqual(err.code, 1, "error code should be 1");
-                done();
-            });
+        it("exits with code 1 if `--test` flag is used with incorrect rule in a wildcard", async () => {
+            const status = await execRunner({test: true, files: ["test/files/incorrect-rule-*"]});
+            assert.equal(status, Status.FatalError, "error code should be 1");
         });
 
-        it("exits with code 0 if `--test` flag is used with custom rule", (done) => {
-            execCli(["--test", "test/files/custom-rule-rule-test"], (err) => {
-                assert.isNull(err, "process should exit without an error");
-                done();
-            });
+        it("exits with code 0 if `--test` flag is used with custom rule", async () => {
+            const status = await execRunner({test: true, files: ["test/files/custom-rule-rule-test"]});
+            assert.equal(status, Status.Ok, "process should exit without an error");
         });
 
-        it("exits with code 0 if `--test` and `-r` flags are used with custom rule", (done) => {
-            execCli(["-r", "test/files/custom-rules-2", "--test", "test/files/custom-rule-rule-test"], (err) => {
-                assert.isNull(err, "process should exit without an error");
-                done();
-            });
+        it("exits with code 0 if `--test` and `-r` flags are used with custom rule", async () => {
+            const status = await execRunner(
+                {test: true, files: ["test/files/custom-rule-cli-rule-test"], rulesDirectory: "test/files/custom-rules-2"},
+            );
+            assert.equal(status, Status.Ok, "process should exit without an error");
         });
 
-        it("exits with code 0 if `--test` flag is used with fixes", (done) => {
-            execCli(["--test", "test/files/fixes-test"], (err) => {
-                assert.isNull(err, "process should exit without an error");
-                done();
-            });
+        it("exits with code 0 if `--test` flag is used with fixes", async () => {
+            const status = await execRunner({test: true, files: ["test/files/fixes-test"]});
+            assert.equal(status, Status.Ok, "process should exit without an error");
         });
 
-        it("exits with code 1 if `--test` flag is used with incorrect fixes", (done) => {
-            execCli(["--test", "test/files/incorrect-fixes-test"], (err) => {
-                assert.isNotNull(err, "process should exit with error");
-                assert.strictEqual(err.code, 1, "error code should be 1");
-                done();
-            });
+        it("exits with code 1 if `--test` flag is used with incorrect fixes", async () => {
+            const status = await execRunner({test: true, files: ["test/files/incorrect-fixes-test"]});
+            assert.equal(status, Status.FatalError, "error code should be 1");
         });
 
-        it("can be used with multiple paths", (done) => {
+        it("can be used with multiple paths", async () => {
             // pass a failing test as second path to make sure it gets executed
-            execCli(["--test", "test/files/custom-rule-rule-test", "test/files/incorrect-fixes-test"], (err) => {
-                assert.isNotNull(err, "process should exit with error");
-                assert.strictEqual(err.code, 1, "error code should be 1");
-                done();
-            });
+            const status = await execRunner({test: true, files: ["test/files/custom-rule-rule-test", "test/files/incorrect-fixes-test"]});
+            assert.equal(status, Status.FatalError, "error code should be 1");
         });
     });
 
     describe("--project flag", () => {
-        it("exits with code 0 if `tsconfig.json` is passed and it specifies files without errors", (done) => {
-            execCli(["-c", "test/files/tsconfig-test/tslint.json", "--project", "test/files/tsconfig-test/tsconfig.json"], (err) => {
-                assert.isNull(err, "process should exit without an error");
-                done();
-            });
-        });
-
-        it("can be passed a directory and defaults to tsconfig.json", (done) => {
-            execCli(["-c", "test/files/tsconfig-test/tslint.json", "--project", "test/files/tsconfig-test"], (err) => {
-                assert.isNull(err, "process should exit without an error");
-                done();
-            });
+        it("exits with code 0 if `tsconfig.json` is passed and it specifies files without errors", async () => {
+            const status = await execRunner(
+                {config: "test/files/tsconfig-test/tslint.json", project: "test/files/tsconfig-test/tsconfig.json"},
+            );
+            assert.equal(status, Status.Ok, "process should exit without an error");
         });
 
-        it("exits with error if passed a directory and there is not tsconfig.json", (done) => {
-            execCli(["-c", "test/files/tsconfig-test/tslint.json", "--project", "test/files"], (err) => {
-                assert.isNotNull(err, "process should exit with an error");
-                assert.strictEqual(err.code, 1, "error code should be 1");
-                done();
-            });
+        it("can be passed a directory and defaults to tsconfig.json", async () => {
+            const status = await execRunner(
+                {config: "test/files/tsconfig-test/tslint.json", project: "test/files/tsconfig-test"},
+            );
+            assert.equal(status, Status.Ok, "process should exit without an error");
         });
 
-        it("exits with error if passed directory does not exist", (done) => {
-            execCli(["-c", "test/files/tsconfig-test/tslint.json", "--project", "test/files/non-existent"], (err) => {
-                assert.isNotNull(err, "process should exit with an error");
-                assert.strictEqual(err.code, 1, "error code should be 1");
-                done();
-            });
+        it("exits with error if passed a directory and there is not tsconfig.json", async () => {
+            const status = await execRunner(
+                {config: "test/files/tsconfig-test/tslint.json", project: "test/files"},
+            );
+            assert.equal(status, Status.FatalError, "exit code should be 1");
         });
 
-        it("exits with code 1 if file is not included in project", (done) => {
-            execCli(
-                [
-                    "-c",
-                    "test/files/tsconfig-test/tslint.json",
-                    "--project",
-                    "test/files/tsconfig-test/tsconfig.json",
-                    "test/files/tsconfig-test/other.test.ts",
-                ],
-                (err) => {
-                    assert.isNotNull(err, "process should exit with error");
-                    assert.strictEqual(err.code, 1, "error code should be 1");
-                    done();
-                });
+        it("exits with error if passed directory does not exist", async () => {
+            const status = await execRunner(
+                {config: "test/files/tsconfig-test/tslint.json", project: "test/files/non-existent"},
+            );
+            assert.equal(status, Status.FatalError, "exit code should be 1");
         });
 
-        it("warns if file-to-lint does not exist", (done) => {
-            execCli(
-                [
-                    "--project",
-                    "test/files/tsconfig-test/tsconfig.json",
-                    "test/files/tsconfig-test/non-existent.test.ts",
-                ],
-                (err, _stdout, stderr) => {
-                    assert.isNull(err, "process should exit without error");
-                    assert.include(stderr, "test/files/tsconfig-test/non-existent.test.ts' does not exist");
-                    done();
-                });
+        it("exits with code 1 if file is not included in project", async () => {
+            const status = await execRunner(
+                {
+                    config: "test/files/tsconfig-test/tslint.json",
+                    files: ["test/files/tsconfig-test/other.test.ts"],
+                    project: "test/files/tsconfig-test/tsconfig.json",
+                },
+            );
+            assert.equal(status, Status.FatalError, "exit code should be 1");
         });
 
-        it("doesn't warn for non-existent file-to-lint if excluded by --exclude", (done) => {
-            execCli(
-                [
-                    "--project",
-                    "test/files/tsconfig-test/tsconfig.json",
-                    "test/files/tsconfig-test/non-existent.test.ts",
-                    "--exclude",
-                    "**/*",
-                ],
-                (err, _stdout, stderr) => {
-                    assert.isNull(err, "process should exit without error");
-                    assert.notInclude(stderr, "does not exist");
-                    done();
-                });
+        it("exits with code 0 if `tsconfig.json` is passed but it includes no ts files", async () => {
+            const status = await execRunner(
+                {config: "test/files/tsconfig-no-ts-files/tslint.json", project: "test/files/tsconfig-no-ts-files/tsconfig.json"},
+            );
+            assert.equal(status, Status.Ok, "process should exit without an error");
         });
 
-        it("doesn't warn ig glob pattern doesn't match any file", (done) => {
-            execCli(
-                [
-                    "--project",
-                    "test/files/tsconfig-test/tsconfig.json",
-                    "*.js",
-                ],
-                (err, _stdout, stderr) => {
-                    assert.isNull(err, "process should exit without error");
-                    assert.notInclude(stderr, "does not exist");
-                    done();
-                });
+        it("can extend `tsconfig.json` with relative path", async () => {
+            const status1 = await execRunner(
+                {
+                    config: "test/files/tsconfig-extends-relative/tslint-ok.json",
+                    project: "test/files/tsconfig-extends-relative/test/tsconfig.json",
+                },
+            );
+            assert.equal(status1, Status.Ok, "process should exit without an error");
+            const status2 = await execRunner(
+                {
+                    config: "test/files/tsconfig-extends-relative/tslint-fail.json",
+                    project: "test/files/tsconfig-extends-relative/test/tsconfig.json",
+                },
+            );
+            assert.equal(status2, Status.LintError, "exit code should be 2");
         });
 
-        it("exits with code 0 if `tsconfig.json` is passed but it includes no ts files", (done) => {
-            execCli(
-                ["-c", "test/files/tsconfig-no-ts-files/tslint.json", "-p", "test/files/tsconfig-no-ts-files/tsconfig.json"],
-                (err) => {
-                    assert.isNull(err, "process should exit without an error");
-                    done();
-                });
+        it("warns if file-to-lint does not exist", async () => {
+            const result = await execRunnerWithOutput(
+                {project: "test/files/tsconfig-test/tsconfig.json", files: ["test/files/tsconfig-test/non-existent.test.ts"]},
+            );
+            assert.strictEqual(result.status, Status.Ok, "process should exit without error");
+            assert.include(result.stderr, `${path.normalize("test/files/tsconfig-test/non-existent.test.ts")}' does not exist`);
         });
 
-        it("reports errors from parsing tsconfig.json", (done) => {
-            execCli(
-                ["-p", "test/files/tsconfig-invalid/syntax-error.json"],
-                (err, _stdout, stderr) => {
-                    assert.isNotNull(err, "process should exit with an error");
-                    assert.equal(err.code, 1, "exit code should be 1");
-                    assert.include(stderr, "error TS");
-                    done();
-                });
+        it("doesn't warn for non-existent file-to-lint if excluded by --exclude", async () => {
+            const result = await execRunnerWithOutput({
+                exclude: ["**/*"],
+                files: ["test/files/tsconfig-test/non-existent.test.ts"],
+                project: "test/files/tsconfig-test/tsconfig.json",
+            });
+            assert.strictEqual(result.status, Status.Ok, "process should exit without error");
+            assert.notInclude(result.stderr, "does not exist");
         });
 
-        it("reports errors from validating tsconfig.json", (done) => {
-            execCli(
-                ["-p", "test/files/tsconfig-invalid/empty-files.json"],
-                (err, _stdout, stderr) => {
-                    assert.isNotNull(err, "process should exit with an error");
-                    assert.equal(err.code, 1, "exit code should be 1");
-                    assert.include(stderr, "error TS");
-                    done();
-                });
+        it("doesn't warn if glob pattern doesn't match any file", async () => {
+            const result = await execRunnerWithOutput({project: "test/files/tsconfig-test/tsconfig.json", files: ["*.js"]});
+            assert.strictEqual(result.status, Status.Ok, "process should exit without error");
+            assert.notInclude(result.stderr, "does not exist");
         });
 
-        it("does not report an error if tsconfig.json matches no files", (done) => {
-            execCli(
-                ["-p", "test/files/tsconfig-invalid/no-match.json"],
-                (err) => {
-                    assert.isNull(err, "process should exit without an error");
-                    done();
-                });
+        it("reports errors from parsing tsconfig.json", async () => {
+            const result = await execRunnerWithOutput({project: "test/files/tsconfig-invalid/syntax-error.json"});
+            assert.strictEqual(result.status, Status.FatalError, "exit code should be 1");
+            assert.include(result.stderr, "error TS");
         });
 
-        it("can extend `tsconfig.json` with relative path", (done) => {
-            execCli(
-                ["-c", "test/files/tsconfig-extends-relative/tslint-ok.json", "-p",
-                 "test/files/tsconfig-extends-relative/test/tsconfig.json"],
-                (err) => {
-                    assert.isNull(err, "process should exit without an error");
-                    done();
-                });
+        it("reports errors from validating tsconfig.json", async () => {
+            const result = await execRunnerWithOutput({project: "test/files/tsconfig-invalid/empty-files.json"});
+            assert.strictEqual(result.status, Status.FatalError, "exit code should be 1");
+            assert.include(result.stderr, "error TS");
         });
 
-        it("can extend `tsconfig.json` with relative path II", (done) => {
-            execCli(
-                ["-c", "test/files/tsconfig-extends-relative/tslint-fail.json", "-p",
-                 "test/files/tsconfig-extends-relative/test/tsconfig.json"],
-                (err) => {
-                    assert.isNotNull(err, "process should exit with error");
-                    assert.strictEqual(err.code, 2, "error code should be 2");
-                    done();
-                });
+        it("does not report an error if tsconfig.json matches no files", async () => {
+            const status = await execRunner({project: "test/files/tsconfig-invalid/no-match.json"});
+            assert.strictEqual(status, Status.Ok, "process should exit without an error");
         });
 
-        it("can execute typed rules without --type-check", (done) => {
-            execCli(
-                [ "-p", "test/files/typed-rule/tsconfig.json"],
-                (err) => {
-                    assert.isNotNull(err, "process should exit with error");
-                    assert.strictEqual(err.code, 2, "error code should be 2");
-                    done();
-                });
+        it("can execute typed rules without --type-check", async () => {
+            const status = await execRunner({project: "test/files/typed-rule/tsconfig.json"});
+            assert.equal(status, Status.LintError, "exit code should be 2");
         });
 
-        it("handles 'allowJs' correctly", (done) => {
-            execCli(
-                [ "-p", "test/files/tsconfig-allow-js/tsconfig.json"],
-                (err) => {
-                    assert.isNotNull(err, "process should exit with error");
-                    assert.strictEqual(err.code, 2, "error code should be 2");
-                    done();
-                });
+        it("handles 'allowJs' correctly", async () => {
+            const status = await execRunner({project: "test/files/tsconfig-allow-js/tsconfig.json"});
+            assert.equal(status, Status.LintError, "exit code should be 2");
         });
 
-        it("doesn't lint external dependencies with 'allowJs'", (done) => {
-            execCli(
-                [ "-p", "test/files/allow-js-exclude-node-modules/tsconfig.json"],
-                (err) => {
-                    assert.isNull(err, "process should exit without error");
-                    done();
-                });
+        it("doesn't lint external dependencies with 'allowJs'", async () => {
+            const status = await execRunner({project: "test/files/allow-js-exclude-node-modules/tsconfig.json"});
+            assert.equal(status, Status.Ok, "process should exit without error");
         });
 
-        it("works with '--exclude'", (done) => {
-            execCli(
-                [ "-p", "test/files/tsconfig-allow-js/tsconfig.json", "-e", "'test/files/tsconfig-allow-js/testfile.test.js'"],
-                (err) => {
-                    assert.isNull(err, "process should exit without an error");
-                    done();
-                });
+        it("works with '--exclude'", async () => {
+            const status = await execRunner(
+                {project: "test/files/tsconfig-allow-js/tsconfig.json", exclude: ["test/files/tsconfig-allow-js/testfile.test.js"]},
+            );
+            assert.equal(status, Status.Ok, "process should exit without an error");
         });
 
-        it("can apply fixes from multiple rules", (done) => {
+        it("can apply fixes from multiple rules", async () => {
             fs.writeFileSync("test/files/project-multiple-fixes/testfile.test.ts",
                              fs.readFileSync("test/files/project-multiple-fixes/before.test.ts", "utf-8"));
-            execCli(
-                [ "-p", "test/files/project-multiple-fixes/", "--fix"],
-                (err) => {
-                    const actual = fs.readFileSync("test/files/project-multiple-fixes/testfile.test.ts", "utf-8");
-                    fs.unlinkSync("test/files/project-multiple-fixes/testfile.test.ts");
-                    assert.isNull(err, "process should exit without an error");
-                    assert.strictEqual(
-                        actual,
-                        fs.readFileSync("test/files/project-multiple-fixes/after.test.ts", "utf-8"),
-                    );
-                    done();
-                });
+            const status = await execRunner({project: "test/files/project-multiple-fixes/", fix: true});
+            const actual = fs.readFileSync("test/files/project-multiple-fixes/testfile.test.ts", "utf-8");
+            fs.unlinkSync("test/files/project-multiple-fixes/testfile.test.ts");
+            assert.equal(status, Status.Ok, "process should exit without an error");
+            assert.strictEqual(
+                actual,
+                fs.readFileSync("test/files/project-multiple-fixes/after.test.ts", "utf-8"),
+            );
         }).timeout(8000);
     });
 
@@ -599,7 +480,6 @@ describe("Executable", function(this: Mocha.ISuiteCallbackContext) {
         afterEach(cleanTempInitFile);
 
         it("exits with code 0 if `--init` flag is used in folder without tslint.json", (done) => {
-
             execCli(["--init"], { cwd: EXECUTABLE_DIR }, (err) => {
                 assert.isNull(err, "process should exit without an error");
                 assert.strictEqual(fs.existsSync(TEMP_JSON_PATH), true, "file should be created");
@@ -625,26 +505,28 @@ describe("Executable", function(this: Mocha.ISuiteCallbackContext) {
         // on Windows - pattern string without any quotes
         // on Linux - list of files that matches glob (may differ from `glob` module results)
 
-        it("exits with code 2 if correctly finds file containing lint errors when glob is in double quotes", (done) => {
+        it("exits with code 2 if correctly finds file containing lint errors when glob is in double quotes", async () => {
             // when glob pattern is passed in double quotes in npm script `process.env` will contain:
             // on Windows - pattern string without any quotes
             // on Linux - pattern string without any quotes (glob is not expanded)
-            execCli(["-c", "./test/config/tslint-custom-rules.json", "-r", "./test/files/custom-rules", "src/**/test.ts"], (err) => {
-                assert.isNotNull(err, "process should exit with error");
-                assert.strictEqual(err.code, 2, "error code should be 2");
-                done();
-            });
+            const status = await execRunner(
+                {config: "./test/config/tslint-custom-rules.json", rulesDirectory: "./test/files/custom-rules", files: ["src/**/test.ts"]},
+            );
+            assert.equal(status, Status.LintError, "error code should be 2");
         });
 
-        it("exits with code 2 if correctly finds file containing lint errors when glob is in single quotes", (done) => {
+        it("exits with code 2 if correctly finds file containing lint errors when glob is in single quotes", async () => {
             // when glob pattern is passed in single quotes in npm script `process.env` will contain:
             // on Windows - pattern string wrapped in single quotes
             // on Linux - pattern string without any quotes (glob is not expanded)
-            execCli(["-c", "./test/config/tslint-custom-rules.json", "-r", "./test/files/custom-rules", "'src/**/test.ts'"], (err) => {
-                assert.isNotNull(err, "process should exit with error");
-                assert.strictEqual(err.code, 2, "error code should be 2");
-                done();
-            });
+            const status = await execRunner(
+                {
+                    config: "./test/config/tslint-custom-rules.json",
+                    files: ["'src/**/test.ts'"],
+                    rulesDirectory: "./test/files/custom-rules",
+                },
+            );
+            assert.equal(status, Status.LintError, "error code should be 2");
         });
 
         it("can handle multiple '--exclude' globs", (done) => {
@@ -688,6 +570,22 @@ function execCli(args: string[], options: cp.ExecFileOptions | ExecFileCallback,
     });
 }
 
+function execRunnerWithOutput(options: Partial<Options>) { // tslint:disable-line:promise-function-async
+    let stdout = "";
+    let stderr = "";
+    return execRunner(
+        options,
+        {
+            log(text) { stdout += text; },
+            error(text) { stderr += text; },
+        },
+    ).then((status) => ({status, stderr, stdout}));
+}
+
+function execRunner(options: Partial<Options>, logger: Logger = dummyLogger) { // tslint:disable-line:promise-function-async
+    return run({exclude: [], files: [], ...options}, logger);
+}
+
 function isFunction(fn: any): fn is Function { // tslint:disable-line:ban-types
     return ({}).toString.call(fn) === "[object Function]";
 }
diff --git a/test/files/multiple-fixes-test/multiple-fixes.test.ts b/test/files/multiple-fixes-test/multiple-fixes.test.ts
deleted file mode 100644
index 83849f1a699..00000000000
--- a/test/files/multiple-fixes-test/multiple-fixes.test.ts
+++ /dev/null
@@ -1,2 +0,0 @@
-import * as x from "b"
-import * as y from "a_long_module";
diff --git a/test/ruleTestRunner.ts b/test/ruleTestRunner.ts
index b0260f57077..3223b863fb4 100644
--- a/test/ruleTestRunner.ts
+++ b/test/ruleTestRunner.ts
@@ -20,19 +20,22 @@ import * as path from "path";
 
 import { consoleTestResultHandler, runTest } from "../src/test";
 
-/* tslint:disable:no-console */
-console.log();
-console.log(chalk.underline("Testing Lint Rules:"));
-/* tslint:enable:no-console */
+process.stdout.write(chalk.underline("\nTesting Lint Rules:\n"));
 
 const testDirectories = glob.sync("test/rules/**/tslint.json").map(path.dirname);
 
 for (const testDirectory of testDirectories) {
     const results = runTest(testDirectory);
-    const didAllTestsPass = consoleTestResultHandler(results);
+    const didAllTestsPass = consoleTestResultHandler(results, {
+        log(m) {
+            process.stdout.write(m);
+        },
+        error(m) {
+            process.stderr.write(m);
+        },
+    });
     if (!didAllTestsPass) {
-        process.exit(1);
+        process.exitCode = 1;
+        break;
     }
 }
-
-process.exit(0);