Specify minimum version of typescript for tests (palantir#2323)

docs/develop/testing-rules/index.md

@@ -97,6 +97,20 @@ Then, at the bottom of our test file, we specify what full message each shortcut
 
 Again, we can run `grunt test` to make sure our rule is producing the output we expect. If it isn't we'll see the difference between the output from the rule and the output we marked.
 
+##### Typescript version requirement #####
+
+Sometimes a rule requires a minimum version of the typescript compiler or your test contains syntax that older versions of the typescript parser cannot handle.
+When testing with multiple versions of typescript - like `tslint` does in CI - those tests will fail.
+
+To avoid failing tests, each test file can specify a version requirement for typescript **in the first line**. If you don't specify one, the test will always be executed.
+
+Example:
+```
+[typescript]: >= 2.1.0
+```
+
+The syntax should look familiar, because it is basically the shorthand syntax from the chapter above. It needs to start with `[typescript]:`.
+The following part can be any [version range](https://github.com/npm/node-semver#ranges). The prerelease suffix will be removed before matching to allow testing with nightly builds.
 
 ### Tips & Tricks ###
 

package.json

@@ -36,13 +36,15 @@
     "verify": "npm-run-all clean compile lint test docs"
   },
   "dependencies": {
+    "@types/semver": "^5.3.30",
     "babel-code-frame": "^6.20.0",
     "colors": "^1.1.2",
     "diff": "^3.0.1",
     "findup-sync": "~0.3.0",
     "glob": "^7.1.1",
     "optimist": "~0.6.0",
     "resolve": "^1.1.7",
+    "semver": "^5.3.0",
     "tsutils": "^1.2.2"
   },
   "peerDependencies": {

src/test.ts

@@ -20,6 +20,7 @@ import * as diff from "diff";
 import * as fs from "fs";
 import * as glob from "glob";
 import * as path from "path";
+import * as semver from "semver";
 import * as ts from "typescript";
 
 import {Fix} from "./language/rule/rule";
@@ -30,17 +31,25 @@ import * as parse from "./test/parse";
 const MARKUP_FILE_EXTENSION = ".lint";
 const FIXES_FILE_EXTENSION = ".fix";
 
+export interface TestOutput {
+    skipped: false;
+    errorsFromLinter: LintError[];
+    errorsFromMarkup: LintError[];
+    fixesFromLinter: string;
+    fixesFromMarkup: string;
+    markupFromLinter: string;
+    markupFromMarkup: string;
+}
+
+export interface SkippedTest {
+    skipped: true;
+    requirement: string;
+}
+
 export interface TestResult {
     directory: string;
     results: {
-        [fileName: string]: {
-            errorsFromLinter: LintError[];
-            errorsFromMarkup: LintError[];
-            fixesFromLinter: string;
-            fixesFromMarkup: string;
-            markupFromLinter: string;
-            markupFromMarkup: string;
-        },
+        [fileName: string]: TestOutput | SkippedTest,
     };
 }
 
@@ -76,7 +85,26 @@ export function runTest(testDirectory: string, rulesDirectory?: string | string[
     for (const fileToLint of filesToLint) {
         const fileBasename = path.basename(fileToLint, MARKUP_FILE_EXTENSION);
         const fileCompileName = fileBasename.replace(/\.lint$/, "");
-        const fileText = fs.readFileSync(fileToLint, "utf8");
+        let fileText = fs.readFileSync(fileToLint, "utf8");
+        const tsVersionRequirement = parse.getTypescriptVersionRequirement(fileText);
+        if (tsVersionRequirement) {
+            const tsVersion = new semver.SemVer(ts.version);
+            // remove prerelease suffix when matching to allow testing with nightly builds
+            if (!semver.satisfies(`${tsVersion.major}.${tsVersion.minor}.${tsVersion.patch}`, tsVersionRequirement)) {
+                results.results[fileToLint] = {
+                    requirement: tsVersionRequirement,
+                    skipped: true,
+                };
+                continue;
+            }
+            // remove the first line from the file before continuing
+            const lineBreak = fileText.search(/\n/);
+            if (lineBreak === -1) {
+                fileText = "";
+            } else {
+                fileText = fileText.substr(lineBreak + 1);
+            }
+        }
         const fileTextWithoutMarkup = parse.removeErrorMarkup(fileText);
         const errorsFromMarkup = parse.parseErrorsFromMarkup(fileText);
 
@@ -161,6 +189,7 @@ export function runTest(testDirectory: string, rulesDirectory?: string | string[
             fixesFromMarkup: fixedFileText,
             markupFromLinter: parse.createMarkupFromErrors(fileTextWithoutMarkup, errorsFromMarkup),
             markupFromMarkup: parse.createMarkupFromErrors(fileTextWithoutMarkup, errorsFromLinter),
+            skipped: false,
         };
     }
 
@@ -186,22 +215,26 @@ export function consoleTestResultHandler(testResult: TestResult): boolean {
         const results = testResult.results[fileName];
         process.stdout.write(`${fileName}:`);
 
-        const markupDiffResults = diff.diffLines(results.markupFromMarkup, results.markupFromLinter);
-        const fixesDiffResults = diff.diffLines(results.fixesFromLinter, results.fixesFromMarkup);
-        const didMarkupTestPass = !markupDiffResults.some((diff) => !!diff.added || !!diff.removed);
-        const didFixesTestPass = !fixesDiffResults.some((diff) => !!diff.added || !!diff.removed);
-
         /* tslint:disable:no-console */
-        if (didMarkupTestPass && didFixesTestPass) {
-            console.log(colors.green(" Passed"));
+        if (results.skipped) {
+            console.log(colors.yellow(` Skipped, requires typescript ${results.requirement}`));
         } else {
-            console.log(colors.red(" Failed!"));
-            didAllTestsPass = false;
-            if (!didMarkupTestPass) {
-                displayDiffResults(markupDiffResults, MARKUP_FILE_EXTENSION);
-            }
-            if (!didFixesTestPass) {
-                displayDiffResults(fixesDiffResults, FIXES_FILE_EXTENSION);
+            const markupDiffResults = diff.diffLines(results.markupFromMarkup, results.markupFromLinter);
+            const fixesDiffResults = diff.diffLines(results.fixesFromLinter, results.fixesFromMarkup);
+            const didMarkupTestPass = !markupDiffResults.some((diff) => !!diff.added || !!diff.removed);
+            const didFixesTestPass = !fixesDiffResults.some((diff) => !!diff.added || !!diff.removed);
+
+            if (didMarkupTestPass && didFixesTestPass) {
+                console.log(colors.green(" Passed"));
+            } else {
+                console.log(colors.red(" Failed!"));
+                didAllTestsPass = false;
+                if (!didMarkupTestPass) {
+                    displayDiffResults(markupDiffResults, MARKUP_FILE_EXTENSION);
+                }
+                if (!didFixesTestPass) {
+                    displayDiffResults(fixesDiffResults, FIXES_FILE_EXTENSION);
+                }
             }
         }
         /* tslint:enable:no-console */

src/test/parse.ts

@@ -26,6 +26,15 @@ import {
 } from "./lines";
 import {errorComparator, LintError, lintSyntaxError} from "./lintError";
 
+export function getTypescriptVersionRequirement(text: string): string | undefined {
+    const lines = text.split(/\r?\n/);
+    const firstLine = parseLine(lines[0]);
+    if (firstLine instanceof MessageSubstitutionLine && firstLine.key === "typescript") {
+        return firstLine.message;
+    }
+    return undefined;
+}
+
 /**
  * Takes the full text of a .lint file and returns the contents of the file
  * with all error markup removed

test/rules/_integration/typescript-version/correct.js.lint

@@ -0,0 +1,3 @@
+[typescript]: >= 2.0.0
+'foo';
+~~~~~ [' should be "]

test/rules/_integration/typescript-version/correct.ts.fix

@@ -0,0 +1 @@
+"foo";

test/rules/_integration/typescript-version/correct.ts.lint

@@ -0,0 +1,3 @@
+[typescript]: >= 2.0.0
+'foo';
+~~~~~ [' should be "]

test/rules/_integration/typescript-version/skip.js.lint

@@ -0,0 +1,2 @@
+[typescript]: < 2.0.0
+'foo';

test/rules/_integration/typescript-version/skip.ts.lint

@@ -0,0 +1,2 @@
+[typescript]: 0.0.0
+'foo';

test/rules/_integration/typescript-version/substitution.ts.lint

@@ -0,0 +1,4 @@
+[typescript]: >= 2.0.0
+'foo'; // substition has the same key as the version requirement, but still works
+~~~~~ [typescript]
+[typescript]: ' should be "

test/rules/_integration/typescript-version/tslint.json

@@ -0,0 +1,8 @@
+{
+  "rules": {
+    "quotemark": [true, "double"]
+  },
+  "jsRules": {
+    "quotemark": [true, "double"]
+  }
+}

yarn.lock

@@ -61,6 +61,10 @@
   dependencies:
     "@types/node" "*"
 
+"@types/semver@^5.3.30":
+  version "5.3.30"
+  resolved "https://registry.yarnpkg.com/@types/semver/-/semver-5.3.30.tgz#b55a3bd07b6b8b35f9d4472e1fc3318b68a493b2"
+
 agent-base@2:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-2.0.1.tgz#bd8f9e86a8eb221fffa07bd14befd55df142815e"
@@ -896,14 +900,14 @@ semver-diff@^2.0.0:
   dependencies:
     semver "^5.0.3"
 
-"semver@2 || 3 || 4 || 5", semver@^5.0.3, semver@~5.0.1:
-  version "5.0.3"
-  resolved "https://registry.yarnpkg.com/semver/-/semver-5.0.3.tgz#77466de589cd5d3c95f138aa78bc569a3cb5d27a"
-
-semver@^5.1.0:
+"semver@2 || 3 || 4 || 5", semver@^5.0.3, semver@^5.1.0, semver@^5.3.0:
   version "5.3.0"
   resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f"
 
+semver@~5.0.1:
+  version "5.0.3"
+  resolved "https://registry.yarnpkg.com/semver/-/semver-5.0.3.tgz#77466de589cd5d3c95f138aa78bc569a3cb5d27a"
+
 shell-quote@^1.6.1:
   version "1.6.1"
   resolved "https://registry.yarnpkg.com/shell-quote/-/shell-quote-1.6.1.tgz#f4781949cce402697127430ea3b3c5476f481767"
@@ -1035,11 +1039,7 @@ tslint@latest:
     tsutils "^1.1.0"
     update-notifier "^2.0.0"
 
-tsutils@^1.1.0:
-  version "1.1.0"
-  resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-1.1.0.tgz#94e0c267624eeb1b63561ba8ec0bcff71b4e2872"
-
-tsutils@^1.2.2:
+tsutils@^1.1.0, tsutils@^1.2.2:
   version "1.2.2"
   resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-1.2.2.tgz#7e165c601367b9f89200b97ff47d9e38d1a6e4c8"