Skip to content

Commit

Permalink
Merge pull request microsoft#3820 from Microsoft/newMochaReporters
Browse files Browse the repository at this point in the history 
New mocha reporter
  • Loading branch information
@danquirk
danquirk committed Jul 14, 2015
2 parents c5aa3ad + 15a8544 commit 649e40b
Show file tree
Hide file tree
Showing 5 changed files with 186 additions and 177 deletions.
2 changes: 1 addition & 1 deletion Jakefile.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -566,7 +566,7 @@ task("runtests", ["tests", builtLocalDirectory], function() {
colors = process.env.colors || process.env.color
colors = colors ? ' --no-colors ' : ' --colors ';
tests = tests ? ' -g ' + tests : '';
reporter = process.env.reporter || process.env.r || 'dot';
reporter = process.env.reporter || process.env.r || 'mocha-fivemat-progress-reporter';
// timeout normally isn't necessary but Travis-CI has been timing out on compiler baselines occasionally
// default timeout is 2sec which really should be enough, but maybe we just need a small amount longer
var cmd = host + " -R " + reporter + tests + colors + ' -t ' + testTimeout + ' ' + run;
Expand Down
1 change: 1 addition & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
"chai": "latest",
"browserify": "latest",
"istanbul": "latest",
"mocha-fivemat-progress-reporter": "latest",
"tslint": "latest"
},
"scripts": {
Expand Down
56 changes: 30 additions & 26 deletions src/harness/compilerRunner.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ const enum CompilerTestType {

class CompilerBaselineRunner extends RunnerBase {
private basePath = 'tests/cases';
private testSuiteName: string;
private errors: boolean;
private emit: boolean;
private decl: boolean;
Expand All @@ -24,16 +25,17 @@ class CompilerBaselineRunner extends RunnerBase {
this.decl = true;
this.output = true;
if (testType === CompilerTestType.Conformance) {
this.basePath += '/conformance';
this.testSuiteName = 'conformance';
}
else if (testType === CompilerTestType.Regressions) {
this.basePath += '/compiler';
this.testSuiteName = 'compiler';
}
else if (testType === CompilerTestType.Test262) {
this.basePath += '/test262';
this.testSuiteName = 'test262';
} else {
this.basePath += '/compiler'; // default to this for historical reasons
this.testSuiteName = 'compiler'; // default to this for historical reasons
}
this.basePath += '/' + this.testSuiteName;
}

public checkTestCodeOutput(fileName: string) {
Expand Down Expand Up @@ -100,7 +102,7 @@ class CompilerBaselineRunner extends RunnerBase {
});

beforeEach(() => {
/* The compiler doesn't handle certain flags flipping during a single compilation setting. Tests on these flags will need
/* The compiler doesn't handle certain flags flipping during a single compilation setting. Tests on these flags will need
a fresh compiler instance for themselves and then create a fresh one for the next test. Would be nice to get dev fixes
eventually to remove this limitation. */
for (var i = 0; i < tcSettings.length; ++i) {
Expand Down Expand Up @@ -261,19 +263,19 @@ class CompilerBaselineRunner extends RunnerBase {

// NEWTODO: Type baselines
if (result.errors.length === 0) {
// The full walker simulates the types that you would get from doing a full
// The full walker simulates the types that you would get from doing a full
// compile. The pull walker simulates the types you get when you just do
// a type query for a random node (like how the LS would do it). Most of the
// time, these will be the same. However, occasionally, they can be different.
// Specifically, when the compiler internally depends on symbol IDs to order
// things, then we may see different results because symbols can be created in a
// things, then we may see different results because symbols can be created in a
// different order with 'pull' operations, and thus can produce slightly differing
// output.
//
// For example, with a full type check, we may see a type outputed as: number | string
// But with a pull type check, we may see it as: string | number
//
// These types are equivalent, but depend on what order the compiler observed
// These types are equivalent, but depend on what order the compiler observed
// certain parts of the program.

let allFiles = toBeCompiled.concat(otherFiles).filter(file => !!program.getSourceFile(file.unitName));
Expand Down Expand Up @@ -384,25 +386,27 @@ class CompilerBaselineRunner extends RunnerBase {
}

public initializeTests() {
describe("Setup compiler for compiler baselines", () => {
var harnessCompiler = Harness.Compiler.getCompiler();
this.parseOptions();
});

// this will set up a series of describe/it blocks to run between the setup and cleanup phases
if (this.tests.length === 0) {
var testFiles = this.enumerateFiles(this.basePath, /\.tsx?$/, { recursive: true });
testFiles.forEach(fn => {
fn = fn.replace(/\\/g, "/");
this.checkTestCodeOutput(fn);
describe(this.testSuiteName + ' tests', () => {
describe("Setup compiler for compiler baselines", () => {
var harnessCompiler = Harness.Compiler.getCompiler();
this.parseOptions();
});
}
else {
this.tests.forEach(test => this.checkTestCodeOutput(test));
}

describe("Cleanup after compiler baselines", () => {
var harnessCompiler = Harness.Compiler.getCompiler();
// this will set up a series of describe/it blocks to run between the setup and cleanup phases
if (this.tests.length === 0) {
var testFiles = this.enumerateFiles(this.basePath, /\.tsx?$/, { recursive: true });
testFiles.forEach(fn => {
fn = fn.replace(/\\/g, "/");
this.checkTestCodeOutput(fn);
});
}
else {
this.tests.forEach(test => this.checkTestCodeOutput(test));
}

describe("Cleanup after compiler baselines", () => {
var harnessCompiler = Harness.Compiler.getCompiler();
});
});
}

Expand Down Expand Up @@ -434,4 +438,4 @@ class CompilerBaselineRunner extends RunnerBase {
}
}
}
}
}
124 changes: 63 additions & 61 deletions src/harness/fourslashRunner.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
///<reference path='harness.ts'/>
///<reference path='runnerbase.ts' />

const enum FourSlashTestType {
const enum FourSlashTestType {
Native,
Shims,
Server
Expand Down Expand Up @@ -35,70 +35,72 @@ class FourSlashRunner extends RunnerBase {
this.tests = this.enumerateFiles(this.basePath, /\.ts/i, { recursive: false });
}

this.tests.forEach((fn: string) => {
describe(fn, () => {
fn = ts.normalizeSlashes(fn);
var justName = fn.replace(/^.*[\\\/]/, '');
describe(this.testSuiteName + ' tests', () => {
this.tests.forEach((fn: string) => {
describe(fn, () => {
fn = ts.normalizeSlashes(fn);
var justName = fn.replace(/^.*[\\\/]/, '');

// Convert to relative path
var testIndex = fn.indexOf('tests/');
if (testIndex >= 0) fn = fn.substr(testIndex);

// Convert to relative path
var testIndex = fn.indexOf('tests/');
if (testIndex >= 0) fn = fn.substr(testIndex);
if (justName && !justName.match(/fourslash\.ts$/i) && !justName.match(/\.d\.ts$/i)) {
it(this.testSuiteName + ' test ' + justName + ' runs correctly',() => {
FourSlash.runFourSlashTest(this.basePath, this.testType, fn);
});
}
});
});

if (justName && !justName.match(/fourslash\.ts$/i) && !justName.match(/\.d\.ts$/i)) {
it(this.testSuiteName + ' test ' + justName + ' runs correctly', () => {
FourSlash.runFourSlashTest(this.basePath, this.testType, fn);
});
describe('Generate Tao XML', () => {
var invalidReasons: any = {};
FourSlash.xmlData.forEach(xml => {
if (xml.invalidReason !== null) {
invalidReasons[xml.invalidReason] = (invalidReasons[xml.invalidReason] || 0) + 1;
}
});
var invalidReport: { reason: string; count: number }[] = [];
for (var reason in invalidReasons) {
if (invalidReasons.hasOwnProperty(reason)) {
invalidReport.push({ reason: reason, count: invalidReasons[reason] });
}
}
});
});
invalidReport.sort((lhs, rhs) => lhs.count > rhs.count ? -1 : lhs.count === rhs.count ? 0 : 1);

describe('Generate Tao XML', () => {
var invalidReasons: any = {};
FourSlash.xmlData.forEach(xml => {
if (xml.invalidReason !== null) {
invalidReasons[xml.invalidReason] = (invalidReasons[xml.invalidReason] || 0) + 1;
}
var lines: string[] = [];
lines.push('<!-- Blocked Test Report');
invalidReport.forEach((reasonAndCount) => {
lines.push(reasonAndCount.count + ' tests blocked by ' + reasonAndCount.reason);
});
lines.push('-->');
lines.push('<TaoTest xmlns="http://microsoft.com/schemas/VSLanguages/TAO">');
lines.push(' <InitTest>');
lines.push(' <StartTarget />');
lines.push(' </InitTest>');
lines.push(' <ScenarioList>');
FourSlash.xmlData.forEach(xml => {
if (xml.invalidReason !== null) {
lines.push('<!-- Skipped ' + xml.originalName + ', reason: ' + xml.invalidReason + ' -->');
} else {
lines.push(' <Scenario Name="' + xml.originalName + '">');
xml.actions.forEach(action => {
lines.push(' ' + action);
});
lines.push(' </Scenario>');
}
});
lines.push(' </ScenarioList>');
lines.push(' <CleanupScenario>');
lines.push(' <CloseAllDocuments />');
lines.push(' <CleanupCreatedFiles />');
lines.push(' </CleanupScenario>');
lines.push(' <CleanupTest>');
lines.push(' <CloseTarget />');
lines.push(' </CleanupTest>');
lines.push('</TaoTest>');
Harness.IO.writeFile('built/local/fourslash.xml', lines.join('\r\n'));
});
var invalidReport: { reason: string; count: number }[] = [];
for (var reason in invalidReasons) {
if (invalidReasons.hasOwnProperty(reason)) {
invalidReport.push({ reason: reason, count: invalidReasons[reason] });
}
}
invalidReport.sort((lhs, rhs) => lhs.count > rhs.count ? -1 : lhs.count === rhs.count ? 0 : 1);

var lines: string[] = [];
lines.push('<!-- Blocked Test Report');
invalidReport.forEach((reasonAndCount) => {
lines.push(reasonAndCount.count + ' tests blocked by ' + reasonAndCount.reason);
});
lines.push('-->');
lines.push('<TaoTest xmlns="http://microsoft.com/schemas/VSLanguages/TAO">');
lines.push(' <InitTest>');
lines.push(' <StartTarget />');
lines.push(' </InitTest>');
lines.push(' <ScenarioList>');
FourSlash.xmlData.forEach(xml => {
if (xml.invalidReason !== null) {
lines.push('<!-- Skipped ' + xml.originalName + ', reason: ' + xml.invalidReason + ' -->');
} else {
lines.push(' <Scenario Name="' + xml.originalName + '">');
xml.actions.forEach(action => {
lines.push(' ' + action);
});
lines.push(' </Scenario>');
}
});
lines.push(' </ScenarioList>');
lines.push(' <CleanupScenario>');
lines.push(' <CloseAllDocuments />');
lines.push(' <CleanupCreatedFiles />');
lines.push(' </CleanupScenario>');
lines.push(' <CleanupTest>');
lines.push(' <CloseTarget />');
lines.push(' </CleanupTest>');
lines.push('</TaoTest>');
Harness.IO.writeFile('built/local/fourslash.xml', lines.join('\r\n'));
});
}
}
Expand All @@ -108,4 +110,4 @@ class GeneratedFourslashRunner extends FourSlashRunner {
super(testType);
this.basePath += '/generated/';
}
}
}
Loading

0 comments on commit 649e40b

Please sign in to comment.