testing: add temporary failure tracker to the selfhost test runner (m…

…icrosoft#212134)

For /fixTestFailures, I want to get more 'real world' tests and test
fixes. This makes a change in the selfhost test provider such that when
a test fails and is then fixed, we record the code changes into a JSON
file in the `.build` directory. In a few days I'll follow up with team
members to collect their test failures and use them as evaluation tests
for copilot. The FailureTracker will be removed when I've gotten enough
data.

.vscode/extensions/vscode-selfhost-test-provider/.vscode/launch.json

@@ -0,0 +1,11 @@
+{
+	"configurations": [
+		{
+			"args": ["--extensionDevelopmentPath=${workspaceFolder}"],
+			"name": "Launch Extension",
+			"outFiles": ["${workspaceFolder}/out/**/*.js"],
+			"request": "launch",
+			"type": "extensionHost"
+		}
+	]
+}

.vscode/extensions/vscode-selfhost-test-provider/src/extension.ts

@@ -19,6 +19,7 @@ import {
 	itemData,
 } from './testTree';
 import { BrowserTestRunner, PlatformTestRunner, VSCodeTestRunner } from './vscodeTestRunner';
+import { FailureTracker } from './failureTracker';
 
 const TEST_FILE_PATTERN = 'src/vs/**/*.{test,integrationTest}.ts';
 
@@ -54,6 +55,8 @@ export async function activate(context: vscode.ExtensionContext) {
 		}
 	};
 
+	let startedTrackingFailures = false;
+
 	const createRunHandler = (
 		runnerCtor: { new(folder: vscode.WorkspaceFolder): VSCodeTestRunner },
 		kind: vscode.TestRunProfileKind,
@@ -68,6 +71,11 @@ export async function activate(context: vscode.ExtensionContext) {
 				return;
 			}
 
+			if (!startedTrackingFailures) {
+				startedTrackingFailures = true;
+				context.subscriptions.push(new FailureTracker(folder.uri.fsPath));
+			}
+
 			const runner = new runnerCtor(folder);
 			const map = await getPendingTestMap(ctrl, req.include ?? gatherTestItems(ctrl.items));
 			const task = ctrl.createTestRun(req);
@@ -225,7 +233,7 @@ export async function activate(context: vscode.ExtensionContext) {
 		vscode.workspace.onDidOpenTextDocument(updateNodeForDocument),
 		vscode.workspace.onDidChangeTextDocument(e => updateNodeForDocument(e.document)),
 		registerSnapshotUpdate(ctrl),
-		new FailingDeepStrictEqualAssertFixer()
+		new FailingDeepStrictEqualAssertFixer(),
 	);
 }
 

.vscode/extensions/vscode-selfhost-test-provider/src/failureTracker.ts

@@ -0,0 +1,126 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { spawn } from 'child_process';
+import { readFile, writeFile } from 'fs/promises';
+import { join } from 'path';
+import * as vscode from 'vscode';
+
+interface IGitState {
+	commitId: string;
+	tracked: string;
+	untracked: Record<string, string>;
+}
+
+interface ITrackedRemediation {
+	snapshot: vscode.TestResultSnapshot;
+	failing: IGitState;
+	passing: IGitState;
+}
+
+const MAX_FAILURES = 10;
+
+export class FailureTracker {
+	private readonly disposables: vscode.Disposable[] = [];
+	private readonly lastFailed = new Map<
+		string,
+		{ snapshot: vscode.TestResultSnapshot; failing: IGitState }
+	>();
+
+	private readonly logFile = join(this.rootDir, '.build/vscode-test-failures.json');
+	private logs?: ITrackedRemediation[];
+
+	constructor(private readonly rootDir: string) {
+		this.disposables.push(
+			vscode.tests.onDidChangeTestResults(() => {
+				const last = vscode.tests.testResults[0];
+				if (!last) {
+					return;
+				}
+
+				let gitState: Promise<IGitState> | undefined;
+				const getGitState = () => gitState ?? (gitState = this.captureGitState());
+
+				const queue = [last.results];
+				for (let i = 0; i < queue.length; i++) {
+					for (const snapshot of queue[i]) {
+						// only interested in states of leaf tests
+						if (snapshot.children.length) {
+							queue.push(snapshot.children);
+							continue;
+						}
+
+						const key = `${snapshot.uri}/${snapshot.id}`;
+						const prev = this.lastFailed.get(key);
+						if (snapshot.taskStates.some(s => s.state === vscode.TestResultState.Failed)) {
+							// unset the parent to avoid a circular JSON structure:
+							getGitState().then(s => this.lastFailed.set(key, { snapshot: { ...snapshot, parent: undefined }, failing: s }));
+						} else if (prev) {
+							this.lastFailed.delete(key);
+							getGitState().then(s => this.append({ ...prev, passing: s }));
+						}
+					}
+				}
+			})
+		);
+	}
+
+	private async append(log: ITrackedRemediation) {
+		if (!this.logs) {
+			try {
+				this.logs = JSON.parse(await readFile(this.logFile, 'utf-8'));
+			} catch {
+				this.logs = [];
+			}
+		}
+
+		const logs = this.logs!;
+		logs.push(log);
+		if (logs.length > MAX_FAILURES) {
+			logs.splice(0, logs.length - MAX_FAILURES);
+		}
+
+		await writeFile(this.logFile, JSON.stringify(logs, undefined, 2));
+	}
+
+	private async captureGitState() {
+		const [commitId, tracked, untracked] = await Promise.all([
+			this.exec('git', ['rev-parse', 'HEAD']),
+			this.exec('git', ['diff', 'HEAD']),
+			this.exec('git', ['ls-files', '--others', '--exclude-standard']).then(async output => {
+				const mapping: Record<string, string> = {};
+				await Promise.all(
+					output
+						.trim()
+						.split('\n')
+						.map(async f => {
+							mapping[f] = await readFile(join(this.rootDir, f), 'utf-8');
+						})
+				);
+				return mapping;
+			}),
+		]);
+		return { commitId, tracked, untracked };
+	}
+
+	public dispose() {
+		this.disposables.forEach(d => d.dispose());
+	}
+
+	private exec(command: string, args: string[]): Promise<string> {
+		return new Promise((resolve, reject) => {
+			const child = spawn(command, args, { stdio: 'pipe', cwd: this.rootDir });
+			let output = '';
+			child.stdout.setEncoding('utf-8').on('data', b => (output += b));
+			child.stderr.setEncoding('utf-8').on('data', b => (output += b));
+			child.on('error', reject);
+			child.on('exit', code =>
+				code === 0
+					? resolve(output)
+					: reject(new Error(`Failed with error code ${code}\n${output}`))
+			);
+		});
+	}
+}

build/lib/watch/watch-win32.ts

@@ -70,7 +70,7 @@ function watch(root: string): Stream {
 
 const cache: { [cwd: string]: Stream } = Object.create(null);
 
-module.exports = function (pattern: string | string[] | filter.FileFunction, options?: { cwd?: string; base?: string }) {
+module.exports = function (pattern: string | string[] | filter.FileFunction, options?: { cwd?: string; base?: string; dot?: boolean }) {
 	options = options || {};
 
 	const cwd = path.normalize(options.cwd || process.cwd());
@@ -86,8 +86,8 @@ module.exports = function (pattern: string | string[] | filter.FileFunction, opt
 	});
 
 	return watcher
-		.pipe(filter(['**', '!.git{,/**}'])) // ignore all things git
-		.pipe(filter(pattern))
+		.pipe(filter(['**', '!.git{,/**}'], { dot: options.dot })) // ignore all things git
+		.pipe(filter(pattern, { dot: options.dot }))
 		.pipe(es.map(function (file: File, cb) {
 			fs.stat(file.path, function (err, stat) {
 				if (err && err.code === 'ENOENT') { return cb(undefined, file); }