Record logs for tests to Amazon S3

.gitignore

@@ -1,4 +1,5 @@
 .DS_Store
+.dart_code_logs/
 .flutter-plugins/
 .nyc_output/
 .packages

.travis.yml

@@ -20,16 +20,26 @@ addons:
       - gcc-4.8
 
 deploy:
-  provider: s3
-  access_key_id: $ARTIFACTS_KEY
-  secret_access_key: $ARTIFACTS_SECRET
-  bucket: test-results.dartcode.org
-  local_dir: .test_results
-  upload_dir: $TRAVIS_BRANCH/$TRAVIS_COMMIT/$TRAVIS_OS_NAME
-  skip_cleanup: true
-  acl: public_read
-  on:
-    all_branches: true
+  - provider: s3
+    access_key_id: $ARTIFACTS_KEY
+    secret_access_key: $ARTIFACTS_SECRET
+    bucket: test-results.dartcode.org
+    local_dir: .test_results
+    upload_dir: $TRAVIS_BRANCH/$TRAVIS_COMMIT/$TRAVIS_OS_NAME
+    skip_cleanup: true
+    acl: public_read
+    on:
+      all_branches: true
+  - provider: s3
+    access_key_id: $ARTIFACTS_KEY
+    secret_access_key: $ARTIFACTS_SECRET
+    bucket: test-results.dartcode.org
+    local_dir: .dart_code_logs
+    upload_dir: logs/$TRAVIS_BRANCH/$TRAVIS_COMMIT/$TRAVIS_OS_NAME/.dart_code_logs
+    skip_cleanup: true
+    acl: public_read
+    on:
+      all_branches: true
 
 before_install:
   - mkdir with\ spaces

.vscodeignore

@@ -1,3 +1,4 @@
+.dart_code_logs/**
 .gitattributes
 .gitignore
 .nyc_output/**

appveyor.yml

@@ -3,21 +3,26 @@ environment:
     secure: Oawgup4p18hTqsE2qGz07QLvtL/xrDiGFmqs9ebmXATs6t/Mp6L/sVS0XbAgmXhuOtZ8Q3KDC//q0ViBEnNiaEcZg3cI3GGPchtIQYjAozEC+3d3SXQY9zcSymAPgYsmc1TOsC+3BbuYXskoTu6myDbRPfh2i0YuU2lr0SRRbGS+68fGhBiPhZtkS6DOfDfu
 
 deploy:
-  provider: S3
-  access_key_id:
-    secure: nQjpvf+MHWHTkWOMvX6/eNKByxvyRMUDJNGaE+m74OY=
-  secret_access_key:
-    secure: Kqh0mg85kGAE2roKfIiMOQXjjbR9ib+uCwBaI11H0JfygkVP27adHUF3JYwqa9AE
-  bucket: test-results.dartcode.org
-  region: us-east-1
-  set_public: true
-  folder: $(APPVEYOR_REPO_BRANCH)/$(APPVEYOR_REPO_COMMIT)
-  artifact: test-results
-
-artifacts:
-  # Folder is renamed from .test-_esults in after_test
-  - path: win\*.xml
-    name: test-results
+  - provider: S3
+    access_key_id:
+      secure: nQjpvf+MHWHTkWOMvX6/eNKByxvyRMUDJNGaE+m74OY=
+    secret_access_key:
+      secure: Kqh0mg85kGAE2roKfIiMOQXjjbR9ib+uCwBaI11H0JfygkVP27adHUF3JYwqa9AE
+    bucket: test-results.dartcode.org
+    region: us-east-1
+    set_public: true
+    folder: $(APPVEYOR_REPO_BRANCH)/$(APPVEYOR_REPO_COMMIT)
+    artifact: test-results
+  - provider: S3
+    access_key_id:
+      secure: nQjpvf+MHWHTkWOMvX6/eNKByxvyRMUDJNGaE+m74OY=
+    secret_access_key:
+      secure: Kqh0mg85kGAE2roKfIiMOQXjjbR9ib+uCwBaI11H0JfygkVP27adHUF3JYwqa9AE
+    bucket: test-results.dartcode.org
+    region: us-east-1
+    set_public: true
+    folder: logs/$(APPVEYOR_REPO_BRANCH)/$(APPVEYOR_REPO_COMMIT)
+    artifact: logs
 
 install:
   - ps: mkdir "with spaces"

src/config.ts

@@ -4,6 +4,9 @@ import { versionIsAtLeast, resolveHomePath } from "./utils";
 class Config {
 	private config: WorkspaceConfiguration;
 
+	// Don't restart extension on config changes in tests
+	public get restartExtensionOnConfigChange() { return !process.env.DC_TEST_LOGS; }
+
 	constructor() {
 		workspace.onDidChangeConfiguration((e) => this.loadConfig());
 		this.loadConfig();

src/debug/dart_debug_impl.ts

@@ -202,6 +202,8 @@ export class DartDebugSession extends DebugSession {
 					}));
 				}
 
+				// TODO: Handle errors (such as these failing because we sent them too early).
+				// https://github.com/Dart-Code/Dart-Code/issues/790
 				Promise.all(promises).then((_) => {
 					this.sendEvent(new InitializedEvent());
 				});

src/extension.ts

@@ -345,7 +345,7 @@ function handleConfigurationChange(sdks: util.Sdks) {
 		});
 	}
 
-	if (settingsChanged) {
+	if (settingsChanged && config.restartExtensionOnConfigChange) {
 		util.reloadExtension();
 	}
 }

test/dart_only/debug/dart_cli.test.ts

@@ -7,6 +7,7 @@ import { activate, ext, helloWorldMainFile, helloWorldBrokenFile, closeAllOpenFi
 
 describe("dart cli debugger", () => {
 	const dc = new DebugClient(process.execPath, path.join(ext.extensionPath, "out/src/debug/dart_debug_entry.js"), "dart");
+	dc.defaultTimeout = 30000;
 
 	beforeEach(() => activate(helloWorldMainFile));
 	beforeEach(() => dc.start());

test/dart_only/debug/debug_config_provider.test.ts

@@ -11,6 +11,7 @@ const testDebugServerPortNumber = 4715;
 // Skipped because of https://github.com/Microsoft/vscode/issues/46028
 describe.skip("debug_config_provider", () => {
 	const dc = new DebugClient("node", "./out/src/debug/dart_debug_entry.js", "dart");
+	dc.defaultTimeout = 30000;
 	const debugConfig: vs.DebugConfiguration = {
 		debugServer: testDebugServerPortNumber,
 		name: "Dart & Flutter",

test/flutter_only/debug/flutter_test.test.ts

@@ -7,8 +7,8 @@ import { activate, ext, closeAllOpenFiles, flutterHelloWorldMainFile, flutterTes
 
 describe("flutter test debugger", () => {
 	const dc = new DebugClient(process.execPath, path.join(ext.extensionPath, "out/src/debug/flutter_test_debug_entry.js"), "dart");
-	// Spawning flutter tests seem to be kinda slow, so we need a higher timeout
-	dc.defaultTimeout = 30000;
+	// Spawning flutter tests seem to be kinda slow (and may fetch packages), so we need a higher timeout
+	dc.defaultTimeout = 60000;
 
 	beforeEach(() => activate(flutterTestMainFile));
 	beforeEach(() => dc.start());
@@ -67,13 +67,18 @@ describe("flutter test debugger", () => {
 		]);
 	});
 
-	it("receives stderr for failing tests", async () => {
+	// Skipped because https://github.com/flutter/flutter/issues/16350
+	it.skip("stops on exception", async () => {
+		await openFile(flutterTestBrokenFile);
 		const config = await configFor(flutterTestBrokenFile);
 		await Promise.all([
 			dc.configurationSequence(),
 			dc.launch(config),
 			dc.assertOutput("stderr", "Test failed. See exception logs above."),
-			dc.waitForEvent("terminated"),
+			dc.assertStoppedLocation("exception", {
+				line: positionOf("^won't find this").line,
+				path: flutterTestBrokenFile.fsPath,
+			}),
 		]);
 	});
 

test/helpers.ts

@@ -69,6 +69,36 @@ export function defer(callback: () => Promise<void> | void): void {
 	deferredItems.push(callback);
 }
 
+// Set up log files for individual test logging.
+// tslint:disable-next-line:only-arrow-functions
+beforeEach(async function () {
+	const logFolder = process.env.DC_TEST_LOGS || path.join(ext.extensionPath, ".dart_code_logs");
+	const prefix = filenameSafe(this.currentTest.fullTitle()) + "_";
+
+	await setLogs(
+		vs.workspace.getConfiguration("dart"),
+		logFolder,
+		prefix,
+		["analyzer", "flutterDaemon"],
+	);
+	await setLogs(
+		vs.workspace.getConfiguration("dart", vs.workspace.workspaceFolders[0].uri),
+		logFolder,
+		prefix,
+		["observatory", "flutterRun", "flutterTest"],
+	);
+});
+
+async function setLogs(conf: vs.WorkspaceConfiguration, logFolder: string, prefix: string, logFiles: string[]): Promise<void> {
+	for (const logFile of logFiles) {
+		const key = logFile + "LogFile";
+		const logPath = path.join(logFolder, `${prefix}${logFile}.txt`);
+		const oldValue = conf.get<string>(key);
+		await conf.update(key, logPath);
+		defer(async () => await conf.update(key, oldValue));
+	}
+}
+
 export function setTestContent(content: string): Thenable<boolean> {
 	const all = new vs.Range(
 		doc.positionAt(0),
@@ -234,3 +264,7 @@ export async function waitForEditorChange(action: () => Thenable<void>): Promise
 	await action();
 	await waitFor(() => doc.version !== oldVersion);
 }
+
+export function filenameSafe(input: string) {
+	return input.replace(/[^a-z0-9]+/gi, "_").toLowerCase();
+}

test/test_all.ts

@@ -65,15 +65,24 @@ async function runTests(testFolder: string, workspaceFolder: string, sdkPaths: s
 	env.MOCHA_FORBID_ONLY = true;
 	env.CODE_TESTS_WORKSPACE = path.join(cwd, "test", "test_projects", workspaceFolder);
 	env.CODE_TESTS_PATH = path.join(cwd, "out", "test", testFolder);
-	if (!fs.existsSync(".nyc_output"))
-		fs.mkdirSync(".nyc_output");
 
 	// Figure out a filename for results...
 	const dartFriendlyName = sdkPaths === process.env.PATH_UNSTABLE ? "dev" : "stable";
 	const codeFriendlyName = codeVersion === "*" ? "stable" : "insiders";
 
+	// Set some paths that are used inside the test run.
+	env.DC_TEST_LOGS = path.join(cwd, ".dart_code_logs", `${testFolder.replace("/", "_")}_${dartFriendlyName}_${codeFriendlyName}`);
 	env.COVERAGE_OUTPUT = path.join(cwd, ".nyc_output", `${testFolder.replace("/", "_")}_${dartFriendlyName}_${codeFriendlyName}.json`);
 	env.TEST_XML_OUTPUT = path.join(cwd, ".test_results", `${testFolder.replace("/", "_")}_${dartFriendlyName}_${codeFriendlyName}.xml`);
+
+	// Ensure any necessary folders exist.
+	if (!fs.existsSync(".nyc_output"))
+		fs.mkdirSync(".nyc_output");
+	if (!fs.existsSync(".dart_code_logs"))
+		fs.mkdirSync(".dart_code_logs");
+	if (!fs.existsSync(env.DC_TEST_LOGS))
+		fs.mkdirSync(env.DC_TEST_LOGS);
+
 	let res = await runNode(cwd, args, env);
 	if (!allowFailures)
 		exitCode = exitCode || res;

test/test_runner.ts

@@ -25,8 +25,8 @@ testRunner.configure({
 			output: process.env.TEST_XML_OUTPUT,
 		},
 	},
-	slow: 1500,       // increased threshold before marking a test as slow
-	timeout: 20000,   // increased timeout because starting up Code, Analyzer, etc. is slooow
+	slow: 10000,       // increased threshold before marking a test as slow
+	timeout: 60000,   // increased timeout because starting up Code, Analyzer, etc. is slooow
 	ui: "bdd",        // the TDD UI is being used in extension.test.ts (suite, test, etc.)
 	useColors: true,  // colored output from test results
 } as any); // TODO: Remove cast once vscode extension is updated.