Implement atom --benchmark-test to ensure benchmarks are valid on CI

atom.sh

@@ -28,7 +28,7 @@ while getopts ":wtfvh-:" opt; do
           REDIRECT_STDERR=1
           EXPECT_OUTPUT=1
           ;;
-        foreground|test)
+        foreground|benchmark|benchmark-test|test)
           EXPECT_OUTPUT=1
           ;;
       esac

benchmarks/benchmark-runner.js

@@ -5,7 +5,7 @@ import glob from 'glob'
 import fs from 'fs-plus'
 import path from 'path'
 
-export default async function (benchmarkPaths) {
+export default async function ({test, benchmarkPaths}) {
   document.body.style.backgroundColor = '#ffffff'
   document.body.style.overflow = 'auto'
 
@@ -19,7 +19,7 @@ export default async function (benchmarkPaths) {
   }
 
   while (paths.length > 0) {
-    const benchmark = require(paths.shift())()
+    const benchmark = require(paths.shift())({test})
     let results
     if (benchmark instanceof Promise) {
       results = await benchmark

benchmarks/text-editor-large-file-construction.bench.js

@@ -4,10 +4,10 @@ import fs from 'fs'
 import temp from 'temp'
 import {TextEditor, TextBuffer} from 'atom'
 
-export default function () {
+export default function ({test}) {
   const data = []
-  const maxLineCount = 10000
-  const step = 500
+  const maxLineCount = test ? 5 : 10000
+  const step = test ? 1 : 500
   const lineText = 'Lorem ipsum dolor sit amet\n'
   const sampleText = lineText.repeat(maxLineCount)
   for (let lineCount = 0; lineCount <= maxLineCount; lineCount += step) {

resources/win/atom.cmd

@@ -5,14 +5,16 @@ SET WAIT=
 SET PSARGS=%*
 
 FOR %%a IN (%*) DO (
-  IF /I "%%a"=="-f"           SET EXPECT_OUTPUT=YES
-  IF /I "%%a"=="--foreground" SET EXPECT_OUTPUT=YES
-  IF /I "%%a"=="-h"           SET EXPECT_OUTPUT=YES
-  IF /I "%%a"=="--help"       SET EXPECT_OUTPUT=YES
-  IF /I "%%a"=="-t"           SET EXPECT_OUTPUT=YES
-  IF /I "%%a"=="--test"       SET EXPECT_OUTPUT=YES
-  IF /I "%%a"=="-v"           SET EXPECT_OUTPUT=YES
-  IF /I "%%a"=="--version"    SET EXPECT_OUTPUT=YES
+  IF /I "%%a"=="-f"               SET EXPECT_OUTPUT=YES
+  IF /I "%%a"=="--foreground"     SET EXPECT_OUTPUT=YES
+  IF /I "%%a"=="-h"               SET EXPECT_OUTPUT=YES
+  IF /I "%%a"=="--help"           SET EXPECT_OUTPUT=YES
+  IF /I "%%a"=="-t"               SET EXPECT_OUTPUT=YES
+  IF /I "%%a"=="--test"           SET EXPECT_OUTPUT=YES
+  IF /I "%%a"=="--benchmark"      SET EXPECT_OUTPUT=YES
+  IF /I "%%a"=="--benchmark-test" SET EXPECT_OUTPUT=YES
+  IF /I "%%a"=="-v"               SET EXPECT_OUTPUT=YES
+  IF /I "%%a"=="--version"        SET EXPECT_OUTPUT=YES
   IF /I "%%a"=="-w"           (
     SET EXPECT_OUTPUT=YES
     SET WAIT=YES

src/initialize-benchmark-window.js

@@ -3,10 +3,11 @@
 import {ipcRenderer, remote} from 'electron'
 import path from 'path'
 import ipcHelpers from './ipc-helpers'
+import util from 'util'
 
 export default function () {
   const {getWindowLoadSettings} = require('./window-load-settings-helpers')
-  const {headless, resourcePath, benchmarkPaths} = getWindowLoadSettings()
+  const {test, headless, resourcePath, benchmarkPaths} = getWindowLoadSettings()
   try {
     const Clipboard = require('../src/clipboard')
     const ApplicationDelegate = require('../src/application-delegate')
@@ -58,14 +59,33 @@ export default function () {
     })
 
     // Prevent benchmarks from modifying application menus
+    global.atom.menu.update()
     global.atom.menu.sendToBrowserProcess = function () { }
 
-    if (!headless) {
+    if (headless) {
+      Object.defineProperties(process, {
+        stdout: { value: remote.process.stdout },
+        stderr: { value: remote.process.stderr }
+      })
+
+      console.log = function (...args) {
+        const formatted = util.format(...args)
+        process.stdout.write(formatted + "\n")
+      }
+      console.warn = function (...args) {
+        const formatted = util.format(...args)
+        process.stderr.write(formatted + "\n")
+      }
+      console.error = function (...args) {
+        const formatted = util.format(...args)
+        process.stderr.write(formatted + "\n")
+      }
+    } else {
       remote.getCurrentWindow().show()
     }
 
     const benchmarkRunner = require('../benchmarks/benchmark-runner')
-    return benchmarkRunner(benchmarkPaths).then((code) => {
+    return benchmarkRunner({test, benchmarkPaths}).then((code) => {
       if (headless) {
         exitWithStatusCode(code)
       }

src/main-process/atom-application.coffee

@@ -42,7 +42,7 @@ class AtomApplication
     # take a few seconds to trigger 'error' event, it could be a bug of node
     # or atom-shell, before it's fixed we check the existence of socketPath to
     # speedup startup.
-    if (process.platform isnt 'win32' and not fs.existsSync options.socketPath) or options.test or options.benchmark
+    if (process.platform isnt 'win32' and not fs.existsSync options.socketPath) or options.test or options.benchmark or options.benchmarkTest
       new AtomApplication(options).initialize(options)
       return
 
@@ -86,7 +86,9 @@ class AtomApplication
 
     @config.onDidChange 'core.useCustomTitleBar', @promptForRestart.bind(this)
 
-    @autoUpdateManager = new AutoUpdateManager(@version, options.test or options.benchmark, @resourcePath, @config)
+    @autoUpdateManager = new AutoUpdateManager(
+      @version, options.test or options.benchmark or options.benchmarkTest, @resourcePath, @config
+    )
     @applicationMenu = new ApplicationMenu(@version, @autoUpdateManager)
     @atomProtocolHandler = new AtomProtocolHandler(@resourcePath, @safeMode)
 
@@ -103,15 +105,17 @@ class AtomApplication
     Promise.all(windowsClosePromises).then(=> @disposable.dispose())
 
   launch: (options) ->
-    if options.pathsToOpen?.length > 0 or options.urlsToOpen?.length > 0 or options.test or options.benchmark
+    if options.pathsToOpen?.length > 0 or options.urlsToOpen?.length > 0 or options.test or options.benchmark or options.benchmarkTest
       @openWithOptions(options)
     else
       @loadState(options) or @openPath(options)
 
   openWithOptions: (options) ->
-    {initialPaths, pathsToOpen, executedFrom, urlsToOpen, benchmark, test,
-     pidToKillWhenClosed, devMode, safeMode, newWindow, logFile, profileStartup,
-     timeout, clearWindowState, addToLastWindow, env} = options
+    {
+      initialPaths, pathsToOpen, executedFrom, urlsToOpen, benchmark,
+      benchmarkTest, test, pidToKillWhenClosed, devMode, safeMode, newWindow,
+      logFile, profileStartup, timeout, clearWindowState, addToLastWindow, env
+    } = options
 
     app.focus()
 
@@ -120,8 +124,8 @@ class AtomApplication
         headless: true, devMode, @resourcePath, executedFrom, pathsToOpen,
         logFile, timeout, env
       })
-    else if benchmark
-      @runBenchmarks({headless: false, @resourcePath, executedFrom, pathsToOpen, timeout, env})
+    else if benchmark or benchmarkTest
+      @runBenchmarks({headless: true, test: benchmarkTest, @resourcePath, executedFrom, pathsToOpen, timeout, env})
     else if pathsToOpen.length > 0
       @openPaths({
         initialPaths, pathsToOpen, executedFrom, pidToKillWhenClosed, newWindow,
@@ -667,7 +671,7 @@ class AtomApplication
     safeMode ?= false
     new AtomWindow(this, @fileRecoveryService, {windowInitializationScript, resourcePath, headless, isSpec, devMode, testRunnerPath, legacyTestRunnerPath, testPaths, logFile, safeMode, env})
 
-  runBenchmarks: ({headless, resourcePath, executedFrom, pathsToOpen, env}) ->
+  runBenchmarks: ({headless, test, resourcePath, executedFrom, pathsToOpen, env}) ->
     if resourcePath isnt @resourcePath and not fs.existsSync(resourcePath)
       resourcePath = @resourcePath
 
@@ -688,7 +692,7 @@ class AtomApplication
     devMode = true
     isSpec = true
     safeMode = false
-    new AtomWindow(this, @fileRecoveryService, {windowInitializationScript, resourcePath, headless, isSpec, devMode, benchmarkPaths, safeMode, env})
+    new AtomWindow(this, @fileRecoveryService, {windowInitializationScript, resourcePath, headless, test, isSpec, devMode, benchmarkPaths, safeMode, env})
 
   resolveTestRunnerPath: (testPath) ->
     FindParentDir ?= require 'find-parent-dir'

src/main-process/main.js

@@ -19,7 +19,7 @@ if (args.resourcePath) {
   const stableResourcePath = path.dirname(path.dirname(__dirname))
   const defaultRepositoryPath = path.join(electron.app.getPath('home'), 'github', 'atom')
 
-  if (args.dev || args.test || args.benchmark) {
+  if (args.dev || args.test || args.benchmark || args.benchmarkTest) {
     if (process.env.ATOM_DEV_RESOURCE_PATH) {
       resourcePath = process.env.ATOM_DEV_RESOURCE_PATH
     } else if (fs.statSyncNoException(defaultRepositoryPath)) {

src/main-process/parse-command-line.js

@@ -45,7 +45,8 @@ module.exports = function parseCommandLine (processArgs) {
     'portable',
     'Set portable mode. Copies the ~/.atom folder to be a sibling of the installed Atom location if a .atom folder is not already there.'
   )
-  options.boolean('benchmark').describe('benchmark', 'Run the specified benchmarks.')
+  options.boolean('benchmark').describe('benchmark', 'Open a new window that runs the specified benchmarks.')
+  options.boolean('benchmark-test').describe('benchmark--test', 'Run a faster version of the benchmarks in headless mode.')
   options.alias('t', 'test').boolean('t').describe('t', 'Run the specified specs and exit with error code on failures.')
   options.alias('m', 'main-process').boolean('m').describe('m', 'Run the specified specs in the main process.')
   options.string('timeout').describe(
@@ -80,6 +81,7 @@ module.exports = function parseCommandLine (processArgs) {
   const safeMode = args['safe']
   const pathsToOpen = args._
   const benchmark = args['benchmark']
+  const benchmarkTest = args['benchmark-test']
   const test = args['test']
   const mainProcess = args['main-process']
   const timeout = args['timeout']
@@ -137,7 +139,7 @@ module.exports = function parseCommandLine (processArgs) {
     resourcePath, devResourcePath, pathsToOpen, urlsToOpen, executedFrom, test,
     version, pidToKillWhenClosed, devMode, safeMode, newWindow, logFile, socketPath,
     userDataDir, profileStartup, timeout, setPortable, clearWindowState,
-    addToLastWindow, mainProcess, benchmark, env: process.env
+    addToLastWindow, mainProcess, benchmark, benchmarkTest, env: process.env
   }
 }
 

src/main-process/start.js

@@ -57,7 +57,7 @@ module.exports = function start (resourcePath, startTime) {
 
   if (args.userDataDir != null) {
     app.setPath('userData', args.userDataDir)
-  } else if (args.test || args.benchmark) {
+  } else if (args.test || args.benchmark || args.benchmarkTest) {
     app.setPath('userData', temp.mkdirSync('atom-test-data'))
   }