run changed/added tests in deploy mode (vercel#67612)

Leverages the work from vercel#66445 to
download artifacts for a particular commit SHA, so that it can be passed
to the `test-deploy` script.

This will let us run deploy tests on PRs, ensuring that they're run from
the Next.js on the target branch rather than `canary`.

This waits until `test-new-tests-dev` and `test-new-tests-start`
complete for 2 reasons:
- No reason to waste deploy resources if the changed/added tests don't
even work in dev/start
- It gives time for deploy-tarball to finish

Sample run:
https://github.com/vercel/next.js/actions/runs/9865244781/job/27241944533?pr=67612#step:28:54

.github/workflows/build_and_deploy.yml

@@ -443,7 +443,7 @@ jobs:
           name: wasm-binaries-${{matrix.target}}
           path: packages/next-swc/crates/wasm/pkg-*
 
-  deployTarball:
+  deploy-tarball:
     if: ${{ needs.deploy-target.outputs.value != 'production' }}
     name: Deploy preview tarball
     runs-on: ubuntu-latest

.github/workflows/build_and_test.yml

@@ -313,6 +313,23 @@ jobs:
 
     secrets: inherit
 
+  test-new-tests-deploy:
+    name: Test new tests when deployed
+    needs: ['test-prod', 'test-new-tests-dev', 'test-new-tests-start']
+    if: ${{ needs.changes.outputs.docs-only == 'false' }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        group: [1/4, 2/4, 3/4, 4/4]
+
+    uses: ./.github/workflows/build_reusable.yml
+    with:
+      afterBuild: node scripts/test-new-tests.mjs --mode deploy --group ${{ matrix.group }}
+      stepName: 'test-new-tests-deploy-${{matrix.group}}'
+
+    secrets: inherit
+
   test-dev:
     name: test dev
     needs: ['changes', 'build-native', 'build-next']
@@ -490,6 +507,7 @@ jobs:
         'test-turbopack-integration',
         'test-new-tests-dev',
         'test-new-tests-start',
+        'test-new-tests-deploy',
         'test-turbopack-production',
         'test-turbopack-production-integration',
       ]

scripts/get-changed-tests.mjs

@@ -5,7 +5,8 @@ import path from 'path'
 
 /**
  * Detects changed tests files by comparing the current branch with `origin/canary`
- * Returns tests separated by test mode (dev/prod)
+ * Returns tests separated by test mode (dev/prod), as well as the corresponding commit hash
+ * that the current branch is pointing to
  */
 export default async function getChangedTests() {
   let eventData = {}
@@ -32,6 +33,11 @@ export default async function getChangedTests() {
     process.env.GITHUB_REPOSITORY ||
     (await execa('git remote get-url origin', EXECA_OPTS)).stdout
 
+  const commitSha =
+    eventData?.head?.sha ||
+    process.env.GITHUB_SHA ||
+    (await execa('git rev-parse HEAD', EXECA_OPTS)).stdout
+
   const isCanary =
     branchName.trim() === 'canary' && remoteUrl.includes('vercel/next.js')
 
@@ -60,6 +66,7 @@ export default async function getChangedTests() {
       branchName,
       remoteUrl,
       isCanary,
+      commitSha,
     },
     `\ngit diff:\n${changesResult.stderr}\n${changesResult.stdout}`
   )
@@ -102,5 +109,5 @@ export default async function getChangedTests() {
     )
   )
 
-  return { devTests, prodTests }
+  return { devTests, prodTests, commitSha }
 }

scripts/test-new-tests.mjs

@@ -40,7 +40,7 @@ async function main() {
   /** @type import('execa').Options */
   const EXECA_OPTS_STDIO = { ...EXECA_OPTS, stdio: 'inherit' }
 
-  const { devTests, prodTests } = await getChangedTests()
+  const { devTests, prodTests, commitSha } = await getChangedTests()
 
   let currentTests = testMode === 'dev' ? devTests : prodTests
 
@@ -79,13 +79,53 @@ async function main() {
 
   const RUN_TESTS_ARGS = ['run-tests.js', '-c', '1', '--retries', '0']
 
+  // Only override the test version for deploy tests, as they need to run against
+  // the artifacts for the pull request. Otherwise, we don't need to specify this property,
+  // as tests will run against the local version of Next.js
+  const nextTestVersion =
+    testMode === 'deploy'
+      ? `https://vercel-packages.vercel.app/next/commits/${commitSha}/next`
+      : undefined
+
+  if (nextTestVersion) {
+    console.log(`Verifying artifacts for commit ${commitSha}`)
+    // Attempt to fetch the deploy artifacts for the commit
+    // These might take a moment to become available, so we'll retry a few times
+    const fetchWithRetry = async (url, retries = 5, timeout = 5000) => {
+      for (let i = 0; i < retries; i++) {
+        const res = await fetch(url)
+        if (res.ok) {
+          return res
+        } else if (i < retries - 1) {
+          console.log(
+            `Attempt ${i + 1} failed. Retrying in ${timeout / 1000} seconds...`
+          )
+          await new Promise((resolve) => setTimeout(resolve, timeout))
+        } else {
+          throw new Error(
+            `Failed to verify artifacts for commit ${commitSha}: ${res.status}`
+          )
+        }
+      }
+    }
+
+    try {
+      await fetchWithRetry(nextTestVersion)
+      console.log(`Artifacts verified for commit ${commitSha}`)
+    } catch (error) {
+      console.error(error.message)
+      throw error
+    }
+  }
+
   for (let i = 0; i < attempts; i++) {
     console.log(`\n\nRun ${i + 1}/${attempts} for ${testMode} tests`)
     await execa('node', [...RUN_TESTS_ARGS, ...currentTests], {
       ...EXECA_OPTS_STDIO,
       env: {
         ...process.env,
         NEXT_TEST_MODE: testMode,
+        NEXT_TEST_VERSION: nextTestVersion,
       },
     })
   }