Speed up SSG prefetching (vercel#18813)

This pull request speeds up Next.js' rendering pipeline by fetching data, parsing it, and loading it into memory instead of only doing the network request.

This will mainly result in improved Firefox/Safari performance since they handled prefetch incorrectly—only Chrome did it right. This also gets us closer to being able to use `no-store` in our caching headers!

---

Fixes vercel#18639
x-ref vercel#18802

packages/next/client/page-loader.ts

@@ -217,25 +217,12 @@ export default class PageLoader {
 
   /**
    * @param {string} href the route href (file-system path)
-   * @param {string} asPath the URL as shown in browser (virtual path); used for dynamic routes
    */
-  prefetchData(href: string, asPath: string, locale?: string | false) {
+  _isSsg(href: string): Promise<boolean> {
     const { pathname: hrefPathname } = parseRelativeUrl(href)
     const route = normalizeRoute(hrefPathname)
-    return this.promisedSsgManifest!.then(
-      (s: ClientSsgManifest, _dataHref?: string) =>
-        // Check if the route requires a data file
-        s.has(route) &&
-        // Try to generate data href, noop when falsy
-        (_dataHref = this.getDataHref(href, asPath, true, locale)) &&
-        // noop when data has already been prefetched (dedupe)
-        !document.querySelector(
-          `link[rel="${relPrefetch}"][href^="${_dataHref}"]`
-        ) &&
-        // Inject the `<link rel=prefetch>` tag for above computed `href`.
-        appendLink(_dataHref, relPrefetch, 'fetch').catch(() => {
-          /* ignore prefetch error */
-        })
+    return this.promisedSsgManifest!.then((s: ClientSsgManifest) =>
+      s.has(route)
     )
   }
 

packages/next/next-server/lib/router/router.ts

@@ -1218,12 +1218,20 @@ export default class Router implements BaseRouter {
 
     const route = removePathTrailingSlash(pathname)
     await Promise.all([
-      this.pageLoader.prefetchData(
-        url,
-        asPath,
-        typeof options.locale !== 'undefined' ? options.locale : this.locale,
-        this.defaultLocale
-      ),
+      this.pageLoader._isSsg(url).then((isSsg: boolean) => {
+        return isSsg
+          ? this._getStaticData(
+              this.pageLoader.getDataHref(
+                url,
+                asPath,
+                true,
+                typeof options.locale !== 'undefined'
+                  ? options.locale
+                  : this.locale
+              )
+            )
+          : false
+      }),
       this.pageLoader[options.priority ? 'loadPage' : 'prefetch'](route),
     ])
   }

test/integration/basepath/test/index.test.js

@@ -1,32 +1,33 @@
 /* eslint-env jest */
 
-import webdriver from 'next-webdriver'
-import { join, resolve } from 'path'
-import url from 'url'
+import assert from 'assert'
+import cheerio from 'cheerio'
+import fs, {
+  existsSync,
+  readFileSync,
+  renameSync,
+  writeFileSync,
+} from 'fs-extra'
 import {
-  launchApp,
-  findPort,
-  killApp,
-  nextBuild,
-  waitFor,
   check,
-  getBrowserBodyText,
-  renderViaHTTP,
+  fetchViaHTTP,
   File,
-  nextStart,
-  initNextServerScript,
+  findPort,
+  getBrowserBodyText,
   getRedboxSource,
   hasRedbox,
-  fetchViaHTTP,
+  initNextServerScript,
+  killApp,
+  launchApp,
+  nextBuild,
+  nextStart,
+  renderViaHTTP,
   startStaticServer,
+  waitFor,
 } from 'next-test-utils'
-import fs, {
-  readFileSync,
-  writeFileSync,
-  renameSync,
-  existsSync,
-} from 'fs-extra'
-import cheerio from 'cheerio'
+import webdriver from 'next-webdriver'
+import { join, resolve } from 'path'
+import url from 'url'
 
 jest.setTimeout(1000 * 60 * 2)
 
@@ -427,28 +428,35 @@ const runTests = (dev = false) => {
       const browser = await webdriver(appPort, `${basePath}/hello`)
       await browser.eval('window.next.router.prefetch("/gssp")')
 
-      await check(
-        async () => {
-          const links = await browser.elementsByCss('link[rel=prefetch]')
-          let found = new Set()
-
-          for (const link of links) {
-            const href = await link.getAttribute('href')
-            if (href.match(/(gsp|gssp|other-page)-.*?\.js$/)) {
-              found.add(href)
-            }
-            if (href.match(/gsp\.json$/)) {
-              found.add(href)
-            }
-          }
-          return found
-        },
-        {
-          test(result) {
-            return result.size === 4
-          },
-        }
-      )
+      await check(async () => {
+        const hrefs = await browser.eval(`Object.keys(window.next.router.sdc)`)
+        hrefs.sort()
+
+        assert.deepEqual(
+          hrefs.map((href) =>
+            new URL(href).pathname.replace(/\/_next\/data\/[^/]+/, '')
+          ),
+          [
+            `${basePath}/gsp.json`,
+            `${basePath}/index.json`,
+            `${basePath}/index/index.json`,
+          ]
+        )
+
+        const prefetches = await browser.eval(
+          `[].slice.call(document.querySelectorAll("link[rel=prefetch]")).map((e) => new URL(e.href).pathname)`
+        )
+        expect(prefetches).toContainEqual(
+          expect.stringMatching(/\/gsp-[^./]+\.js/)
+        )
+        expect(prefetches).toContainEqual(
+          expect.stringMatching(/\/gssp-[^./]+\.js/)
+        )
+        expect(prefetches).toContainEqual(
+          expect.stringMatching(/\/other-page-[^./]+\.js/)
+        )
+        return 'yes'
+      }, 'yes')
     })
   }
 
@@ -602,23 +610,18 @@ const runTests = (dev = false) => {
     expect(await browser.elementByCss('#pathname').text()).toBe('/')
 
     if (!dev) {
-      const prefetches = await browser.elementsByCss('link[rel="prefetch"]')
-      let found = false
-
-      for (const prefetch of prefetches) {
-        const fullHref = await prefetch.getAttribute('href')
-        const href = url.parse(fullHref).pathname
-
-        if (
-          href.startsWith(`${basePath}/_next/data`) &&
-          href.endsWith('index.json') &&
-          !href.endsWith('index/index.json')
-        ) {
-          found = true
-        }
-      }
-
-      expect(found).toBe(true)
+      const hrefs = await browser.eval(`Object.keys(window.next.router.sdc)`)
+      hrefs.sort()
+
+      expect(
+        hrefs.map((href) =>
+          new URL(href).pathname.replace(/\/_next\/data\/[^/]+/, '')
+        )
+      ).toEqual([
+        `${basePath}/gsp.json`,
+        `${basePath}/index.json`,
+        `${basePath}/index/index.json`,
+      ])
     }
   })
 
@@ -636,22 +639,18 @@ const runTests = (dev = false) => {
     expect(await browser.elementByCss('#pathname').text()).toBe('/index')
 
     if (!dev) {
-      const prefetches = await browser.elementsByCss('link[rel="prefetch"]')
-      let found = false
-
-      for (const prefetch of prefetches) {
-        const fullHref = await prefetch.getAttribute('href')
-        const href = url.parse(fullHref).pathname
-
-        if (
-          href.startsWith(`${basePath}/_next/data`) &&
-          href.endsWith('index/index.json')
-        ) {
-          found = true
-        }
-      }
-
-      expect(found).toBe(true)
+      const hrefs = await browser.eval(`Object.keys(window.next.router.sdc)`)
+      hrefs.sort()
+
+      expect(
+        hrefs.map((href) =>
+          new URL(href).pathname.replace(/\/_next\/data\/[^/]+/, '')
+        )
+      ).toEqual([
+        `${basePath}/gsp.json`,
+        `${basePath}/index.json`,
+        `${basePath}/index/index.json`,
+      ])
     }
   })
 

test/integration/i18n-support/test/index.test.js

@@ -7,6 +7,7 @@ import cheerio from 'cheerio'
 import { join } from 'path'
 import webdriver from 'next-webdriver'
 import escapeRegex from 'escape-string-regexp'
+import assert from 'assert'
 import {
   fetchViaHTTP,
   findPort,
@@ -312,23 +313,20 @@ function runTests(isDev) {
       })()`)
 
       await check(async () => {
-        for (const dataPath of [
-          '/fr/gsp.json',
-          '/fr/gsp/fallback/first.json',
-          '/fr/gsp/fallback/hello.json',
-        ]) {
-          const found = await browser.eval(`(function() {
-            const links = [].slice.call(document.querySelectorAll('link'))
-
-            for (var i = 0; i < links.length; i++) {
-              if (links[i].href.indexOf("${dataPath}") > -1) {
-                return true
-              }
-            }
-            return false
-          })()`)
-          return found ? 'yes' : 'no'
-        }
+        const hrefs = await browser.eval(`Object.keys(window.next.router.sdc)`)
+        hrefs.sort()
+
+        assert.deepEqual(
+          hrefs.map((href) =>
+            new URL(href).pathname.replace(/^\/_next\/data\/[^/]+/, '')
+          ),
+          [
+            '/fr/gsp.json',
+            '/fr/gsp/fallback/first.json',
+            '/fr/gsp/fallback/hello.json',
+          ]
+        )
+        return 'yes'
       }, 'yes')
     }
 
@@ -518,23 +516,20 @@ function runTests(isDev) {
       })()`)
 
       await check(async () => {
-        for (const dataPath of [
-          '/fr/gsp.json',
-          '/fr/gsp/fallback/first.json',
-          '/fr/gsp/fallback/hello.json',
-        ]) {
-          const found = await browser.eval(`(function() {
-            const links = [].slice.call(document.querySelectorAll('link'))
-
-            for (var i = 0; i < links.length; i++) {
-              if (links[i].href.indexOf("${dataPath}") > -1) {
-                return true
-              }
-            }
-            return false
-          })()`)
-          return found ? 'yes' : 'no'
-        }
+        const hrefs = await browser.eval(`Object.keys(window.next.router.sdc)`)
+        hrefs.sort()
+
+        assert.deepEqual(
+          hrefs.map((href) =>
+            new URL(href).pathname.replace(/^\/_next\/data\/[^/]+/, '')
+          ),
+          [
+            '/fr/gsp.json',
+            '/fr/gsp/fallback/first.json',
+            '/fr/gsp/fallback/hello.json',
+          ]
+        )
+        return 'yes'
       }, 'yes')
     }