Remove glob package (vercel#6415)

We don't use a lot of the features of `glob`, so let's remove it in favor of a leaner approach using regex.

It's failing on windows and I have no idea why and don't own a windows machine 🤦🏼‍♂️

(Ignore some of the commits in here, I forgot to create the new branch before I started working)

bench/readdir/.gitignore

@@ -0,0 +1 @@
+fixtures

bench/readdir/create-fixtures.sh

@@ -0,0 +1,4 @@
+# Uses https://github.com/divmain/fuzzponent
+mkdir fixtures
+cd fixtures
+fuzzponent -d 2 -s 20

bench/readdir/glob.js

@@ -0,0 +1,23 @@
+const { join } = require('path')
+const { promisify } = require('util')
+const globMod = require('glob')
+const glob = promisify(globMod)
+const resolveDataDir = join(__dirname, 'fixtures', '**/*')
+
+async function test () {
+  const time = process.hrtime()
+  await glob(resolveDataDir)
+
+  const hrtime = process.hrtime(time)
+  const nanoseconds = (hrtime[0] * 1e9) + hrtime[1]
+  const milliseconds = nanoseconds / 1e6
+  console.log(milliseconds)
+}
+
+async function run () {
+  for (let i = 0; i < 50; i++) {
+    await test()
+  }
+}
+
+run()

bench/readdir/recursive-readdir.js

@@ -0,0 +1,21 @@
+const { join } = require('path')
+const { recursiveReadDir } = require('next/dist/lib/recursive-readdir')
+const resolveDataDir = join(__dirname, 'fixtures')
+
+async function test () {
+  const time = process.hrtime()
+  await recursiveReadDir(resolveDataDir, /\.js$/)
+
+  const hrtime = process.hrtime(time)
+  const nanoseconds = (hrtime[0] * 1e9) + hrtime[1]
+  const milliseconds = nanoseconds / 1e6
+  console.log(milliseconds)
+}
+
+async function run () {
+  for (let i = 0; i < 50; i++) {
+    await test()
+  }
+}
+
+run()

packages/next/build/entries.ts

@@ -9,7 +9,7 @@ type PagesMapping = {
 
 export function createPagesMapping(pagePaths: string[], extensions: string[]): PagesMapping {
   const pages: PagesMapping = pagePaths.reduce((result: PagesMapping, pagePath): PagesMapping => {
-    const page = `/${pagePath.replace(new RegExp(`\\.+(${extensions.join('|')})$`), '').replace(/\\/g, '/')}`.replace(/\/index$/, '')
+    const page = `${pagePath.replace(new RegExp(`\\.+(${extensions.join('|')})$`), '').replace(/\\/g, '/')}`.replace(/\/index$/, '')
     result[page === '' ? '/' : page] = join(PAGES_DIR_ALIAS, pagePath).replace(/\\/g, '/')
     return result
   }, {})

packages/next/build/index.ts

@@ -7,19 +7,16 @@ import { generateBuildId } from './generate-build-id'
 import { writeBuildId } from './write-build-id'
 import { isWriteable } from './is-writeable'
 import { runCompiler, CompilerResult } from './compiler'
-import globModule from 'glob'
-import { promisify } from 'util'
+import {recursiveReadDir} from '../lib/recursive-readdir'
 import { createPagesMapping, createEntrypoints } from './entries'
 import formatWebpackMessages from '../client/dev-error-overlay/format-webpack-messages'
 import chalk from 'chalk'
 
-const glob = promisify(globModule)
-
 function collectPages(
   directory: string,
   pageExtensions: string[]
 ): Promise<string[]> {
-  return glob(`**/*.+(${pageExtensions.join('|')})`, { cwd: directory })
+  return recursiveReadDir(directory, new RegExp(`\\.(?:${pageExtensions.join('|')})$`))
 }
 
 function printTreeView(list: string[]) {

packages/next/lib/recursive-readdir.ts

@@ -0,0 +1,36 @@
+import fs from 'fs'
+import { join } from 'path'
+import { promisify } from 'util'
+
+const readdir = promisify(fs.readdir)
+const stat = promisify(fs.stat)
+
+/**
+ * Recursively read directory
+ * @param  {string} dir Directory to read
+ * @param  {RegExp} filter Filter for the file name, only the name part is considered, not the full path
+ * @param  {string[]=[]} arr This doesn't have to be provided, it's used for the recursion
+ * @param  {string=dir`} rootDir Used to replace the initial path, only the relative path is left, it's faster than path.relative.
+ * @returns Promise array holding all relative paths
+ */
+export async function recursiveReadDir(dir: string, filter: RegExp, arr: string[] = [], rootDir: string = dir): Promise<string[]> {
+  const result = await readdir(dir)
+
+  await Promise.all(result.map(async (part: string) => {
+    const absolutePath = join(dir, part)
+    const pathStat = await stat(absolutePath)
+
+    if (pathStat.isDirectory()) {
+      await recursiveReadDir(absolutePath, filter, arr, rootDir)
+      return
+    }
+
+    if (!filter.test(part)) {
+      return
+    }
+
+    arr.push(absolutePath.replace(rootDir, ''))
+  }))
+
+  return arr
+}

packages/next/package.json

@@ -64,7 +64,6 @@
     "find-cache-dir": "2.0.0",
     "find-up": "2.1.0",
     "fresh": "0.5.2",
-    "glob": "7.1.2",
     "hoist-non-react-statics": "3.2.0",
     "launch-editor": "2.2.1",
     "loader-utils": "1.1.0",
@@ -111,7 +110,6 @@
     "@types/cross-spawn": "6.0.0",
     "@types/etag": "1.8.0",
     "@types/fresh": "0.5.0",
-    "@types/glob": "7.1.1",
     "@types/loader-utils": "1.1.3",
     "@types/mkdirp": "0.5.2",
     "@types/nanoid": "1.2.0",

packages/next/server/hot-reloader.js

@@ -8,12 +8,11 @@ import webpack from 'webpack'
 import getBaseWebpackConfig from '../build/webpack-config'
 import { IS_BUNDLED_PAGE_REGEX, ROUTE_NAME_REGEX, BLOCKED_PAGES } from 'next-server/constants'
 import { route } from 'next-server/dist/server/router'
-import globModule from 'glob'
 import { promisify } from 'util'
 import { createPagesMapping, createEntrypoints } from '../build/entries'
 import { watchCompiler } from '../build/output'
+import { findPageFile } from './lib/find-page-file'
 
-const glob = promisify(globModule)
 const rimraf = promisify(rimrafModule)
 
 export async function renderScriptError (res, error) {
@@ -166,8 +165,13 @@ export default class HotReloader {
   }
 
   async getWebpackConfig () {
-    const pagePaths = await glob(`+(_app|_document).+(${this.config.pageExtensions.join('|')})`, { cwd: join(this.dir, 'pages') })
-    const pages = createPagesMapping(pagePaths, this.config.pageExtensions)
+    const pagesDir = join(this.dir, 'pages')
+    const pagePaths = await Promise.all([
+      findPageFile(pagesDir, '/_app', this.config.pageExtensions),
+      findPageFile(pagesDir, '/_document', this.config.pageExtensions)
+    ])
+
+    const pages = createPagesMapping(pagePaths.filter(i => i !== null), this.config.pageExtensions)
     const entrypoints = createEntrypoints(pages, 'server', this.buildId, this.config)
     return [
       getBaseWebpackConfig(this.dir, { dev: true, isServer: false, config: this.config, buildId: this.buildId, entrypoints: entrypoints.client }),

packages/next/server/lib/find-page-file.ts

@@ -0,0 +1,21 @@
+import { join } from 'path'
+import {isWriteable} from '../../build/is-writeable'
+
+export async function findPageFile(rootDir: string, normalizedPagePath: string, pageExtensions: string[]): Promise<string|null> {
+  for (const extension of pageExtensions) {
+    const relativePagePath = `${normalizedPagePath}.${extension}`
+    const pagePath = join(rootDir, relativePagePath)
+
+    if (await isWriteable(pagePath)) {
+      return relativePagePath
+    }
+
+    const relativePagePathWithIndex = join(normalizedPagePath, `index.${extension}`)
+    const pagePathWithIndex = join(rootDir, relativePagePathWithIndex)
+    if (await isWriteable(pagePathWithIndex)) {
+      return relativePagePathWithIndex
+    }
+  }
+
+  return null
+}

packages/next/server/on-demand-entry-handler.js

@@ -2,21 +2,17 @@ import DynamicEntryPlugin from 'webpack/lib/DynamicEntryPlugin'
 import { EventEmitter } from 'events'
 import { join } from 'path'
 import { parse } from 'url'
-import fs from 'fs'
-import { promisify } from 'util'
-import globModule from 'glob'
 import { pageNotFoundError } from 'next-server/dist/server/require'
 import { normalizePagePath } from 'next-server/dist/server/normalize-page-path'
 import { ROUTE_NAME_REGEX, IS_BUNDLED_PAGE_REGEX } from 'next-server/constants'
 import { stringify } from 'querystring'
+import { findPageFile } from './lib/find-page-file'
+import { isWriteable } from '../build/is-writeable'
 
 const ADDED = Symbol('added')
 const BUILDING = Symbol('building')
 const BUILT = Symbol('built')
 
-const glob = promisify(globModule)
-const access = promisify(fs.access)
-
 // Based on https://github.com/webpack/webpack/blob/master/lib/DynamicEntryPlugin.js#L29-L37
 function addEntry (compilation, context, name, entry) {
   return new Promise((resolve, reject) => {
@@ -36,6 +32,7 @@ export default function onDemandEntryHandler (devMiddleware, multiCompiler, {
   maxInactiveAge,
   pagesBufferLength
 }) {
+  const pagesDir = join(dir, 'pages')
   const clients = new Set()
   const evtSourceHeaders = {
     'Access-Control-Allow-Origin': '*',
@@ -61,9 +58,8 @@ export default function onDemandEntryHandler (devMiddleware, multiCompiler, {
 
       const allEntries = Object.keys(entries).map(async (page) => {
         const { name, absolutePagePath } = entries[page]
-        try {
-          await access(absolutePagePath, (fs.constants || fs).W_OK)
-        } catch (err) {
+        const pageExists = await isWriteable(absolutePagePath)
+        if (!pageExists) {
           console.warn('Page was removed', page)
           delete entries[page]
           return
@@ -220,22 +216,18 @@ export default function onDemandEntryHandler (devMiddleware, multiCompiler, {
         throw pageNotFoundError(normalizedPagePath)
       }
 
-      const extensions = pageExtensions.join('|')
-      const pagesDir = join(dir, 'pages')
-
-      let paths = await glob(`{${normalizedPagePath.slice(1)}/index,${normalizedPagePath.slice(1)}}.+(${extensions})`, { cwd: pagesDir })
+      let pagePath = await findPageFile(pagesDir, normalizedPagePath, pageExtensions)
 
       // Default the /_error route to the Next.js provided default page
-      if (page === '/_error' && paths.length === 0) {
-        paths = ['next/dist/pages/_error']
+      if (page === '/_error' && pagePath === null) {
+        pagePath = 'next/dist/pages/_error'
       }
 
-      if (paths.length === 0) {
+      if (pagePath === null) {
         throw pageNotFoundError(normalizedPagePath)
       }
 
-      const pagePath = paths[0]
-      let pageUrl = `/${pagePath.replace(new RegExp(`\\.+(${extensions})$`), '').replace(/\\/g, '/')}`.replace(/\/index$/, '')
+      let pageUrl = `/${pagePath.replace(new RegExp(`\\.+(?:${pageExtensions.join('|')})$`), '').replace(/\\/g, '/')}`.replace(/\/index$/, '')
       pageUrl = pageUrl === '' ? '/' : pageUrl
       const bundleFile = pageUrl === '/' ? '/index.js' : `${pageUrl}.js`
       const name = join('static', buildId, 'pages', bundleFile)

test/unit/find-page-file.test.js

@@ -0,0 +1,28 @@
+/* eslint-env jest */
+import { findPageFile } from 'next/dist/server/lib/find-page-file'
+import { normalizePagePath } from 'next-server/dist/server/normalize-page-path'
+
+import { join } from 'path'
+
+const resolveDataDir = join(__dirname, '..', 'isolated', '_resolvedata')
+const dirWithPages = join(resolveDataDir, 'readdir', 'pages')
+
+describe('findPageFile', () => {
+  it('should work', async () => {
+    const pagePath = normalizePagePath('/nav/about')
+    const result = await findPageFile(dirWithPages, pagePath, ['jsx', 'js'])
+    expect(result).toMatch(/^[\\/]nav[\\/]about\.js/)
+  })
+
+  it('should work with nested index.js', async () => {
+    const pagePath = normalizePagePath('/nested')
+    const result = await findPageFile(dirWithPages, pagePath, ['jsx', 'js'])
+    expect(result).toMatch(/^[\\/]nested[\\/]index\.js/)
+  })
+
+  it('should prefer prefered.js before prefered/index.js', async () => {
+    const pagePath = normalizePagePath('/prefered')
+    const result = await findPageFile(dirWithPages, pagePath, ['jsx', 'js'])
+    expect(result).toMatch(/^[\\/]prefered\.js/)
+  })
+})

test/unit/recursive-readdir.test.js

@@ -0,0 +1,22 @@
+/* eslint-env jest */
+import { recursiveReadDir } from 'next/dist/lib/recursive-readdir'
+import { join } from 'path'
+
+const resolveDataDir = join(__dirname, '..', 'isolated', '_resolvedata')
+const dirWithPages = join(resolveDataDir, 'readdir', 'pages')
+
+describe('recursiveReadDir', () => {
+  it('should work', async () => {
+    const result = await recursiveReadDir(dirWithPages, /\.js/)
+    const pages = [/^[\\/]index\.js/, /^[\\/]prefered\.js/, /^[\\/]nav[\\/]about\.js/, /^[\\/]nav[\\/]index\.js/, /^[\\/]nested[\\/]index\.js/, /^[\\/]prefered[\\/]index\.js/, /^[\\/]nav[\\/]products[\\/]product\.js/]
+    expect(result.filter((item) => {
+      for (const page of pages) {
+        if (page.test(item)) {
+          return false
+        }
+      }
+
+      return true
+    }).length).toBe(0)
+  })
+})

yarn.lock

@@ -1550,7 +1550,7 @@
   resolved "https://registry.yarnpkg.com/@types/fresh/-/fresh-0.5.0.tgz#4d09231027d69c4369cfb01a9af5ef083d0d285f"
   integrity sha512-eGPzuyc6wZM3sSHJdF7NM2jW6B/xsB014Rqg/iDa6xY02mlfy1w/TE2sYhR8vbHxkzJOXiGo6NuIk3xk35vsgQ==
 
-"@types/glob@*", "@types/[email protected]":
+"@types/glob@*":
   version "7.1.1"
   resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.1.1.tgz#aa59a1c6e3fbc421e07ccd31a944c30eba521575"
   integrity sha512-1Bh06cbWJUHMC97acuD6UMG29nMt0Aqz1vF3guLfG+kHHJhy3AyohZFFxYk2f7Q1SQIrNwvncxAE0N/9s70F2w==
@@ -1599,9 +1599,9 @@
     "@types/node" "*"
 
 "@types/node@*":
-  version "11.9.4"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-11.9.4.tgz#ceb0048a546db453f6248f2d1d95e937a6f00a14"
-  integrity sha512-Zl8dGvAcEmadgs1tmSPcvwzO1YRsz38bVJQvH1RvRqSR9/5n61Q1ktcDL0ht3FXWR+ZpVmXVwN1LuH4Ax23NsA==
+  version "11.9.5"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-11.9.5.tgz#011eece9d3f839a806b63973e228f85967b79ed3"
+  integrity sha512-vVjM0SVzgaOUpflq4GYBvCpozes8OgIIS5gVXVka+OfK3hvnkC1i93U8WiY2OtNE4XUWyyy/86Kf6e0IHTQw1Q==
 
 "@types/prop-types@*":
   version "15.5.9"
@@ -1981,9 +1981,9 @@ ajv@^5.1.0:
     json-schema-traverse "^0.3.0"
 
 ajv@^6.0.1, ajv@^6.1.0, ajv@^6.5.0, ajv@^6.5.5:
-  version "6.9.1"
-  resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.9.1.tgz#a4d3683d74abc5670e75f0b16520f70a20ea8dc1"
-  integrity sha512-XDN92U311aINL77ieWHmqCcNlwjoP5cHXDxIxbf2MaPYuCXOHS7gHH8jktxeK5omgd52XbSTX6a4Piwd1pQmzA==
+  version "6.9.2"
+  resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.9.2.tgz#4927adb83e7f48e5a32b45729744c71ec39c9c7b"
+  integrity sha512-4UFy0/LgDo7Oa/+wOAlj44tp9K78u38E5/359eSrqEp1Z5PdVfimCcs7SluXMP755RUQu6d2b4AvF0R1C9RZjg==
   dependencies:
     fast-deep-equal "^2.0.1"
     fast-json-stable-stringify "^2.0.0"
@@ -5679,18 +5679,6 @@ glob-to-regexp@^0.3.0:
   resolved "https://registry.yarnpkg.com/glob-to-regexp/-/glob-to-regexp-0.3.0.tgz#8c5a1494d2066c570cc3bfe4496175acc4d502ab"
   integrity sha1-jFoUlNIGbFcMw7/kSWF1rMTVAqs=
 
-[email protected]:
-  version "7.1.2"
-  resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15"
-  integrity sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==
-  dependencies:
-    fs.realpath "^1.0.0"
-    inflight "^1.0.4"
-    inherits "2"
-    minimatch "^3.0.4"
-    once "^1.3.0"
-    path-is-absolute "^1.0.0"
-
 glob@^7.0.0, glob@^7.0.3, glob@^7.0.5, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@~7.1.1:
   version "7.1.3"
   resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.3.tgz#3960832d3f1574108342dafd3a67b332c0969df1"