[1.0] optimizations around prefetching page resources (gatsbyjs#1133)

* Copy cache-dir files into site's .cache

* Got new loader/componentrenderer working in development

* Use ComponentRenderer for rendering

* On demand loading of async page resources working for prod builds

* Use script loader so loading scripts don't block rendering

* Add gatsby-module-loader to replace bundle-loader to add ability to delay executing scripts after loading

* Split out loading/executing resources into functions

* Add prefetcher

* changing order to see if it fixes not removing event handler

* Just don't have a loader for now

* Updates to service worker plugin

* Add tests for find-page + make it work with prefixed links

* Proritize links that mount first i.e. are higher on the page

* Wait for all links to mount before fetching so resource count is correct

* Add plop template for adding new example sites

* Add plugin for nprogress to auto show when page loading is delayed

* Don't break layout if there's no content

* Log at end of build how long build took

* Not using this package

* Remove logging + run format

* Add the nprogress plugin

* Fix some emitter code

* Make it easy to override the default color of the nprogress bar

* Fix check on variable

* Fix prefetching

* Fix test

* Add nprogress plugin to list

docs/docs/plugins.md

@@ -47,6 +47,7 @@ on using each plugin.
 * [gatsby-plugin-glamor](/docs/packages/gatsby-plugin-glamor/)
 * [gatsby-plugin-google-analytics](/docs/packages/gatsby-plugin-google-analytics/)
 * [gatsby-plugin-manifest](/docs/packages/gatsby-plugin-manifest/)
+* [gatsby-plugin-nprogress](/docs/packages/gatsby-plugin-nprogress/)
 * [gatsby-plugin-offline](/docs/packages/gatsby-plugin-offline/)
 * [gatsby-plugin-preact](/docs/packages/gatsby-plugin-preact/)
 * [gatsby-plugin-react-helmet](/docs/packages/gatsby-plugin-react-helmet/)

examples/feed/gatsby-node.js

@@ -1,25 +1,27 @@
-const path = require('path')
+const path = require(`path`)
 
 exports.onCreateNode = ({ node, boundActionCreators, getNode }) => {
   const { createNodeField } = boundActionCreators
 
-  if (node.internal.type === 'MarkdownRemark') {
+  if (node.internal.type === `MarkdownRemark`) {
     const fileNode = getNode(node.parent)
-    let nodeSlug 
-    nodeSlug = ensureSlashes(path.basename(fileNode.relativePath, path.extname(fileNode.relativePath)))
+    let nodeSlug
+    nodeSlug = ensureSlashes(
+      path.basename(fileNode.relativePath, path.extname(fileNode.relativePath))
+    )
     if (nodeSlug) {
-      createNodeField({ node, fieldName: 'slug', fieldValue: nodeSlug })
+      createNodeField({ node, fieldName: `slug`, fieldValue: nodeSlug })
     }
   }
 }
 
 function ensureSlashes(slug) {
-  if (slug.charAt(0) !== '/') {
-    slug = '/' + slug
+  if (slug.charAt(0) !== `/`) {
+    slug = `/` + slug
   }
 
-  if (slug.charAt(slug.length -1) !== '/') {
-    slug = slug + '/'
+  if (slug.charAt(slug.length - 1) !== `/`) {
+    slug = slug + `/`
   }
 
   return slug

examples/feed/src/layouts/index.js

@@ -1,4 +1,4 @@
-import React from 'react'
+import React from "react"
 
 export default class DefaultLayout extends React.Component {
   render() {

examples/feed/src/pages/index.js

@@ -1,14 +1,12 @@
-import React from 'react'
+import React from "react"
 
-const IndexRoute = () => (
+const IndexRoute = () =>
   <div>
     <p>
       Welcome to the GatsbyJS RSS Demo.
       <a href="/rss.xml">Click here</a>
       to see the generated RSS Feed.
     </p>
   </div>
-)
-
 
 export default IndexRoute

examples/redux/src/layouts/index.js

@@ -1,14 +1,13 @@
-import React from 'react'
-import PropTypes from 'prop-types'
-import Link from 'gatsby-link'
-import { connect } from 'react-redux'
+import React from "react"
+import PropTypes from "prop-types"
+import Link from "gatsby-link"
+import { connect } from "react-redux"
 
-const Counter = ({ count, increment }) => (
+const Counter = ({ count, increment }) =>
   <div>
     <p>Count: {count}</p>
     <button onClick={increment}>Increment</button>
   </div>
-)
 
 Counter.propTypes = {
   count: PropTypes.number.isRequired,
@@ -19,7 +18,7 @@ const mapStateToProps = ({ count }) => {
   return { count }
 }
 
-const mapDispatchToProps = (dispatch) => {
+const mapDispatchToProps = dispatch => {
   return { increment: () => dispatch({ type: `INCREMENT` }) }
 }
 
@@ -34,7 +33,7 @@ class DefaultLayout extends React.Component {
             Redux example
           </h3>
         </Link>
-        <ConnectedCounter/>
+        <ConnectedCounter />
         <ul>
           <li><Link to="/a/">a</Link></li>
           <li><Link to="/b/">b</Link></li>

examples/redux/src/state/createStore.js

@@ -1,12 +1,12 @@
-import { createStore as reduxCreateStore } from 'redux'
+import { createStore as reduxCreateStore } from "redux"
 
 const reducer = (state, action) => {
-    if (action.type === `INCREMENT`) {
-        return Object.assign({}, state, {
-            count: state.count + 1,
-        })
-    }
-    return state
+  if (action.type === `INCREMENT`) {
+    return Object.assign({}, state, {
+      count: state.count + 1,
+    })
+  }
+  return state
 }
 
 const initialState = { count: 0 }

examples/sitemap/gatsby-node.js

@@ -1,25 +1,27 @@
-const path = require('path')
+const path = require(`path`)
 
 exports.onCreateNode = ({ node, boundActionCreators, getNode }) => {
   const { createNodeField } = boundActionCreators
 
-  if (node.internal.type === 'MarkdownRemark') {
+  if (node.internal.type === `MarkdownRemark`) {
     const fileNode = getNode(node.parent)
-    let nodeSlug 
-    nodeSlug = ensureSlashes(path.basename(fileNode.relativePath, path.extname(fileNode.relativePath)))
+    let nodeSlug
+    nodeSlug = ensureSlashes(
+      path.basename(fileNode.relativePath, path.extname(fileNode.relativePath))
+    )
     if (nodeSlug) {
-      createNodeField({ node, fieldName: 'slug', fieldValue: nodeSlug })
+      createNodeField({ node, fieldName: `slug`, fieldValue: nodeSlug })
     }
   }
 }
 
 function ensureSlashes(slug) {
-  if (slug.charAt(0) !== '/') {
-    slug = '/' + slug
+  if (slug.charAt(0) !== `/`) {
+    slug = `/` + slug
   }
 
-  if (slug.charAt(slug.length -1) !== '/') {
-    slug = slug + '/'
+  if (slug.charAt(slug.length - 1) !== `/`) {
+    slug = slug + `/`
   }
 
   return slug

examples/sitemap/src/layouts/index.js

@@ -1,4 +1,4 @@
-import React from 'react'
+import React from "react"
 
 export default class DefaultLayout extends React.Component {
   render() {

examples/sitemap/src/pages/index.js

@@ -1,13 +1,11 @@
-import React from 'react'
+import React from "react"
 
-const IndexRoute = () => (
+const IndexRoute = () =>
   <div>
     <p>
       Welcome to the GatsbyJS Sitemap Demo.
       Visit <a href="/sitemap.xml">to see the generated sitemap.</a>
     </p>
   </div>
-)
-
 
 export default IndexRoute

examples/using-page-loading-indicator/.eslintrc

@@ -0,0 +1,8 @@
+{
+  "env": {
+    "browser": true
+  },
+  "globals": {
+    "graphql": false
+  }
+}

examples/using-page-loading-indicator/.gitignore

@@ -0,0 +1,3 @@
+public
+.cache
+node_modules

examples/using-page-loading-indicator/README.md

@@ -0,0 +1,15 @@
+# using-page-loading-indicator
+
+https://using-page-loading-indicator.gatsbyjs.org
+
+The loading indicator only will show up in the production version of the site.
+
+So first run `gatsby build` then `gatsby serve`.
+
+Then you'll need to open up the chrome (or equivalent in other browsers)
+devtools and go to the Network tab. There where it says "No throttling" along the
+top, click that and use the "Slow 3G" preset (or an equivalent slow speed).
+
+Then open the built site (generally at localhost:9000) and (important) wait until
+the initial four JavaScript files are downloaded then click on the "Page 2" link
+and after a second, you'll see the nprogress bar along the top come in.

examples/using-page-loading-indicator/gatsby-config.js

@@ -0,0 +1,3 @@
+module.exports = {
+  plugins: [`gatsby-plugin-nprogress`],
+}

examples/using-page-loading-indicator/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "using-page-loading-indicator",
+  "private": true,
+  "description": "Gatsby example site using using-page-loading-indicator",
+  "author": "Kyle Mathews<[email protected]>",
+  "dependencies": {
+    "gatsby": "canary",
+    "gatsby-link": "canary",
+    "gatsby-plugin-nprogress": "^0.2.0"
+  },
+  "license": "MIT",
+  "main": "n/a",
+  "scripts": {
+    "develop": "gatsby develop",
+    "build": "gatsby build"
+  }
+}

examples/using-page-loading-indicator/src/pages/index.js

@@ -0,0 +1,15 @@
+import React from "react"
+import Link from "gatsby-link"
+
+class IndexComponent extends React.Component {
+  render() {
+    return (
+      <div>
+        <h1>Hello world</h1>
+        <Link to="/page-2/">Page 2</Link>
+      </div>
+    )
+  }
+}
+
+export default IndexComponent

examples/using-page-loading-indicator/src/pages/page-2.js

@@ -0,0 +1,3 @@
+import React from "react"
+
+export default () => <div>Page 2</div>

packages/gatsby-dev-cli/src/watch.js

@@ -10,9 +10,22 @@ const debouncedQuit = _.debounce(() => {
   process.exit()
 }, 500)
 
+const copyPath = (oldPath, newPath, quiet) => {
+  fs.copy(oldPath, newPath, err => {
+    if (err) {
+      return console.error(err)
+    }
+
+    numCopied += 1
+    if (!quiet) {
+      console.log(`Copied ${oldPath} to ${newPath}`)
+    }
+  })
+}
+
 function watch(root, packages, { scanOnce, quiet }) {
   packages.forEach(p => {
-    const prefix = `${root}/packages/${p}`
+    const prefix = syspath.join(root, `/packages/`, p)
 
     const ignoreRegs = [
       /[\/\\]node_modules[\/\\]/i,
@@ -37,16 +50,17 @@ function watch(root, packages, { scanOnce, quiet }) {
             `./node_modules/${p}`,
             syspath.relative(prefix, path)
           )
-          fs.copy(path, newPath, err => {
-            if (err) {
-              return console.error(err)
-            }
-
-            numCopied += 1
-            if (!quiet) {
-              console.log(`Copied ${path} to ${newPath}`)
-            }
-          })
+
+          copyPath(path, newPath, quiet)
+
+          // If this is from "cache-dir" also copy it into the site's .cache
+          if (_.includes(path, `dist/cache-dir`)) {
+            const newCachePath = syspath.join(
+              `.cache/`,
+              syspath.relative(syspath.join(prefix, `dist`, `cache-dir`), path)
+            )
+            copyPath(path, newCachePath, quiet)
+          }
 
           if (scanOnce) {
             debouncedQuit()

packages/gatsby-link/src/index.js

@@ -2,10 +2,6 @@ import React from "react"
 import { Link } from "react-router-dom"
 import PropTypes from "prop-types"
 
-if (typeof window !== `undefined`) {
-  require(`ric`)
-}
-
 let linkPrefix = ``
 if (__PREFIX_LINKS__) {
   linkPrefix = __LINK_PREFIX__
@@ -17,20 +13,7 @@ class GatsbyLink extends React.Component {
     onClick: PropTypes.func,
   }
   componentDidMount() {
-    // Only enable prefetching of Link resources in production and for browsers
-    // that don't support service workers *cough* Safari/IE *cough*.
-    //
-    // TODO also add check if user is using SW, e.g. window.caches as if
-    // not we should preload here too.
-    if (
-      process.env.NODE_ENV === `production` &&
-      (!(`serviceWorker` in window.navigator) ||
-        window.location.protocol !== `https:`)
-    ) {
-      requestUserIdle(() => {
-        ___loadScriptsForPath(this.props.to)
-      })
-    }
+    ___loader.enqueue(this.props.to)
   }
 
   render() {

packages/gatsby-plugin-feed/src/gatsby-node.js

@@ -1,20 +1,19 @@
-import path from 'path'
-import { defaultOptions, runQuery, writeFile } from './internals'
+import path from "path"
+import { defaultOptions, runQuery, writeFile } from "./internals"
 
-const publicPath = './public'
+const publicPath = `./public`
 
 // A default function to transform query data into feed entries.
-const serialize = ({ site, allMarkdownRemark }) => (
-  allMarkdownRemark.edges.map(edge => ({
-    ...edge.node.frontmatter,
-    description: edge.node.excerpt,
-    url: site.siteMetadata.site_url + edge.node.fields.slug,
-    guid: site.siteMetadata.site_url + edge.node.fields.slug,
-    custom_elements: [
-      { 'content:encoded': edge.node.html }
-    ]
-  }))
-)
+const serialize = ({ site, allMarkdownRemark }) =>
+  allMarkdownRemark.edges.map(edge => {
+    return {
+      ...edge.node.frontmatter,
+      description: edge.node.excerpt,
+      url: site.siteMetadata.site_url + edge.node.fields.slug,
+      guid: site.siteMetadata.site_url + edge.node.fields.slug,
+      custom_elements: [{ "content:encoded": edge.node.html }],
+    }
+  })
 
 exports.onPostBuild = async ({ graphql }, pluginOptions) => {
   delete pluginOptions.plugins