diff --git a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/.prettierignore b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/.prettierignore
new file mode 100644
index 0000000000000..1d085cacc9f8e
--- /dev/null
+++ b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/.prettierignore
@@ -0,0 +1 @@
+**
diff --git a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-default.input.js b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-default.input.js
index 8792ec86df8a3..ab65e47bd0177 100644
--- a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-default.input.js
+++ b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-default.input.js
@@ -1,5 +1,5 @@
 /* eslint-disable */
-import Gatsby from "gatsby"
+import Gatsby from "gatsby";
 
 export const query = graphql`
   query {
@@ -7,4 +7,4 @@ export const query = graphql`
       prefix
     }
   }
-`
+`;
diff --git a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-default.output.js b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-default.output.js
index 324487b5c656e..46afcf9782382 100644
--- a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-default.output.js
+++ b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-default.output.js
@@ -1,5 +1,5 @@
 /* eslint-disable */
-import Gatsby, { graphql } from "gatsby"
+import Gatsby, { graphql } from "gatsby";
 
 export const query = graphql`
   query {
@@ -7,4 +7,4 @@ export const query = graphql`
       prefix
     }
   }
-`
+`;
diff --git a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-named-exports.input.js b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-named-exports.input.js
index 51e9a74220f55..df7334580796d 100644
--- a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-named-exports.input.js
+++ b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-named-exports.input.js
@@ -1,5 +1,5 @@
 /* eslint-disable */
-import { Link } from "gatsby"
+import { Link } from "gatsby";
 
 export const query = graphql`
   query {
@@ -7,4 +7,4 @@ export const query = graphql`
       prefix
     }
   }
-`
+`;
diff --git a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-named-exports.output.js b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-named-exports.output.js
index 5cc716b82df17..6f04ef3d88480 100644
--- a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-named-exports.output.js
+++ b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-named-exports.output.js
@@ -1,5 +1,5 @@
 /* eslint-disable */
-import { Link, graphql } from "gatsby"
+import { Link, graphql } from "gatsby";
 
 export const query = graphql`
   query {
@@ -7,4 +7,4 @@ export const query = graphql`
       prefix
     }
   }
-`
+`;
diff --git a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-namespace.input.js b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-namespace.input.js
index 72eb7fab7f880..f3a2b6d645868 100644
--- a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-namespace.input.js
+++ b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-namespace.input.js
@@ -1,5 +1,5 @@
 /* eslint-disable */
-import * as Gatsby from "gatsby"
+import * as Gatsby from "gatsby";
 
 export const query = graphql`
   query {
@@ -7,4 +7,4 @@ export const query = graphql`
       prefix
     }
   }
-`
+`;
diff --git a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-namespace.output.js b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-namespace.output.js
index 1fb5d7d35ed0b..4e24b425e7c70 100644
--- a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-namespace.output.js
+++ b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-namespace.output.js
@@ -1,5 +1,5 @@
 /* eslint-disable */
-import * as Gatsby from "gatsby"
+import * as Gatsby from "gatsby";
 
 export const query = Gatsby.graphql`
   query {
@@ -7,4 +7,4 @@ export const query = Gatsby.graphql`
       prefix
     }
   }
-`
+`;
diff --git a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/no-import-esm.input.js b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/no-import-esm.input.js
index 1f891f9fe7460..edd1b8d82a8aa 100644
--- a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/no-import-esm.input.js
+++ b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/no-import-esm.input.js
@@ -6,4 +6,4 @@ export const query = graphql`
       prefix
     }
   }
-`
+`;
diff --git a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/no-import-esm.output.js b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/no-import-esm.output.js
index 664d51a772833..93634daeec07c 100644
--- a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/no-import-esm.output.js
+++ b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/no-import-esm.output.js
@@ -1,6 +1,6 @@
 /* eslint-disable */
 // TODO: update codemod to make this test pass
-import { graphql } from "gatsby"
+import { graphql } from "gatsby";
 
 export const query = graphql`
   query {
@@ -8,4 +8,4 @@ export const query = graphql`
       prefix
     }
   }
-`
+`;
diff --git a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/require-destructure.input.js b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/require-destructure.input.js
index e0762febfd0aa..4cf420b484cf1 100644
--- a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/require-destructure.input.js
+++ b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/require-destructure.input.js
@@ -1,5 +1,5 @@
 /* eslint-disable */
-const { Link } = require(`gatsby`)
+const { Link } = require(`gatsby`);
 
 export const query = graphql`
   query {
@@ -7,4 +7,4 @@ export const query = graphql`
       prefix
     }
   }
-`
+`;
diff --git a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/require-destructure.output.js b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/require-destructure.output.js
index 9f1bd5af2ffb8..6913f5a518f1b 100644
--- a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/require-destructure.output.js
+++ b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/require-destructure.output.js
@@ -1,5 +1,8 @@
 /* eslint-disable */
-const { Link, graphql } = require(`gatsby`)
+const {
+  Link,
+  graphql
+} = require(`gatsby`);
 
 export const query = graphql`
   query {
@@ -7,4 +10,4 @@ export const query = graphql`
       prefix
     }
   }
-`
+`;
diff --git a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/require-namespace.input.js b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/require-namespace.input.js
index 73126d44a1a46..10e90e653ef5d 100644
--- a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/require-namespace.input.js
+++ b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/require-namespace.input.js
@@ -1,5 +1,5 @@
 /* eslint-disable */
-const Gatsby = require(`gatsby`)
+const Gatsby = require(`gatsby`);
 
 export const query = graphql`
   query {
@@ -7,4 +7,4 @@ export const query = graphql`
       prefix
     }
   }
-`
+`;
diff --git a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/require-namespace.output.js b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/require-namespace.output.js
index ea2e3eafa45c5..7bdd0ff50de2a 100644
--- a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/require-namespace.output.js
+++ b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/require-namespace.output.js
@@ -1,5 +1,5 @@
 /* eslint-disable */
-const Gatsby = require(`gatsby`)
+const Gatsby = require(`gatsby`);
 
 export const query = Gatsby.graphql`
   query {
@@ -7,4 +7,4 @@ export const query = Gatsby.graphql`
       prefix
     }
   }
-`
+`;
diff --git a/packages/gatsby-plugin-offline/README.md b/packages/gatsby-plugin-offline/README.md
index 0f6de83e533f9..cd11d5a6238d6 100644
--- a/packages/gatsby-plugin-offline/README.md
+++ b/packages/gatsby-plugin-offline/README.md
@@ -29,34 +29,39 @@ and AppCache setup by changing these options so tread carefully.
 
 ```javascript
 const options = {
-  staticFileGlobs: [
-    `${rootDir}/**/*.{woff2}`,
-    `${rootDir}/commons-*js`,
-    `${rootDir}/app-*js`,
+  staticFileGlobs: files.concat([
     `${rootDir}/index.html`,
     `${rootDir}/manifest.json`,
     `${rootDir}/manifest.webmanifest`,
     `${rootDir}/offline-plugin-app-shell-fallback/index.html`,
-  ],
+    ...criticalFilePaths,
+  ]),
   stripPrefix: rootDir,
+  // If `pathPrefix` is configured by user, we should replace
+  // the `public` prefix with `pathPrefix`.
+  // See more at:
+  // https://github.com/GoogleChrome/sw-precache#replaceprefix-string
+  replacePrefix: args.pathPrefix || ``,
   navigateFallback: `/offline-plugin-app-shell-fallback/index.html`,
-  // Only match URLs without extensions.
+  // Only match URLs without extensions or the query `no-cache=1`.
   // So example.com/about/ will pass but
+  // example.com/about/?no-cache=1 and
   // example.com/cheeseburger.jpg will not.
   // We only want the service worker to handle our "clean"
   // URLs and not any files hosted on the site.
-  navigateFallbackWhitelist: [/^.*(?!\.\w?$)/],
+  //
+  // Regex based on http://stackoverflow.com/a/18017805
+  navigateFallbackWhitelist: [/^.*([^.]{5}|.html)(?<!(\?|&)no-cache=1)$/],
   cacheId: `gatsby-plugin-offline`,
-  // Do cache bust JS URLs until can figure out how to make Webpack's
-  // URLs truely content-addressed.
-  dontCacheBustUrlsMatching: /(.\w{8}.woff2)/, //|-\w{20}.js)/,
+  // Don't cache-bust JS files and anything in the static directory
+  dontCacheBustUrlsMatching: /(.*js$|\/static\/)/,
   runtimeCaching: [
     {
-      // Add runtime caching of images.
-      urlPattern: /\.(?:png|jpg|jpeg|webp|svg|gif|tiff)$/,
+      // Add runtime caching of various page resources.
+      urlPattern: /\.(?:png|jpg|jpeg|webp|svg|gif|tiff|js|woff|woff2|json|css)$/,
       handler: `fastest`,
     },
   ],
-  skipWaiting: false,
+  skipWaiting: true,
 }
 ```
diff --git a/packages/gatsby-plugin-offline/package.json b/packages/gatsby-plugin-offline/package.json
index 0359aef18b7e2..22d90d98a6084 100644
--- a/packages/gatsby-plugin-offline/package.json
+++ b/packages/gatsby-plugin-offline/package.json
@@ -10,6 +10,7 @@
     "@babel/runtime": "7.0.0-beta.52",
     "cheerio": "^1.0.0-rc.2",
     "lodash": "^4.17.10",
+    "replace-in-file": "^3.4.2",
     "sw-precache": "^5.2.1"
   },
   "devDependencies": {
diff --git a/packages/gatsby-plugin-offline/src/app-shell.js b/packages/gatsby-plugin-offline/src/app-shell.js
index 653248b9fd283..aaa44c74ead76 100644
--- a/packages/gatsby-plugin-offline/src/app-shell.js
+++ b/packages/gatsby-plugin-offline/src/app-shell.js
@@ -2,7 +2,7 @@ import React from "react"
 
 class AppShell extends React.Component {
   render() {
-    return <div />
+    return <span />
   }
 }
 
diff --git a/packages/gatsby-plugin-offline/src/gatsby-browser.js b/packages/gatsby-plugin-offline/src/gatsby-browser.js
index d15533cbe6ca5..c4ab8148165a6 100644
--- a/packages/gatsby-plugin-offline/src/gatsby-browser.js
+++ b/packages/gatsby-plugin-offline/src/gatsby-browser.js
@@ -9,7 +9,7 @@ exports.onPrefetchPathname = ({ pathname, getResourcesForPathname }) => {
   if (swNotInstalled && `serviceWorker` in navigator) {
     pathnameResources.push(
       new Promise(resolve => {
-        getResourcesForPathname(pathname, resources => {
+        getResourcesForPathname(pathname).then(resources => {
           resolve(resources)
         })
       })
@@ -22,24 +22,27 @@ exports.onServiceWorkerInstalled = () => {
   swNotInstalled = false
 
   // grab nodes from head of document
-  const nodes = document.querySelectorAll(
-    `head > script[src], head > link[as=script]`
-  )
+  const nodes = document.querySelectorAll(`
+    head > script[src],
+    head > link[as=script],
+    head > link[rel=stylesheet],
+    head > style[data-href]
+  `)
 
-  // get all script URLs
-  const scripts = [].slice
+  // get all resource URLs
+  const resources = [].slice
     .call(nodes)
-    .map(node => (node.src ? node.src : node.href))
+    .map(node => node.src || node.href || node.getAttribute(`data-href`))
+
+  for (const resource of resources) {
+    fetch(resource)
+  }
 
   // loop over all resources and fetch the page component and JSON
   // thereby storing it in SW cache
   Promise.all(pathnameResources).then(pageResources => {
-    pageResources.forEach(pageResource => {
-      const [script] = scripts.filter(s =>
-        s.includes(pageResource.page.componentChunkName)
-      )
-      fetch(pageResource.page.jsonURL)
-      fetch(script)
-    })
+    for (const pageResource of pageResources) {
+      if (pageResource) fetch(pageResource.page.jsonURL)
+    }
   })
 }
diff --git a/packages/gatsby-plugin-offline/src/gatsby-node.js b/packages/gatsby-plugin-offline/src/gatsby-node.js
index 4f39f7297da5f..1a9c69903e5f5 100644
--- a/packages/gatsby-plugin-offline/src/gatsby-node.js
+++ b/packages/gatsby-plugin-offline/src/gatsby-node.js
@@ -3,6 +3,7 @@ const precache = require(`sw-precache`)
 const path = require(`path`)
 const slash = require(`slash`)
 const _ = require(`lodash`)
+const replace = require(`replace-in-file`)
 
 const getResourcesFromHTML = require(`./get-resources-from-html`)
 
@@ -46,8 +47,14 @@ exports.onPostBuild = (args, pluginOptions) => {
     rootDir
   )
 
-  const criticalFilePaths = getResourcesFromHTML(
-    `${process.cwd()}/${rootDir}/index.html`
+  const criticalFilePaths = _.uniq(
+    _.concat(
+      getResourcesFromHTML(`${process.cwd()}/${rootDir}/index.html`),
+      getResourcesFromHTML(`${process.cwd()}/${rootDir}/404.html`),
+      getResourcesFromHTML(
+        `${process.cwd()}/${rootDir}/offline-plugin-app-shell-fallback/index.html`
+      )
+    )
   )
 
   const options = {
@@ -65,21 +72,22 @@ exports.onPostBuild = (args, pluginOptions) => {
     // https://github.com/GoogleChrome/sw-precache#replaceprefix-string
     replacePrefix: args.pathPrefix || ``,
     navigateFallback: `/offline-plugin-app-shell-fallback/index.html`,
-    // Only match URLs without extensions.
+    // Only match URLs without extensions or the query `no-cache=1`.
     // So example.com/about/ will pass but
+    // example.com/about/?no-cache=1 and
     // example.com/cheeseburger.jpg will not.
     // We only want the service worker to handle our "clean"
     // URLs and not any files hosted on the site.
     //
-    // Regex from http://stackoverflow.com/a/18017805
-    navigateFallbackWhitelist: [/^.*([^.]{5}|.html)$/],
+    // Regex based on http://stackoverflow.com/a/18017805
+    navigateFallbackWhitelist: [/^.*([^.]{5}|.html)(?<!(\?|&)no-cache=1)$/],
     cacheId: `gatsby-plugin-offline`,
     // Don't cache-bust JS files and anything in the static directory
     dontCacheBustUrlsMatching: /(.*js$|\/static\/)/,
     runtimeCaching: [
       {
-        // Add runtime caching of images.
-        urlPattern: /\.(?:png|jpg|jpeg|webp|svg|gif|tiff|js|woff|woff2)$/,
+        // Add runtime caching of various page resources.
+        urlPattern: /\.(?:png|jpg|jpeg|webp|svg|gif|tiff|js|woff|woff2|json|css)$/,
         handler: `fastest`,
       },
     ],
@@ -88,5 +96,19 @@ exports.onPostBuild = (args, pluginOptions) => {
 
   const combinedOptions = _.defaults(pluginOptions, options)
 
-  return precache.write(`public/sw.js`, combinedOptions)
+  return precache.write(`public/sw.js`, combinedOptions).then(() =>
+    // Patch sw.js to include search queries when matching URLs against navigateFallbackWhitelist
+    replace({
+      files: `public/sw.js`,
+      from: `path = (new URL(absoluteUrlString)).pathname`,
+      to: `url = new URL(absoluteUrlString), path = url.pathname + url.search`,
+    }).then(changes => {
+      // Check that the patch has been applied correctly
+      if (changes.length !== 1)
+        throw new Error(
+          `Patching sw.js failed - sw-precache has probably been modified upstream.\n` +
+            `Please report this issue at https://github.com/gatsbyjs/gatsby/issues`
+        )
+    })
+  )
 }
diff --git a/packages/gatsby-plugin-offline/src/get-resources-from-html.js b/packages/gatsby-plugin-offline/src/get-resources-from-html.js
index aaa2d00b8f9b3..7c2037fb8c6bd 100644
--- a/packages/gatsby-plugin-offline/src/get-resources-from-html.js
+++ b/packages/gatsby-plugin-offline/src/get-resources-from-html.js
@@ -5,7 +5,20 @@ const _ = require(`lodash`)
 
 module.exports = htmlPath => {
   // load index.html to pull scripts/links necessary for proper offline reload
-  const html = fs.readFileSync(path.resolve(htmlPath))
+  let html
+  try {
+    // load index.html to pull scripts/links necessary for proper offline reload
+    html = fs.readFileSync(path.resolve(htmlPath))
+  } catch (err) {
+    // ENOENT means the file doesn't exist, which is to be expected when trying
+    // to open 404.html if the user hasn't created a custom 404 page -- return
+    // an empty array.
+    if (err.code === `ENOENT`) {
+      return []
+    } else {
+      throw err
+    }
+  }
 
   // party like it's 2006
   const $ = cheerio.load(html)
@@ -13,22 +26,28 @@ module.exports = htmlPath => {
   // holds any paths for scripts and links
   const criticalFilePaths = []
 
-  $(`script[src], link[as=script], link[as=font], link[as=fetch]`).each(
-    (_, elem) => {
-      const $elem = $(elem)
-      const url = $elem.attr(`src`) || $elem.attr(`href`)
-      const blackListRegex = /\.xml$/
-
-      if (!blackListRegex.test(url)) {
-        let path = url
-        if (url.substr(0, 4) !== `http`) {
-          path = `public${url}`
-        }
+  $(`
+    script[src],
+    link[as=script],
+    link[as=font],
+    link[as=fetch],
+    link[rel=stylesheet],
+    style[data-href]
+  `).each((_, elem) => {
+    const $elem = $(elem)
+    const url =
+      $elem.attr(`src`) || $elem.attr(`href`) || $elem.attr(`data-href`)
+    const blackListRegex = /\.xml$/
 
-        criticalFilePaths.push(path)
+    if (!blackListRegex.test(url)) {
+      let path = url
+      if (url.substr(0, 4) !== `http`) {
+        path = `public${url}`
       }
+
+      criticalFilePaths.push(path)
     }
-  )
+  })
 
   return _.uniq(criticalFilePaths)
 }
diff --git a/packages/gatsby/cache-dir/load-directly-or-404.js b/packages/gatsby/cache-dir/load-directly-or-404.js
new file mode 100644
index 0000000000000..a368fcb0ea4f4
--- /dev/null
+++ b/packages/gatsby/cache-dir/load-directly-or-404.js
@@ -0,0 +1,48 @@
+/**
+ * When other parts of the code can't find resources for a page, they load the 404 page's
+ * resources (if it exists) and then pass them here. This module then does the following:
+ * 1. Checks if 404 pages resources exist. If not, just navigate directly to the desired URL
+ * to show whatever server 404 page exists.
+ * 2. Try fetching the desired page to see if it exists on the server but we
+ * were just prevented from seeing it due to loading the site from a SW. If this is the case,
+ * trigger a hard reload to grab that page from the server.
+ * 3. If the page doesn't exist, show the normal 404 page component.
+ * 4. If the fetch failed (generally meaning we're offline), then navigate anyways to show
+ * either the browser's offline page or whatever the server error is.
+ */
+export default function(resources, path) {
+  return new Promise(resolve => {
+    const url = new URL(window.location.origin + path)
+
+    // Append the appropriate query to the URL.
+    if (url.search) {
+      url.search += `&no-cache=1`
+    } else {
+      url.search = `?no-cache=1`
+    }
+
+    // Always navigate directly if a custom 404 page doesn't exist.
+    if (!resources) {
+      window.location = url
+    } else {
+      // Now test if the page is available directly
+      fetch(url.href)
+        .then(response => {
+          if (response.status !== 404) {
+            // Redirect there if there isn't a 404. If a different HTTP
+            // error occurs, the appropriate error message will be
+            // displayed after loading the page directly.
+            window.location.replace(url)
+          } else {
+            // If a 404 occurs, show the custom 404 page.
+            resolve()
+          }
+        })
+        .catch(() => {
+          // If an error occurs (usually when offline), navigate to the
+          // page anyway to show the browser's proper offline error page
+          window.location = url
+        })
+    }
+  })
+}
diff --git a/packages/gatsby/cache-dir/loader.js b/packages/gatsby/cache-dir/loader.js
index 021836bc40c59..72c2d58984202 100644
--- a/packages/gatsby/cache-dir/loader.js
+++ b/packages/gatsby/cache-dir/loader.js
@@ -276,37 +276,6 @@ const queue = {
 
   getPage: pathname => findPage(pathname),
 
-  // If we're loading from a service worker (it's already activated on
-  // this initial render) and we can't find a page, there's a good chance
-  // we're on a new page that this (now old) service worker doesn't know
-  // about so we'll unregister it and reload.
-  checkIfDoingInitialRenderForSW: path => {
-    if (
-      inInitialRender &&
-      navigator &&
-      navigator.serviceWorker &&
-      navigator.serviceWorker.controller &&
-      navigator.serviceWorker.controller.state === `activated`
-    ) {
-      if (!findPage(path)) {
-        navigator.serviceWorker
-          .getRegistrations()
-          .then(function(registrations) {
-            // We would probably need this to
-            // prevent unnecessary reloading of the page
-            // while unregistering of ServiceWorker is not happening
-            if (registrations.length) {
-              for (let registration of registrations) {
-                registration.unregister()
-              }
-
-              window.location.reload()
-            }
-          })
-      }
-    }
-  },
-
   getResourcesForPathnameSync: path => {
     const page = findPage(path)
     if (page) {
@@ -320,8 +289,6 @@ const queue = {
   // if necessary and then the code/data bundles. Used for prefetching
   // and getting resources for page changes.
   getResourcesForPathname: path => {
-    queue.checkIfDoingInitialRenderForSW(path)
-
     return new Promise((resolve, reject) => {
       const doingInitialRender = inInitialRender
       inInitialRender = false
@@ -352,6 +319,12 @@ const queue = {
 
       if (!page) {
         console.log(`A page wasn't found for "${path}"`)
+
+        // Preload the custom 404 page when running `gatsby develop`
+        if (path !== `/404.html` && process.env.NODE_ENV !== `production`) {
+          queue.getResourcesForPathname(`/404.html`)
+        }
+
         return resolve()
       }
 
diff --git a/packages/gatsby/cache-dir/navigation.js b/packages/gatsby/cache-dir/navigation.js
index 8030e35fb29c6..9a64e1eeb5729 100644
--- a/packages/gatsby/cache-dir/navigation.js
+++ b/packages/gatsby/cache-dir/navigation.js
@@ -5,6 +5,7 @@ import emitter from "./emitter"
 import { globalHistory } from "@reach/router/lib/history"
 import { navigate as reachNavigate } from "@reach/router"
 import parsePath from "./parse-path"
+import loadDirectlyOr404 from "./load-directly-or-404"
 
 // Convert to a map for faster lookup in maybeRedirect()
 const redirectMap = redirects.reduce((map, redirect) => {
@@ -72,10 +73,12 @@ const navigate = (to, options) => {
 
   loader.getResourcesForPathname(pathname).then(pageResources => {
     if (!pageResources && process.env.NODE_ENV === `production`) {
-      loader.getResourcesForPathname(`/404.html`).then(() => {
+      loader.getResourcesForPathname(`/404.html`).then(resources => {
         clearTimeout(timeoutId)
         onPreRouteUpdate(window.location)
-        reachNavigate(to, options).then(() => onRouteUpdate(window.location))
+        loadDirectlyOr404(resources, to).then(() =>
+          reachNavigate(to, options).then(() => onRouteUpdate(window.location))
+        )
       })
     } else {
       onPreRouteUpdate(window.location)
diff --git a/packages/gatsby/cache-dir/page-renderer.js b/packages/gatsby/cache-dir/page-renderer.js
index 38742ae7566c2..7b10d736ce2b0 100644
--- a/packages/gatsby/cache-dir/page-renderer.js
+++ b/packages/gatsby/cache-dir/page-renderer.js
@@ -119,7 +119,23 @@ class PageRenderer extends React.Component {
   }
 
   render() {
-    if (!this.state.pageResources) return null
+    if (
+      !(this.state.pageResources && this.state.pageResources.json) &&
+      process.env.NODE_ENV === `production`
+    ) {
+      // Try to load the page directly - this should result in a 404 or
+      // network offline error.
+
+      const url = new URL(window.location)
+      if (url.search) {
+        url.search += `&no-cache=1`
+      } else {
+        url.search = `?no-cache=1`
+      }
+      window.location.replace(url)
+
+      return null
+    }
 
     const pathContext =
       process.env.NODE_ENV !== `production`
diff --git a/packages/gatsby/cache-dir/production-app.js b/packages/gatsby/cache-dir/production-app.js
index 55de00e1d3839..1e31bf74d1b76 100644
--- a/packages/gatsby/cache-dir/production-app.js
+++ b/packages/gatsby/cache-dir/production-app.js
@@ -15,6 +15,7 @@ window.___emitter = emitter
 import PageRenderer from "./page-renderer"
 import asyncRequires from "./async-requires"
 import loader from "./loader"
+import loadDirectlyOr404 from "./load-directly-or-404"
 
 window.asyncRequires = asyncRequires
 window.___emitter = emitter
@@ -83,7 +84,11 @@ apiRunnerAsync(`onClientEntry`).then(() => {
     .getResourcesForPathname(window.location.pathname)
     .then(() => {
       if (!loader.getPage(window.location.pathname)) {
-        return loader.getResourcesForPathname(`/404.html`)
+        return loader
+          .getResourcesForPathname(`/404.html`)
+          .then(resources =>
+            loadDirectlyOr404(resources, window.location.pathname)
+          )
       }
     })
     .then(() => {
diff --git a/packages/gatsby/cache-dir/register-service-worker.js b/packages/gatsby/cache-dir/register-service-worker.js
index 391f944431d09..3592a490a9787 100644
--- a/packages/gatsby/cache-dir/register-service-worker.js
+++ b/packages/gatsby/cache-dir/register-service-worker.js
@@ -23,8 +23,14 @@ if (`serviceWorker` in navigator) {
                 // It's the perfect time to display a "Content is cached for offline use." message.
                 console.log(`Content is now available offline!`)
 
-                // post to service worker that install is complete
-                apiRunner(`onServiceWorkerInstalled`, { serviceWorker: reg })
+                // Post to service worker that install is complete.
+                // Delay to allow time for the event listener to be added --
+                // otherwise fetch is called too soon and resources aren't cached.
+                window.setTimeout(() => {
+                  apiRunner(`onServiceWorkerInstalled`, {
+                    serviceWorker: reg,
+                  })
+                }, 100)
               }
               break
 
diff --git a/packages/gatsby/cache-dir/root.js b/packages/gatsby/cache-dir/root.js
index 2d72df68562c7..6790683cf6f32 100644
--- a/packages/gatsby/cache-dir/root.js
+++ b/packages/gatsby/cache-dir/root.js
@@ -54,9 +54,8 @@ class RouteHandler extends React.Component {
   }
 
   render() {
-    const { location } = this.props
-    const { pathname } = location
-    const pageResources = loader.getResourcesForPathnameSync(pathname)
+    let { location } = this.props
+    const pageResources = loader.getResourcesForPathnameSync(location.pathname)
     const isPage = !!(pageResources && pageResources.component)
     let child
     if (isPage) {
@@ -67,8 +66,17 @@ class RouteHandler extends React.Component {
           pageResources={pageResources}
         />
       )
+    } else if (loader.getPage(`/404.html`)) {
+      location.pathname = `/404.html`
+      child = (
+        <JSONStore
+          pages={pages}
+          {...this.props}
+          pageResources={loader.getResourcesForPathnameSync(location.pathname)}
+        />
+      )
     } else {
-      const dev404Page = pages.find(p => /^\/dev-404-page/.test(p.path))
+      const dev404Page = pages.find(p => /^\/dev-404-page\/$/.test(p.path))
       child = createElement(
         syncRequires.components[dev404Page.componentChunkName],
         {
diff --git a/packages/gatsby/src/internal-plugins/dev-404-page/raw_dev-404-page.js b/packages/gatsby/src/internal-plugins/dev-404-page/raw_dev-404-page.js
index d2001ddc47325..827811f980c57 100644
--- a/packages/gatsby/src/internal-plugins/dev-404-page/raw_dev-404-page.js
+++ b/packages/gatsby/src/internal-plugins/dev-404-page/raw_dev-404-page.js
@@ -9,7 +9,9 @@ class Dev404Page extends React.Component {
   }
   render() {
     const { pathname } = this.props.location
-    const pages = this.props.pages.filter(p => !/^\/dev-404-page/.test(p.path))
+    const pages = this.props.pages.filter(
+      p => !/^\/dev-404-page\/$/.test(p.path)
+    )
     let newFilePath
     if (pathname === `/`) {
       newFilePath = `src/pages/index.js`
diff --git a/yarn.lock b/yarn.lock
index 00de4bf06ee29..8b1d3cd4e0e29 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -13017,6 +13017,14 @@ replace-ext@1.0.0, replace-ext@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/replace-ext/-/replace-ext-1.0.0.tgz#de63128373fcbf7c3ccfa4de5a480c45a67958eb"
 
+replace-in-file@^3.4.2:
+  version "3.4.2"
+  resolved "https://registry.yarnpkg.com/replace-in-file/-/replace-in-file-3.4.2.tgz#6d40f076ac86948e28efeb6fab73fbad5c0bfa2a"
+  dependencies:
+    chalk "^2.4.1"
+    glob "^7.1.2"
+    yargs "^12.0.1"
+
 request-promise-core@1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/request-promise-core/-/request-promise-core-1.1.1.tgz#3eee00b2c5aa83239cfb04c5700da36f81cd08b6"