From 7391f2a6f529d8dbcd1b5a2e3f8e96aca1d350da Mon Sep 17 00:00:00 2001 From: David Bailey Date: Wed, 15 Aug 2018 17:32:01 +0100 Subject: [PATCH 01/20] Fix various bugs with offline support and caching --- packages/gatsby-plugin-offline/package.json | 1 + .../gatsby-plugin-offline/src/app-shell.js | 2 +- .../src/gatsby-browser.js | 29 ++++++++-------- .../gatsby-plugin-offline/src/gatsby-node.js | 31 ++++++++++++----- packages/gatsby/cache-dir/loader.js | 33 ------------------- packages/gatsby/cache-dir/page-renderer.js | 9 ++++- .../cache-dir/register-service-worker.js | 10 ++++-- 7 files changed, 57 insertions(+), 58 deletions(-) diff --git a/packages/gatsby-plugin-offline/package.json b/packages/gatsby-plugin-offline/package.json index 0359aef18b7e2..22d90d98a6084 100644 --- a/packages/gatsby-plugin-offline/package.json +++ b/packages/gatsby-plugin-offline/package.json @@ -10,6 +10,7 @@ "@babel/runtime": "7.0.0-beta.52", "cheerio": "^1.0.0-rc.2", "lodash": "^4.17.10", + "replace-in-file": "^3.4.2", "sw-precache": "^5.2.1" }, "devDependencies": { diff --git a/packages/gatsby-plugin-offline/src/app-shell.js b/packages/gatsby-plugin-offline/src/app-shell.js index 653248b9fd283..aaa44c74ead76 100644 --- a/packages/gatsby-plugin-offline/src/app-shell.js +++ b/packages/gatsby-plugin-offline/src/app-shell.js @@ -2,7 +2,7 @@ import React from "react" class AppShell extends React.Component { render() { - return
+ return } } diff --git a/packages/gatsby-plugin-offline/src/gatsby-browser.js b/packages/gatsby-plugin-offline/src/gatsby-browser.js index d15533cbe6ca5..f94e8d95a7df3 100644 --- a/packages/gatsby-plugin-offline/src/gatsby-browser.js +++ b/packages/gatsby-plugin-offline/src/gatsby-browser.js @@ -9,7 +9,7 @@ exports.onPrefetchPathname = ({ pathname, getResourcesForPathname }) => { if (swNotInstalled && `serviceWorker` in navigator) { pathnameResources.push( new Promise(resolve => { - getResourcesForPathname(pathname, resources => { + getResourcesForPathname(pathname).then(resources => { resolve(resources) }) }) @@ -22,24 +22,27 @@ exports.onServiceWorkerInstalled = () => { swNotInstalled = false // grab nodes from head of document - const nodes = document.querySelectorAll( - `head > script[src], head > link[as=script]` - ) + const nodes = document.querySelectorAll(` + head > script[src], + head > link[as=script], + head > link[rel=stylesheet], + head > style[data-href] + `) - // get all script URLs - const scripts = [].slice + // get all resource URLs + const resources = [].slice .call(nodes) - .map(node => (node.src ? node.src : node.href)) + .map(node => (node.src || node.href || node.getAttribute(`data-href`))) + + for (const resource of resources) { + fetch(resource) + } // loop over all resources and fetch the page component and JSON // thereby storing it in SW cache Promise.all(pathnameResources).then(pageResources => { - pageResources.forEach(pageResource => { - const [script] = scripts.filter(s => - s.includes(pageResource.page.componentChunkName) - ) + for (const pageResource of pageResources) { fetch(pageResource.page.jsonURL) - fetch(script) - }) + } }) } diff --git a/packages/gatsby-plugin-offline/src/gatsby-node.js b/packages/gatsby-plugin-offline/src/gatsby-node.js index 4f39f7297da5f..8d6175d42f30d 100644 --- a/packages/gatsby-plugin-offline/src/gatsby-node.js +++ b/packages/gatsby-plugin-offline/src/gatsby-node.js @@ -3,6 +3,7 @@ const precache = require(`sw-precache`) const path = require(`path`) const slash = require(`slash`) const _ = require(`lodash`) +const replace = require(`replace-in-file`) const getResourcesFromHTML = require(`./get-resources-from-html`) @@ -46,8 +47,11 @@ exports.onPostBuild = (args, pluginOptions) => { rootDir ) - const criticalFilePaths = getResourcesFromHTML( - `${process.cwd()}/${rootDir}/index.html` + const criticalFilePaths = _.concat( + getResourcesFromHTML(`${process.cwd()}/${rootDir}/index.html`), + getResourcesFromHTML( + `${process.cwd()}/${rootDir}/offline-plugin-app-shell-fallback/index.html` + ) ) const options = { @@ -56,6 +60,9 @@ exports.onPostBuild = (args, pluginOptions) => { `${rootDir}/manifest.json`, `${rootDir}/manifest.webmanifest`, `${rootDir}/offline-plugin-app-shell-fallback/index.html`, + // Webpack chunks aren't always detected by `getResourcesFromHTML` + // since they're sometimes loaded from other chunks. + `${rootDir}/+([1234567890])-*.js`, ...criticalFilePaths, ]), stripPrefix: rootDir, @@ -65,21 +72,22 @@ exports.onPostBuild = (args, pluginOptions) => { // https://github.com/GoogleChrome/sw-precache#replaceprefix-string replacePrefix: args.pathPrefix || ``, navigateFallback: `/offline-plugin-app-shell-fallback/index.html`, - // Only match URLs without extensions. + // Only match URLs without extensions or the query `no-cache=1`. // So example.com/about/ will pass but + // example.com/about/?no-cache=1 and // example.com/cheeseburger.jpg will not. // We only want the service worker to handle our "clean" // URLs and not any files hosted on the site. // - // Regex from http://stackoverflow.com/a/18017805 - navigateFallbackWhitelist: [/^.*([^.]{5}|.html)$/], + // Regex based on http://stackoverflow.com/a/18017805 + navigateFallbackWhitelist: [/^.*([^.]{5}|.html)(? { const combinedOptions = _.defaults(pluginOptions, options) - return precache.write(`public/sw.js`, combinedOptions) + return precache.write(`public/sw.js`, combinedOptions).then(() => + // Patch sw.js to include search queries when matching URLs against navigateFallbackWhitelist + replace({ + files: `public/sw.js`, + from: `path = (new URL(absoluteUrlString)).pathname`, + to: `url = new URL(absoluteUrlString), path = url.pathname + url.search`, + }) + ) } diff --git a/packages/gatsby/cache-dir/loader.js b/packages/gatsby/cache-dir/loader.js index 021836bc40c59..b2a227606cb91 100644 --- a/packages/gatsby/cache-dir/loader.js +++ b/packages/gatsby/cache-dir/loader.js @@ -276,37 +276,6 @@ const queue = { getPage: pathname => findPage(pathname), - // If we're loading from a service worker (it's already activated on - // this initial render) and we can't find a page, there's a good chance - // we're on a new page that this (now old) service worker doesn't know - // about so we'll unregister it and reload. - checkIfDoingInitialRenderForSW: path => { - if ( - inInitialRender && - navigator && - navigator.serviceWorker && - navigator.serviceWorker.controller && - navigator.serviceWorker.controller.state === `activated` - ) { - if (!findPage(path)) { - navigator.serviceWorker - .getRegistrations() - .then(function(registrations) { - // We would probably need this to - // prevent unnecessary reloading of the page - // while unregistering of ServiceWorker is not happening - if (registrations.length) { - for (let registration of registrations) { - registration.unregister() - } - - window.location.reload() - } - }) - } - } - }, - getResourcesForPathnameSync: path => { const page = findPage(path) if (page) { @@ -320,8 +289,6 @@ const queue = { // if necessary and then the code/data bundles. Used for prefetching // and getting resources for page changes. getResourcesForPathname: path => { - queue.checkIfDoingInitialRenderForSW(path) - return new Promise((resolve, reject) => { const doingInitialRender = inInitialRender inInitialRender = false diff --git a/packages/gatsby/cache-dir/page-renderer.js b/packages/gatsby/cache-dir/page-renderer.js index 00f7453d6a3d6..f2919b38f60fe 100644 --- a/packages/gatsby/cache-dir/page-renderer.js +++ b/packages/gatsby/cache-dir/page-renderer.js @@ -119,7 +119,14 @@ class PageRenderer extends React.Component { } render() { - if (!this.state.pageResources) return null + if (!(this.state.pageResources && this.state.pageResources.json)) { + if (window.location.search) { + window.location.search += `&no-cache=1` + } else { + window.location.search = `?no-cache=1` + } + return null + } const pathContext = process.env.NODE_ENV !== `production` diff --git a/packages/gatsby/cache-dir/register-service-worker.js b/packages/gatsby/cache-dir/register-service-worker.js index 391f944431d09..3592a490a9787 100644 --- a/packages/gatsby/cache-dir/register-service-worker.js +++ b/packages/gatsby/cache-dir/register-service-worker.js @@ -23,8 +23,14 @@ if (`serviceWorker` in navigator) { // It's the perfect time to display a "Content is cached for offline use." message. console.log(`Content is now available offline!`) - // post to service worker that install is complete - apiRunner(`onServiceWorkerInstalled`, { serviceWorker: reg }) + // Post to service worker that install is complete. + // Delay to allow time for the event listener to be added -- + // otherwise fetch is called too soon and resources aren't cached. + window.setTimeout(() => { + apiRunner(`onServiceWorkerInstalled`, { + serviceWorker: reg, + }) + }, 100) } break From 15ee19294647dbcbb7861507d5d7ccb80211645f Mon Sep 17 00:00:00 2001 From: David Bailey Date: Wed, 15 Aug 2018 17:44:54 +0100 Subject: [PATCH 02/20] Bump gatsby-plugin-offline version and update docs --- packages/gatsby-plugin-offline/README.md | 34 +++++++++++++-------- packages/gatsby-plugin-offline/package.json | 2 +- packages/gatsby/cache-dir/page-renderer.js | 7 +++++ 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/packages/gatsby-plugin-offline/README.md b/packages/gatsby-plugin-offline/README.md index 0f6de83e533f9..debf71d0fcc7b 100644 --- a/packages/gatsby-plugin-offline/README.md +++ b/packages/gatsby-plugin-offline/README.md @@ -29,34 +29,42 @@ and AppCache setup by changing these options so tread carefully. ```javascript const options = { - staticFileGlobs: [ - `${rootDir}/**/*.{woff2}`, - `${rootDir}/commons-*js`, - `${rootDir}/app-*js`, + staticFileGlobs: files.concat([ `${rootDir}/index.html`, `${rootDir}/manifest.json`, `${rootDir}/manifest.webmanifest`, `${rootDir}/offline-plugin-app-shell-fallback/index.html`, - ], + // Webpack chunks aren't always detected by `getResourcesFromHTML` + // since they're sometimes loaded from other chunks. + `${rootDir}/+([1234567890])-*.js`, + ...criticalFilePaths, + ]), stripPrefix: rootDir, + // If `pathPrefix` is configured by user, we should replace + // the `public` prefix with `pathPrefix`. + // See more at: + // https://github.com/GoogleChrome/sw-precache#replaceprefix-string + replacePrefix: args.pathPrefix || ``, navigateFallback: `/offline-plugin-app-shell-fallback/index.html`, - // Only match URLs without extensions. + // Only match URLs without extensions or the query `no-cache=1`. // So example.com/about/ will pass but + // example.com/about/?no-cache=1 and // example.com/cheeseburger.jpg will not. // We only want the service worker to handle our "clean" // URLs and not any files hosted on the site. - navigateFallbackWhitelist: [/^.*(?!\.\w?$)/], + // + // Regex based on http://stackoverflow.com/a/18017805 + navigateFallbackWhitelist: [/^.*([^.]{5}|.html)(?", "bugs": { "url": "https://github.com/gatsbyjs/gatsby/issues" diff --git a/packages/gatsby/cache-dir/page-renderer.js b/packages/gatsby/cache-dir/page-renderer.js index f2919b38f60fe..203b8ab90fc2c 100644 --- a/packages/gatsby/cache-dir/page-renderer.js +++ b/packages/gatsby/cache-dir/page-renderer.js @@ -120,6 +120,13 @@ class PageRenderer extends React.Component { render() { if (!(this.state.pageResources && this.state.pageResources.json)) { + // Try to load the page directly. + // + // Usually the page either doesn't exist, resulting in a 404 error, or + // isn't available offline. Appending the query ensures the correct error + // message is displayed, or if the page is available online, it will + // load properly. + if (window.location.search) { window.location.search += `&no-cache=1` } else { From 51c2e101493291b0e08789b16563013352272e67 Mon Sep 17 00:00:00 2001 From: David Bailey Date: Wed, 15 Aug 2018 18:21:23 +0100 Subject: [PATCH 03/20] Restore previous gatsby-plugin-offline version --- packages/gatsby-plugin-offline/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gatsby-plugin-offline/package.json b/packages/gatsby-plugin-offline/package.json index 7451b3fa85fc4..22d90d98a6084 100644 --- a/packages/gatsby-plugin-offline/package.json +++ b/packages/gatsby-plugin-offline/package.json @@ -1,7 +1,7 @@ { "name": "gatsby-plugin-offline", "description": "Gatsby plugin which sets up a site to be able to run offline", - "version": "2.0.0-beta.10", + "version": "2.0.0-beta.9", "author": "Kyle Mathews ", "bugs": { "url": "https://github.com/gatsbyjs/gatsby/issues" From c0578cacfdde6de1ec8e29508da0ba749bff95b8 Mon Sep 17 00:00:00 2001 From: David Bailey Date: Wed, 15 Aug 2018 23:20:34 +0100 Subject: [PATCH 04/20] Handle offline 404 pages properly --- .../gatsby-plugin-offline/src/gatsby-node.js | 1 + .../src/get-resources-from-html.js | 21 +++++++++++++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/packages/gatsby-plugin-offline/src/gatsby-node.js b/packages/gatsby-plugin-offline/src/gatsby-node.js index 8d6175d42f30d..8212bc3d5de34 100644 --- a/packages/gatsby-plugin-offline/src/gatsby-node.js +++ b/packages/gatsby-plugin-offline/src/gatsby-node.js @@ -49,6 +49,7 @@ exports.onPostBuild = (args, pluginOptions) => { const criticalFilePaths = _.concat( getResourcesFromHTML(`${process.cwd()}/${rootDir}/index.html`), + getResourcesFromHTML(`${process.cwd()}/${rootDir}/404.html`), getResourcesFromHTML( `${process.cwd()}/${rootDir}/offline-plugin-app-shell-fallback/index.html` ) diff --git a/packages/gatsby-plugin-offline/src/get-resources-from-html.js b/packages/gatsby-plugin-offline/src/get-resources-from-html.js index aaa2d00b8f9b3..dce450ceff991 100644 --- a/packages/gatsby-plugin-offline/src/get-resources-from-html.js +++ b/packages/gatsby-plugin-offline/src/get-resources-from-html.js @@ -4,8 +4,21 @@ const fs = require(`fs`) const _ = require(`lodash`) module.exports = htmlPath => { - // load index.html to pull scripts/links necessary for proper offline reload - const html = fs.readFileSync(path.resolve(htmlPath)) + let html + + try { + // load index.html to pull scripts/links necessary for proper offline reload + html = fs.readFileSync(path.resolve(htmlPath)) + } catch (err) { + // ENOENT means the file doesn't exist, which is to be expected when trying + // to open 404.html if the user hasn't created a custom 404 page -- return + // an empty array. + if (err.code === `ENOENT`) { + return [] + } else { + throw err + } + } // party like it's 2006 const $ = cheerio.load(html) @@ -13,10 +26,10 @@ module.exports = htmlPath => { // holds any paths for scripts and links const criticalFilePaths = [] - $(`script[src], link[as=script], link[as=font], link[as=fetch]`).each( + $(`script[src], link[as=script], link[as=font], link[as=fetch], link[rel=stylesheet], style[data-href]`).each( (_, elem) => { const $elem = $(elem) - const url = $elem.attr(`src`) || $elem.attr(`href`) + const url = $elem.attr(`src`) || $elem.attr(`href`) || $elem.attr(`data-href`) const blackListRegex = /\.xml$/ if (!blackListRegex.test(url)) { From 5ae987e85acd7093729bf725e86fa27c724891a4 Mon Sep 17 00:00:00 2001 From: David Bailey Date: Thu, 16 Aug 2018 09:50:03 +0100 Subject: [PATCH 05/20] Avoid incorrectly showing 404 pages --- packages/gatsby/cache-dir/production-app.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gatsby/cache-dir/production-app.js b/packages/gatsby/cache-dir/production-app.js index 94dc2c475b6e3..b3dd1aa14d4aa 100644 --- a/packages/gatsby/cache-dir/production-app.js +++ b/packages/gatsby/cache-dir/production-app.js @@ -82,7 +82,7 @@ apiRunnerAsync(`onClientEntry`).then(() => { loader .getResourcesForPathname(window.location.pathname) .then(() => { - if (!loader.getPage(window.location.pathname)) { + if (!loader.getPage(window.location.pathname) && window.location.match(/(? Date: Thu, 16 Aug 2018 09:50:58 +0100 Subject: [PATCH 06/20] Tidy up get-resources-from-html.js --- .../src/get-resources-from-html.js | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/packages/gatsby-plugin-offline/src/get-resources-from-html.js b/packages/gatsby-plugin-offline/src/get-resources-from-html.js index dce450ceff991..0fc265c391999 100644 --- a/packages/gatsby-plugin-offline/src/get-resources-from-html.js +++ b/packages/gatsby-plugin-offline/src/get-resources-from-html.js @@ -26,22 +26,28 @@ module.exports = htmlPath => { // holds any paths for scripts and links const criticalFilePaths = [] - $(`script[src], link[as=script], link[as=font], link[as=fetch], link[rel=stylesheet], style[data-href]`).each( - (_, elem) => { - const $elem = $(elem) - const url = $elem.attr(`src`) || $elem.attr(`href`) || $elem.attr(`data-href`) - const blackListRegex = /\.xml$/ - - if (!blackListRegex.test(url)) { - let path = url - if (url.substr(0, 4) !== `http`) { - path = `public${url}` - } - - criticalFilePaths.push(path) + $(` + script[src], + link[as=script], + link[as=font], + link[as=fetch], + link[rel=stylesheet], + style[data-href] + `).each((_, elem) => { + const $elem = $(elem) + const url = + $elem.attr(`src`) || $elem.attr(`href`) || $elem.attr(`data-href`) + const blackListRegex = /\.xml$/ + + if (!blackListRegex.test(url)) { + let path = url + if (url.substr(0, 4) !== `http`) { + path = `public${url}` } + + criticalFilePaths.push(path) } - ) + }) return _.uniq(criticalFilePaths) } From ef09930141ed6b84e2b65698b82d508c0938eaee Mon Sep 17 00:00:00 2001 From: David Bailey Date: Thu, 16 Aug 2018 10:21:13 +0100 Subject: [PATCH 07/20] Fix remaining issues with 404 pages --- packages/gatsby/cache-dir/loader.js | 16 ++++++++++++++++ packages/gatsby/cache-dir/page-renderer.js | 15 +-------------- packages/gatsby/cache-dir/production-app.js | 5 ++++- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/packages/gatsby/cache-dir/loader.js b/packages/gatsby/cache-dir/loader.js index b2a227606cb91..2413c86d09b15 100644 --- a/packages/gatsby/cache-dir/loader.js +++ b/packages/gatsby/cache-dir/loader.js @@ -319,6 +319,22 @@ const queue = { if (!page) { console.log(`A page wasn't found for "${path}"`) + + // Try to load the page directly. + // + // Usually the page either doesn't exist, resulting in a 404 error, or + // isn't available offline. Appending the query ensures the correct error + // message is displayed, or if the page is available online, it will + // load properly. + + if (!window.location.search.match(/(\?|&)no-cache=1$/)) { + if (window.location.search) { + window.location.search += `&no-cache=1` + } else { + window.location.search = `?no-cache=1` + } + } + return resolve() } diff --git a/packages/gatsby/cache-dir/page-renderer.js b/packages/gatsby/cache-dir/page-renderer.js index 203b8ab90fc2c..96dec9a8becb6 100644 --- a/packages/gatsby/cache-dir/page-renderer.js +++ b/packages/gatsby/cache-dir/page-renderer.js @@ -119,21 +119,8 @@ class PageRenderer extends React.Component { } render() { - if (!(this.state.pageResources && this.state.pageResources.json)) { - // Try to load the page directly. - // - // Usually the page either doesn't exist, resulting in a 404 error, or - // isn't available offline. Appending the query ensures the correct error - // message is displayed, or if the page is available online, it will - // load properly. - - if (window.location.search) { - window.location.search += `&no-cache=1` - } else { - window.location.search = `?no-cache=1` - } + if (!(this.state.pageResources && this.state.pageResources.json)) return null - } const pathContext = process.env.NODE_ENV !== `production` diff --git a/packages/gatsby/cache-dir/production-app.js b/packages/gatsby/cache-dir/production-app.js index b3dd1aa14d4aa..70f17ed6748d9 100644 --- a/packages/gatsby/cache-dir/production-app.js +++ b/packages/gatsby/cache-dir/production-app.js @@ -82,7 +82,10 @@ apiRunnerAsync(`onClientEntry`).then(() => { loader .getResourcesForPathname(window.location.pathname) .then(() => { - if (!loader.getPage(window.location.pathname) && window.location.match(/(? Date: Thu, 16 Aug 2018 10:26:08 +0100 Subject: [PATCH 08/20] 404 page is no longer critical --- .../gatsby-plugin-offline/src/gatsby-node.js | 1 - .../src/get-resources-from-html.js | 17 ++--------------- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/packages/gatsby-plugin-offline/src/gatsby-node.js b/packages/gatsby-plugin-offline/src/gatsby-node.js index 8212bc3d5de34..8d6175d42f30d 100644 --- a/packages/gatsby-plugin-offline/src/gatsby-node.js +++ b/packages/gatsby-plugin-offline/src/gatsby-node.js @@ -49,7 +49,6 @@ exports.onPostBuild = (args, pluginOptions) => { const criticalFilePaths = _.concat( getResourcesFromHTML(`${process.cwd()}/${rootDir}/index.html`), - getResourcesFromHTML(`${process.cwd()}/${rootDir}/404.html`), getResourcesFromHTML( `${process.cwd()}/${rootDir}/offline-plugin-app-shell-fallback/index.html` ) diff --git a/packages/gatsby-plugin-offline/src/get-resources-from-html.js b/packages/gatsby-plugin-offline/src/get-resources-from-html.js index 0fc265c391999..786f96e7c2b71 100644 --- a/packages/gatsby-plugin-offline/src/get-resources-from-html.js +++ b/packages/gatsby-plugin-offline/src/get-resources-from-html.js @@ -4,21 +4,8 @@ const fs = require(`fs`) const _ = require(`lodash`) module.exports = htmlPath => { - let html - - try { - // load index.html to pull scripts/links necessary for proper offline reload - html = fs.readFileSync(path.resolve(htmlPath)) - } catch (err) { - // ENOENT means the file doesn't exist, which is to be expected when trying - // to open 404.html if the user hasn't created a custom 404 page -- return - // an empty array. - if (err.code === `ENOENT`) { - return [] - } else { - throw err - } - } + // load index.html to pull scripts/links necessary for proper offline reload + const html = fs.readFileSync(path.resolve(htmlPath)) // party like it's 2006 const $ = cheerio.load(html) From 9c5f24a0d66dc322e0b247097158859a18596800 Mon Sep 17 00:00:00 2001 From: David Bailey Date: Thu, 16 Aug 2018 17:16:35 +0100 Subject: [PATCH 09/20] Throw an error if the sw.js patch fails --- packages/gatsby-plugin-offline/src/gatsby-node.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/gatsby-plugin-offline/src/gatsby-node.js b/packages/gatsby-plugin-offline/src/gatsby-node.js index 8d6175d42f30d..fc46822587fd4 100644 --- a/packages/gatsby-plugin-offline/src/gatsby-node.js +++ b/packages/gatsby-plugin-offline/src/gatsby-node.js @@ -102,6 +102,13 @@ exports.onPostBuild = (args, pluginOptions) => { files: `public/sw.js`, from: `path = (new URL(absoluteUrlString)).pathname`, to: `url = new URL(absoluteUrlString), path = url.pathname + url.search`, + }).then(changes => { + // Check that the patch has been applied correctly + if (changes.length !== 1) + throw new Error( + `Patching sw.js failed - sw-precache has probably been modified upstream.\n` + + `Please report this issue at https://github.com/gatsbyjs/gatsby/issues` + ) }) ) } From 02e0d8ba059e0a9cdaf98c5a769d704b888eb6ef Mon Sep 17 00:00:00 2001 From: David Bailey Date: Fri, 17 Aug 2018 00:14:30 +0100 Subject: [PATCH 10/20] Fix remaining 404 and caching issues --- .../src/gatsby-browser.js | 4 +- .../gatsby-plugin-offline/src/gatsby-node.js | 1 + .../src/get-resources-from-html.js | 15 ++++++- packages/gatsby/cache-dir/loader.js | 44 ++++++++++++++----- packages/gatsby/cache-dir/navigation.js | 11 ++++- packages/gatsby/cache-dir/page-renderer.js | 17 ++++++- packages/gatsby/cache-dir/production-app.js | 5 +-- yarn.lock | 8 ++++ 8 files changed, 84 insertions(+), 21 deletions(-) diff --git a/packages/gatsby-plugin-offline/src/gatsby-browser.js b/packages/gatsby-plugin-offline/src/gatsby-browser.js index f94e8d95a7df3..c4ab8148165a6 100644 --- a/packages/gatsby-plugin-offline/src/gatsby-browser.js +++ b/packages/gatsby-plugin-offline/src/gatsby-browser.js @@ -32,7 +32,7 @@ exports.onServiceWorkerInstalled = () => { // get all resource URLs const resources = [].slice .call(nodes) - .map(node => (node.src || node.href || node.getAttribute(`data-href`))) + .map(node => node.src || node.href || node.getAttribute(`data-href`)) for (const resource of resources) { fetch(resource) @@ -42,7 +42,7 @@ exports.onServiceWorkerInstalled = () => { // thereby storing it in SW cache Promise.all(pathnameResources).then(pageResources => { for (const pageResource of pageResources) { - fetch(pageResource.page.jsonURL) + if (pageResource) fetch(pageResource.page.jsonURL) } }) } diff --git a/packages/gatsby-plugin-offline/src/gatsby-node.js b/packages/gatsby-plugin-offline/src/gatsby-node.js index fc46822587fd4..4027856974622 100644 --- a/packages/gatsby-plugin-offline/src/gatsby-node.js +++ b/packages/gatsby-plugin-offline/src/gatsby-node.js @@ -49,6 +49,7 @@ exports.onPostBuild = (args, pluginOptions) => { const criticalFilePaths = _.concat( getResourcesFromHTML(`${process.cwd()}/${rootDir}/index.html`), + getResourcesFromHTML(`${process.cwd()}/${rootDir}/404.html`), getResourcesFromHTML( `${process.cwd()}/${rootDir}/offline-plugin-app-shell-fallback/index.html` ) diff --git a/packages/gatsby-plugin-offline/src/get-resources-from-html.js b/packages/gatsby-plugin-offline/src/get-resources-from-html.js index 786f96e7c2b71..7c2037fb8c6bd 100644 --- a/packages/gatsby-plugin-offline/src/get-resources-from-html.js +++ b/packages/gatsby-plugin-offline/src/get-resources-from-html.js @@ -5,7 +5,20 @@ const _ = require(`lodash`) module.exports = htmlPath => { // load index.html to pull scripts/links necessary for proper offline reload - const html = fs.readFileSync(path.resolve(htmlPath)) + let html + try { + // load index.html to pull scripts/links necessary for proper offline reload + html = fs.readFileSync(path.resolve(htmlPath)) + } catch (err) { + // ENOENT means the file doesn't exist, which is to be expected when trying + // to open 404.html if the user hasn't created a custom 404 page -- return + // an empty array. + if (err.code === `ENOENT`) { + return [] + } else { + throw err + } + } // party like it's 2006 const $ = cheerio.load(html) diff --git a/packages/gatsby/cache-dir/loader.js b/packages/gatsby/cache-dir/loader.js index 2413c86d09b15..38443a1d6f605 100644 --- a/packages/gatsby/cache-dir/loader.js +++ b/packages/gatsby/cache-dir/loader.js @@ -320,19 +320,41 @@ const queue = { if (!page) { console.log(`A page wasn't found for "${path}"`) - // Try to load the page directly. - // - // Usually the page either doesn't exist, resulting in a 404 error, or - // isn't available offline. Appending the query ensures the correct error - // message is displayed, or if the page is available online, it will - // load properly. - - if (!window.location.search.match(/(\?|&)no-cache=1$/)) { - if (window.location.search) { - window.location.search += `&no-cache=1` + // Try to load the page directly (as opposed to from the cache). + + // Store the URL for testing later with `fetch` + let url = new URL(window.location) + + // Check the page isn't already loaded directly; that it's the page + // we're on currently, not a page which failed to preload; and that + // we're not running `gatsby develop` + if ( + !url.search.match(/(\?|&)no-cache=1$/) && + window.location.pathname === path && + process.env.NODE_ENV === `production` + ) { + // Append the appropriate query to the URL + if (url.search) { + url.search += `&no-cache=1` } else { - window.location.search = `?no-cache=1` + url.search = `?no-cache=1` } + + // Now test if the page is available directly + fetch(url.href) + .then(response => { + // Redirect there if there isn't a 404 + if (response.status !== 404) window.location.replace(url) + + // If a 404 occurs, we don't need to redirect since a 404 page + // is displayed anyway. If another HTTP error occurs, a more + // appropriate error message will be displayed after redirecting. + }) + .catch(() => { + // If an error occurs (usually when offline), navigate to the + // page anyway to show the browser's proper offline error page + window.location.replace(url) + }) } return resolve() diff --git a/packages/gatsby/cache-dir/navigation.js b/packages/gatsby/cache-dir/navigation.js index 8030e35fb29c6..167097fa2fac5 100644 --- a/packages/gatsby/cache-dir/navigation.js +++ b/packages/gatsby/cache-dir/navigation.js @@ -72,10 +72,17 @@ const navigate = (to, options) => { loader.getResourcesForPathname(pathname).then(pageResources => { if (!pageResources && process.env.NODE_ENV === `production`) { - loader.getResourcesForPathname(`/404.html`).then(() => { + loader.getResourcesForPathname(`/404.html`).then(response => { clearTimeout(timeoutId) onPreRouteUpdate(window.location) - reachNavigate(to, options).then(() => onRouteUpdate(window.location)) + + // Show the server's 404 page by navigating directly if a custom page + // doesn't exist (otherwise the page contents won't change) + if (!response) { + window.location.href = to + } else { + reachNavigate(to, options).then(() => onRouteUpdate(window.location)) + } }) } else { onPreRouteUpdate(window.location) diff --git a/packages/gatsby/cache-dir/page-renderer.js b/packages/gatsby/cache-dir/page-renderer.js index 96dec9a8becb6..682de795a8aad 100644 --- a/packages/gatsby/cache-dir/page-renderer.js +++ b/packages/gatsby/cache-dir/page-renderer.js @@ -119,8 +119,23 @@ class PageRenderer extends React.Component { } render() { - if (!(this.state.pageResources && this.state.pageResources.json)) + if ( + !(this.state.pageResources && this.state.pageResources.json) && + process.env.NODE_ENV === `production` + ) { + // Try to load the page directly - this should result in a 404 or + // network offline error. + + const url = new URL(window.location) + if (url.search) { + url.search += `&no-cache=1` + } else { + url.search += `?no-cache=1` + } + window.location.replace(url) + return null + } const pathContext = process.env.NODE_ENV !== `production` diff --git a/packages/gatsby/cache-dir/production-app.js b/packages/gatsby/cache-dir/production-app.js index 70f17ed6748d9..94dc2c475b6e3 100644 --- a/packages/gatsby/cache-dir/production-app.js +++ b/packages/gatsby/cache-dir/production-app.js @@ -82,10 +82,7 @@ apiRunnerAsync(`onClientEntry`).then(() => { loader .getResourcesForPathname(window.location.pathname) .then(() => { - if ( - !loader.getPage(window.location.pathname) && - window.location.search.match(/(\?|&)no-cache=1$/) - ) { + if (!loader.getPage(window.location.pathname)) { return loader.getResourcesForPathname(`/404.html`) } }) diff --git a/yarn.lock b/yarn.lock index 83221bb4c7b0b..55ddc0be6276d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -13022,6 +13022,14 @@ replace-ext@1.0.0, replace-ext@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/replace-ext/-/replace-ext-1.0.0.tgz#de63128373fcbf7c3ccfa4de5a480c45a67958eb" +replace-in-file@^3.4.2: + version "3.4.2" + resolved "https://registry.yarnpkg.com/replace-in-file/-/replace-in-file-3.4.2.tgz#6d40f076ac86948e28efeb6fab73fbad5c0bfa2a" + dependencies: + chalk "^2.4.1" + glob "^7.1.2" + yargs "^12.0.1" + request-promise-core@1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/request-promise-core/-/request-promise-core-1.1.1.tgz#3eee00b2c5aa83239cfb04c5700da36f81cd08b6" From 2c1cec92540a31732c0f28620b28e42f47868c1f Mon Sep 17 00:00:00 2001 From: David Bailey Date: Fri, 17 Aug 2018 17:16:54 +0100 Subject: [PATCH 11/20] Move 404 logic from loader.js to navigation.js --- packages/gatsby/cache-dir/loader.js | 38 ------------------------- packages/gatsby/cache-dir/navigation.js | 36 ++++++++++++++++++++++- 2 files changed, 35 insertions(+), 39 deletions(-) diff --git a/packages/gatsby/cache-dir/loader.js b/packages/gatsby/cache-dir/loader.js index 38443a1d6f605..b2a227606cb91 100644 --- a/packages/gatsby/cache-dir/loader.js +++ b/packages/gatsby/cache-dir/loader.js @@ -319,44 +319,6 @@ const queue = { if (!page) { console.log(`A page wasn't found for "${path}"`) - - // Try to load the page directly (as opposed to from the cache). - - // Store the URL for testing later with `fetch` - let url = new URL(window.location) - - // Check the page isn't already loaded directly; that it's the page - // we're on currently, not a page which failed to preload; and that - // we're not running `gatsby develop` - if ( - !url.search.match(/(\?|&)no-cache=1$/) && - window.location.pathname === path && - process.env.NODE_ENV === `production` - ) { - // Append the appropriate query to the URL - if (url.search) { - url.search += `&no-cache=1` - } else { - url.search = `?no-cache=1` - } - - // Now test if the page is available directly - fetch(url.href) - .then(response => { - // Redirect there if there isn't a 404 - if (response.status !== 404) window.location.replace(url) - - // If a 404 occurs, we don't need to redirect since a 404 page - // is displayed anyway. If another HTTP error occurs, a more - // appropriate error message will be displayed after redirecting. - }) - .catch(() => { - // If an error occurs (usually when offline), navigate to the - // page anyway to show the browser's proper offline error page - window.location.replace(url) - }) - } - return resolve() } diff --git a/packages/gatsby/cache-dir/navigation.js b/packages/gatsby/cache-dir/navigation.js index 167097fa2fac5..a7edca28421d7 100644 --- a/packages/gatsby/cache-dir/navigation.js +++ b/packages/gatsby/cache-dir/navigation.js @@ -81,7 +81,41 @@ const navigate = (to, options) => { if (!response) { window.location.href = to } else { - reachNavigate(to, options).then(() => onRouteUpdate(window.location)) + // Try to load the page directly (as opposed to from the cache). + // + // Store the URL for testing later with `fetch`. + let url = new URL(window.location) + + // Check the page isn't already loaded directly. + if (!url.search.match(/(\?|&)no-cache=1$/)) { + // Append the appropriate query to the URL + if (url.search) { + url.search += `&no-cache=1` + } else { + url.search = `?no-cache=1` + } + + // Now test if the page is available directly + fetch(url.href) + .then(response => { + if (response.status !== 404) { + // Redirect there if there isn't a 404. If a different HTTP + // error occurs, the appropriate error message will be + // displayed after loading the page directly. + window.location.replace(url) + } else { + // If a 404 occurs, show the custom 404 page. + reachNavigate(to, options).then(() => + onRouteUpdate(window.location) + ) + } + }) + .catch(() => { + // If an error occurs (usually when offline), navigate to the + // page anyway to show the browser's proper offline error page + window.location.replace(url) + }) + } } }) } else { From bf8c67c5daf8717f6077ffb6849b184c8ef1d22e Mon Sep 17 00:00:00 2001 From: David Bailey Date: Fri, 17 Aug 2018 22:54:23 +0100 Subject: [PATCH 12/20] Fix 404 page issues with gatsby dev --- packages/gatsby/cache-dir/loader.js | 6 ++++++ packages/gatsby/cache-dir/root.js | 16 ++++++++++++---- packages/gatsby/src/commands/serve.js | 7 ------- .../dev-404-page/raw_dev-404-page.js | 4 +++- 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/packages/gatsby/cache-dir/loader.js b/packages/gatsby/cache-dir/loader.js index b2a227606cb91..d4959ea87cf88 100644 --- a/packages/gatsby/cache-dir/loader.js +++ b/packages/gatsby/cache-dir/loader.js @@ -319,6 +319,12 @@ const queue = { if (!page) { console.log(`A page wasn't found for "${path}"`) + + // Preload the custom 404 page when running gatsby-dev + if (path !== `/404.html` && process.env.NODE_ENV !== `production`) { + queue.getResourcesForPathname(`/404.html`) + } + return resolve() } diff --git a/packages/gatsby/cache-dir/root.js b/packages/gatsby/cache-dir/root.js index 266e07350a254..9c69044592915 100644 --- a/packages/gatsby/cache-dir/root.js +++ b/packages/gatsby/cache-dir/root.js @@ -54,9 +54,8 @@ class RouteHandler extends React.Component { } render() { - const { location } = this.props - const { pathname } = location - const pageResources = loader.getResourcesForPathnameSync(pathname) + let { location } = this.props + const pageResources = loader.getResourcesForPathnameSync(location.pathname) const isPage = !!(pageResources && pageResources.component) let child if (isPage) { @@ -67,8 +66,17 @@ class RouteHandler extends React.Component { pageResources={pageResources} /> ) + } else if (pages.find(p => /^\/404.html$/.test(p.path))) { + location.pathname = `/404.html` + child = ( + + ) } else { - const dev404Page = pages.find(p => /^\/dev-404-page/.test(p.path)) + const dev404Page = pages.find(p => /^\/dev-404-page\/$/.test(p.path)) child = createElement( syncRequires.components[dev404Page.componentChunkName], { diff --git a/packages/gatsby/src/commands/serve.js b/packages/gatsby/src/commands/serve.js index b2109f410cffc..d1351b05b58c5 100644 --- a/packages/gatsby/src/commands/serve.js +++ b/packages/gatsby/src/commands/serve.js @@ -11,13 +11,6 @@ module.exports = program => { const app = express() app.use(compression()) app.use(express.static(`public`)) - app.use((req, res, next) => { - if (req.accepts(`html`)) { - res.status(404).sendFile(`404.html`, { root: `public` }) - } else { - next() - } - }) const server = app.listen(port, () => { let openUrlString = `http://localhost:` + port diff --git a/packages/gatsby/src/internal-plugins/dev-404-page/raw_dev-404-page.js b/packages/gatsby/src/internal-plugins/dev-404-page/raw_dev-404-page.js index d2001ddc47325..827811f980c57 100644 --- a/packages/gatsby/src/internal-plugins/dev-404-page/raw_dev-404-page.js +++ b/packages/gatsby/src/internal-plugins/dev-404-page/raw_dev-404-page.js @@ -9,7 +9,9 @@ class Dev404Page extends React.Component { } render() { const { pathname } = this.props.location - const pages = this.props.pages.filter(p => !/^\/dev-404-page/.test(p.path)) + const pages = this.props.pages.filter( + p => !/^\/dev-404-page\/$/.test(p.path) + ) let newFilePath if (pathname === `/`) { newFilePath = `src/pages/index.js` From 74ac8e55c3778a23999a7a9cf3498e5d59949da2 Mon Sep 17 00:00:00 2001 From: David Bailey Date: Sat, 18 Aug 2018 01:14:05 +0100 Subject: [PATCH 13/20] Optimise UX and fix bugs --- .../gatsby/cache-dir/load-directly-or-404.js | 39 +++++++++++++++ packages/gatsby/cache-dir/navigation.js | 48 ++----------------- packages/gatsby/cache-dir/page-renderer.js | 2 +- packages/gatsby/cache-dir/production-app.js | 7 ++- packages/gatsby/cache-dir/root.js | 2 +- packages/gatsby/src/commands/serve.js | 7 +++ 6 files changed, 59 insertions(+), 46 deletions(-) create mode 100644 packages/gatsby/cache-dir/load-directly-or-404.js diff --git a/packages/gatsby/cache-dir/load-directly-or-404.js b/packages/gatsby/cache-dir/load-directly-or-404.js new file mode 100644 index 0000000000000..06c3900752444 --- /dev/null +++ b/packages/gatsby/cache-dir/load-directly-or-404.js @@ -0,0 +1,39 @@ +export default function(resources, path) { + return new Promise(resolve => { + let url = new URL(window.location.origin + path) + + // Check the page isn't already loaded directly. + if (!url.search.match(/(\?|&)no-cache=1$/)) { + // Append the appropriate query to the URL. + if (url.search) { + url.search += `&no-cache=1` + } else { + url.search = `?no-cache=1` + } + + // Always navigate directly if a custom 404 page doesn't exist. + if (!resources) { + window.location = url + } else { + // Now test if the page is available directly + fetch(url.href) + .then(response => { + if (response.status !== 404) { + // Redirect there if there isn't a 404. If a different HTTP + // error occurs, the appropriate error message will be + // displayed after loading the page directly. + window.location.replace(url) + } else { + // If a 404 occurs, show the custom 404 page. + resolve() + } + }) + .catch(() => { + // If an error occurs (usually when offline), navigate to the + // page anyway to show the browser's proper offline error page + window.location = url + }) + } + } + }) +} diff --git a/packages/gatsby/cache-dir/navigation.js b/packages/gatsby/cache-dir/navigation.js index a7edca28421d7..9a64e1eeb5729 100644 --- a/packages/gatsby/cache-dir/navigation.js +++ b/packages/gatsby/cache-dir/navigation.js @@ -5,6 +5,7 @@ import emitter from "./emitter" import { globalHistory } from "@reach/router/lib/history" import { navigate as reachNavigate } from "@reach/router" import parsePath from "./parse-path" +import loadDirectlyOr404 from "./load-directly-or-404" // Convert to a map for faster lookup in maybeRedirect() const redirectMap = redirects.reduce((map, redirect) => { @@ -72,51 +73,12 @@ const navigate = (to, options) => { loader.getResourcesForPathname(pathname).then(pageResources => { if (!pageResources && process.env.NODE_ENV === `production`) { - loader.getResourcesForPathname(`/404.html`).then(response => { + loader.getResourcesForPathname(`/404.html`).then(resources => { clearTimeout(timeoutId) onPreRouteUpdate(window.location) - - // Show the server's 404 page by navigating directly if a custom page - // doesn't exist (otherwise the page contents won't change) - if (!response) { - window.location.href = to - } else { - // Try to load the page directly (as opposed to from the cache). - // - // Store the URL for testing later with `fetch`. - let url = new URL(window.location) - - // Check the page isn't already loaded directly. - if (!url.search.match(/(\?|&)no-cache=1$/)) { - // Append the appropriate query to the URL - if (url.search) { - url.search += `&no-cache=1` - } else { - url.search = `?no-cache=1` - } - - // Now test if the page is available directly - fetch(url.href) - .then(response => { - if (response.status !== 404) { - // Redirect there if there isn't a 404. If a different HTTP - // error occurs, the appropriate error message will be - // displayed after loading the page directly. - window.location.replace(url) - } else { - // If a 404 occurs, show the custom 404 page. - reachNavigate(to, options).then(() => - onRouteUpdate(window.location) - ) - } - }) - .catch(() => { - // If an error occurs (usually when offline), navigate to the - // page anyway to show the browser's proper offline error page - window.location.replace(url) - }) - } - } + loadDirectlyOr404(resources, to).then(() => + reachNavigate(to, options).then(() => onRouteUpdate(window.location)) + ) }) } else { onPreRouteUpdate(window.location) diff --git a/packages/gatsby/cache-dir/page-renderer.js b/packages/gatsby/cache-dir/page-renderer.js index 682de795a8aad..79150c5fde3a0 100644 --- a/packages/gatsby/cache-dir/page-renderer.js +++ b/packages/gatsby/cache-dir/page-renderer.js @@ -130,7 +130,7 @@ class PageRenderer extends React.Component { if (url.search) { url.search += `&no-cache=1` } else { - url.search += `?no-cache=1` + url.search = `?no-cache=1` } window.location.replace(url) diff --git a/packages/gatsby/cache-dir/production-app.js b/packages/gatsby/cache-dir/production-app.js index 94dc2c475b6e3..16f983f98049d 100644 --- a/packages/gatsby/cache-dir/production-app.js +++ b/packages/gatsby/cache-dir/production-app.js @@ -15,6 +15,7 @@ window.___emitter = emitter import PageRenderer from "./page-renderer" import asyncRequires from "./async-requires" import loader from "./loader" +import loadDirectlyOr404 from "./load-directly-or-404" window.asyncRequires = asyncRequires window.___emitter = emitter @@ -83,7 +84,11 @@ apiRunnerAsync(`onClientEntry`).then(() => { .getResourcesForPathname(window.location.pathname) .then(() => { if (!loader.getPage(window.location.pathname)) { - return loader.getResourcesForPathname(`/404.html`) + return loader + .getResourcesForPathname(`/404.html`) + .then(resources => + loadDirectlyOr404(resources, window.location.pathname) + ) } }) .then(() => { diff --git a/packages/gatsby/cache-dir/root.js b/packages/gatsby/cache-dir/root.js index 9c69044592915..d61e15bc789f2 100644 --- a/packages/gatsby/cache-dir/root.js +++ b/packages/gatsby/cache-dir/root.js @@ -66,7 +66,7 @@ class RouteHandler extends React.Component { pageResources={pageResources} /> ) - } else if (pages.find(p => /^\/404.html$/.test(p.path))) { + } else if (loader.getPage(`/404.html`)) { location.pathname = `/404.html` child = ( { const app = express() app.use(compression()) app.use(express.static(`public`)) + app.use((req, res, next) => { + if (req.accepts(`html`)) { + res.status(404).sendFile(`404.html`, { root: `public` }) + } else { + next() + } + }) const server = app.listen(port, () => { let openUrlString = `http://localhost:` + port From f4f76abce6ec7c72b55fcb9cbeb900d1c42da2c5 Mon Sep 17 00:00:00 2001 From: David Bailey <4248177+davidbailey00@users.noreply.github.com> Date: Sat, 18 Aug 2018 02:03:31 +0100 Subject: [PATCH 14/20] url can be a constant --- packages/gatsby/cache-dir/load-directly-or-404.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gatsby/cache-dir/load-directly-or-404.js b/packages/gatsby/cache-dir/load-directly-or-404.js index 06c3900752444..be49d4bad6500 100644 --- a/packages/gatsby/cache-dir/load-directly-or-404.js +++ b/packages/gatsby/cache-dir/load-directly-or-404.js @@ -1,6 +1,6 @@ export default function(resources, path) { return new Promise(resolve => { - let url = new URL(window.location.origin + path) + const url = new URL(window.location.origin + path) // Check the page isn't already loaded directly. if (!url.search.match(/(\?|&)no-cache=1$/)) { From 35eb86a26e07cd1dcbb3f02ecdb496d583e68300 Mon Sep 17 00:00:00 2001 From: David Bailey Date: Sat, 18 Aug 2018 15:21:56 +0100 Subject: [PATCH 15/20] fix typo --- packages/gatsby/cache-dir/loader.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gatsby/cache-dir/loader.js b/packages/gatsby/cache-dir/loader.js index d4959ea87cf88..72c2d58984202 100644 --- a/packages/gatsby/cache-dir/loader.js +++ b/packages/gatsby/cache-dir/loader.js @@ -320,7 +320,7 @@ const queue = { if (!page) { console.log(`A page wasn't found for "${path}"`) - // Preload the custom 404 page when running gatsby-dev + // Preload the custom 404 page when running `gatsby develop` if (path !== `/404.html` && process.env.NODE_ENV !== `production`) { queue.getResourcesForPathname(`/404.html`) } From a8ca6bc296b026178bea9105e71c8aa1b9e5582d Mon Sep 17 00:00:00 2001 From: David Bailey Date: Sat, 18 Aug 2018 15:30:41 +0100 Subject: [PATCH 16/20] don't cache all webpack chunks --- packages/gatsby-plugin-offline/README.md | 3 --- packages/gatsby-plugin-offline/src/gatsby-node.js | 3 --- 2 files changed, 6 deletions(-) diff --git a/packages/gatsby-plugin-offline/README.md b/packages/gatsby-plugin-offline/README.md index debf71d0fcc7b..cd11d5a6238d6 100644 --- a/packages/gatsby-plugin-offline/README.md +++ b/packages/gatsby-plugin-offline/README.md @@ -34,9 +34,6 @@ const options = { `${rootDir}/manifest.json`, `${rootDir}/manifest.webmanifest`, `${rootDir}/offline-plugin-app-shell-fallback/index.html`, - // Webpack chunks aren't always detected by `getResourcesFromHTML` - // since they're sometimes loaded from other chunks. - `${rootDir}/+([1234567890])-*.js`, ...criticalFilePaths, ]), stripPrefix: rootDir, diff --git a/packages/gatsby-plugin-offline/src/gatsby-node.js b/packages/gatsby-plugin-offline/src/gatsby-node.js index 4027856974622..9e5eccbadc2e4 100644 --- a/packages/gatsby-plugin-offline/src/gatsby-node.js +++ b/packages/gatsby-plugin-offline/src/gatsby-node.js @@ -61,9 +61,6 @@ exports.onPostBuild = (args, pluginOptions) => { `${rootDir}/manifest.json`, `${rootDir}/manifest.webmanifest`, `${rootDir}/offline-plugin-app-shell-fallback/index.html`, - // Webpack chunks aren't always detected by `getResourcesFromHTML` - // since they're sometimes loaded from other chunks. - `${rootDir}/+([1234567890])-*.js`, ...criticalFilePaths, ]), stripPrefix: rootDir, From 63ca8318eb114c521a776e70c6de3782f811e2a9 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 20 Aug 2018 17:11:09 -0700 Subject: [PATCH 17/20] Update load-directly-or-404.js * Document module * There doesn't seem to be any way to reach this page w/o the page resources having loaded so remove that check --- .../gatsby/cache-dir/load-directly-or-404.js | 73 +++++++++++-------- 1 file changed, 41 insertions(+), 32 deletions(-) diff --git a/packages/gatsby/cache-dir/load-directly-or-404.js b/packages/gatsby/cache-dir/load-directly-or-404.js index be49d4bad6500..0fe50af00c03a 100644 --- a/packages/gatsby/cache-dir/load-directly-or-404.js +++ b/packages/gatsby/cache-dir/load-directly-or-404.js @@ -1,39 +1,48 @@ +/** + * When other parts of the code can't find resources for a page, they load the 404 page's + * resources (if it exists) and then pass them here. This module then does the following: + * 1. Checks if 404 pages resources exist. If not, just navigate directly to the desired URL + * to show whatever server 404 page exists. + * 2. Try fetching the desired page to see if it exists on the server but we + * were just prevented from seeing it due to loading the site from a SW. If this is the case, + * trigger a hard reload to grab that page from the server. + * 3. If the page doesn't exist, show the normal 404 page component. + * 4. If the fetch failed (generally meaning we're offline), then navigate anyways to show + * either the browser's offline page or whatever the server error is. + */ export default function(resources, path) { return new Promise(resolve => { const url = new URL(window.location.origin + path) + + // Append the appropriate query to the URL. + if (url.search) { + url.search += `&no-cache=1` + } else { + url.search = `?no-cache=1` + } - // Check the page isn't already loaded directly. - if (!url.search.match(/(\?|&)no-cache=1$/)) { - // Append the appropriate query to the URL. - if (url.search) { - url.search += `&no-cache=1` - } else { - url.search = `?no-cache=1` - } - - // Always navigate directly if a custom 404 page doesn't exist. - if (!resources) { - window.location = url - } else { - // Now test if the page is available directly - fetch(url.href) - .then(response => { - if (response.status !== 404) { - // Redirect there if there isn't a 404. If a different HTTP - // error occurs, the appropriate error message will be - // displayed after loading the page directly. - window.location.replace(url) - } else { - // If a 404 occurs, show the custom 404 page. - resolve() - } - }) - .catch(() => { - // If an error occurs (usually when offline), navigate to the - // page anyway to show the browser's proper offline error page - window.location = url - }) - } + // Always navigate directly if a custom 404 page doesn't exist. + if (!resources) { + window.location = url + } else { + // Now test if the page is available directly + fetch(url.href) + .then(response => { + if (response.status !== 404) { + // Redirect there if there isn't a 404. If a different HTTP + // error occurs, the appropriate error message will be + // displayed after loading the page directly. + window.location.replace(url) + } else { + // If a 404 occurs, show the custom 404 page. + resolve() + } + }) + .catch(() => { + // If an error occurs (usually when offline), navigate to the + // page anyway to show the browser's proper offline error page + window.location = url + }) } }) } From 3129a551d5ad4e732c1064e87ca6f4d0ca1ee258 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 20 Aug 2018 17:13:14 -0700 Subject: [PATCH 18/20] Ensure precached files are unique --- packages/gatsby-plugin-offline/src/gatsby-node.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/gatsby-plugin-offline/src/gatsby-node.js b/packages/gatsby-plugin-offline/src/gatsby-node.js index 9e5eccbadc2e4..7de2dc621d4c1 100644 --- a/packages/gatsby-plugin-offline/src/gatsby-node.js +++ b/packages/gatsby-plugin-offline/src/gatsby-node.js @@ -47,13 +47,13 @@ exports.onPostBuild = (args, pluginOptions) => { rootDir ) - const criticalFilePaths = _.concat( + const criticalFilePaths = _.uniq(_.concat( getResourcesFromHTML(`${process.cwd()}/${rootDir}/index.html`), getResourcesFromHTML(`${process.cwd()}/${rootDir}/404.html`), getResourcesFromHTML( `${process.cwd()}/${rootDir}/offline-plugin-app-shell-fallback/index.html` ) - ) + )) const options = { staticFileGlobs: files.concat([ From 8e1efb9d360efca3414ecc80f509bb6f9294a49a Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 20 Aug 2018 17:13:55 -0700 Subject: [PATCH 19/20] Prettier --- .../gatsby-plugin-offline/src/gatsby-node.js | 16 +++++++++------- .../gatsby/cache-dir/load-directly-or-404.js | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/gatsby-plugin-offline/src/gatsby-node.js b/packages/gatsby-plugin-offline/src/gatsby-node.js index 7de2dc621d4c1..1a9c69903e5f5 100644 --- a/packages/gatsby-plugin-offline/src/gatsby-node.js +++ b/packages/gatsby-plugin-offline/src/gatsby-node.js @@ -47,13 +47,15 @@ exports.onPostBuild = (args, pluginOptions) => { rootDir ) - const criticalFilePaths = _.uniq(_.concat( - getResourcesFromHTML(`${process.cwd()}/${rootDir}/index.html`), - getResourcesFromHTML(`${process.cwd()}/${rootDir}/404.html`), - getResourcesFromHTML( - `${process.cwd()}/${rootDir}/offline-plugin-app-shell-fallback/index.html` + const criticalFilePaths = _.uniq( + _.concat( + getResourcesFromHTML(`${process.cwd()}/${rootDir}/index.html`), + getResourcesFromHTML(`${process.cwd()}/${rootDir}/404.html`), + getResourcesFromHTML( + `${process.cwd()}/${rootDir}/offline-plugin-app-shell-fallback/index.html` + ) ) - )) + ) const options = { staticFileGlobs: files.concat([ @@ -105,7 +107,7 @@ exports.onPostBuild = (args, pluginOptions) => { if (changes.length !== 1) throw new Error( `Patching sw.js failed - sw-precache has probably been modified upstream.\n` + - `Please report this issue at https://github.com/gatsbyjs/gatsby/issues` + `Please report this issue at https://github.com/gatsbyjs/gatsby/issues` ) }) ) diff --git a/packages/gatsby/cache-dir/load-directly-or-404.js b/packages/gatsby/cache-dir/load-directly-or-404.js index 0fe50af00c03a..a368fcb0ea4f4 100644 --- a/packages/gatsby/cache-dir/load-directly-or-404.js +++ b/packages/gatsby/cache-dir/load-directly-or-404.js @@ -13,7 +13,7 @@ export default function(resources, path) { return new Promise(resolve => { const url = new URL(window.location.origin + path) - + // Append the appropriate query to the URL. if (url.search) { url.search += `&no-cache=1` From b280ffe0e45f40e621b7d6b4134c9dc6ca0a8d49 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 20 Aug 2018 17:36:42 -0700 Subject: [PATCH 20/20] Fix test broken earlier --- .../__testfixtures__/global-graphql-calls/.prettierignore | 1 + .../global-graphql-calls/import-default.input.js | 4 ++-- .../global-graphql-calls/import-default.output.js | 4 ++-- .../global-graphql-calls/import-named-exports.input.js | 4 ++-- .../global-graphql-calls/import-named-exports.output.js | 4 ++-- .../global-graphql-calls/import-namespace.input.js | 4 ++-- .../global-graphql-calls/import-namespace.output.js | 4 ++-- .../global-graphql-calls/no-import-esm.input.js | 2 +- .../global-graphql-calls/no-import-esm.output.js | 4 ++-- .../global-graphql-calls/require-destructure.input.js | 4 ++-- .../global-graphql-calls/require-destructure.output.js | 7 +++++-- .../global-graphql-calls/require-namespace.input.js | 4 ++-- .../global-graphql-calls/require-namespace.output.js | 4 ++-- 13 files changed, 27 insertions(+), 23 deletions(-) create mode 100644 packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/.prettierignore diff --git a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/.prettierignore b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/.prettierignore new file mode 100644 index 0000000000000..1d085cacc9f8e --- /dev/null +++ b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/.prettierignore @@ -0,0 +1 @@ +** diff --git a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-default.input.js b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-default.input.js index 8792ec86df8a3..ab65e47bd0177 100644 --- a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-default.input.js +++ b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-default.input.js @@ -1,5 +1,5 @@ /* eslint-disable */ -import Gatsby from "gatsby" +import Gatsby from "gatsby"; export const query = graphql` query { @@ -7,4 +7,4 @@ export const query = graphql` prefix } } -` +`; diff --git a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-default.output.js b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-default.output.js index 324487b5c656e..46afcf9782382 100644 --- a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-default.output.js +++ b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-default.output.js @@ -1,5 +1,5 @@ /* eslint-disable */ -import Gatsby, { graphql } from "gatsby" +import Gatsby, { graphql } from "gatsby"; export const query = graphql` query { @@ -7,4 +7,4 @@ export const query = graphql` prefix } } -` +`; diff --git a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-named-exports.input.js b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-named-exports.input.js index 51e9a74220f55..df7334580796d 100644 --- a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-named-exports.input.js +++ b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-named-exports.input.js @@ -1,5 +1,5 @@ /* eslint-disable */ -import { Link } from "gatsby" +import { Link } from "gatsby"; export const query = graphql` query { @@ -7,4 +7,4 @@ export const query = graphql` prefix } } -` +`; diff --git a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-named-exports.output.js b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-named-exports.output.js index 5cc716b82df17..6f04ef3d88480 100644 --- a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-named-exports.output.js +++ b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-named-exports.output.js @@ -1,5 +1,5 @@ /* eslint-disable */ -import { Link, graphql } from "gatsby" +import { Link, graphql } from "gatsby"; export const query = graphql` query { @@ -7,4 +7,4 @@ export const query = graphql` prefix } } -` +`; diff --git a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-namespace.input.js b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-namespace.input.js index 72eb7fab7f880..f3a2b6d645868 100644 --- a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-namespace.input.js +++ b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-namespace.input.js @@ -1,5 +1,5 @@ /* eslint-disable */ -import * as Gatsby from "gatsby" +import * as Gatsby from "gatsby"; export const query = graphql` query { @@ -7,4 +7,4 @@ export const query = graphql` prefix } } -` +`; diff --git a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-namespace.output.js b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-namespace.output.js index 1fb5d7d35ed0b..4e24b425e7c70 100644 --- a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-namespace.output.js +++ b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/import-namespace.output.js @@ -1,5 +1,5 @@ /* eslint-disable */ -import * as Gatsby from "gatsby" +import * as Gatsby from "gatsby"; export const query = Gatsby.graphql` query { @@ -7,4 +7,4 @@ export const query = Gatsby.graphql` prefix } } -` +`; diff --git a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/no-import-esm.input.js b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/no-import-esm.input.js index 1f891f9fe7460..edd1b8d82a8aa 100644 --- a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/no-import-esm.input.js +++ b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/no-import-esm.input.js @@ -6,4 +6,4 @@ export const query = graphql` prefix } } -` +`; diff --git a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/no-import-esm.output.js b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/no-import-esm.output.js index 664d51a772833..93634daeec07c 100644 --- a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/no-import-esm.output.js +++ b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/no-import-esm.output.js @@ -1,6 +1,6 @@ /* eslint-disable */ // TODO: update codemod to make this test pass -import { graphql } from "gatsby" +import { graphql } from "gatsby"; export const query = graphql` query { @@ -8,4 +8,4 @@ export const query = graphql` prefix } } -` +`; diff --git a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/require-destructure.input.js b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/require-destructure.input.js index e0762febfd0aa..4cf420b484cf1 100644 --- a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/require-destructure.input.js +++ b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/require-destructure.input.js @@ -1,5 +1,5 @@ /* eslint-disable */ -const { Link } = require(`gatsby`) +const { Link } = require(`gatsby`); export const query = graphql` query { @@ -7,4 +7,4 @@ export const query = graphql` prefix } } -` +`; diff --git a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/require-destructure.output.js b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/require-destructure.output.js index 9f1bd5af2ffb8..6913f5a518f1b 100644 --- a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/require-destructure.output.js +++ b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/require-destructure.output.js @@ -1,5 +1,8 @@ /* eslint-disable */ -const { Link, graphql } = require(`gatsby`) +const { + Link, + graphql +} = require(`gatsby`); export const query = graphql` query { @@ -7,4 +10,4 @@ export const query = graphql` prefix } } -` +`; diff --git a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/require-namespace.input.js b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/require-namespace.input.js index 73126d44a1a46..10e90e653ef5d 100644 --- a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/require-namespace.input.js +++ b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/require-namespace.input.js @@ -1,5 +1,5 @@ /* eslint-disable */ -const Gatsby = require(`gatsby`) +const Gatsby = require(`gatsby`); export const query = graphql` query { @@ -7,4 +7,4 @@ export const query = graphql` prefix } } -` +`; diff --git a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/require-namespace.output.js b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/require-namespace.output.js index ea2e3eafa45c5..7bdd0ff50de2a 100644 --- a/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/require-namespace.output.js +++ b/packages/gatsby-codemods/src/transforms/__testfixtures__/global-graphql-calls/require-namespace.output.js @@ -1,5 +1,5 @@ /* eslint-disable */ -const Gatsby = require(`gatsby`) +const Gatsby = require(`gatsby`); export const query = Gatsby.graphql` query { @@ -7,4 +7,4 @@ export const query = Gatsby.graphql` prefix } } -` +`;