feat(gatsby): Page build optimisations for incremental data changes

docs/docs/overview-of-the-gatsby-build-process.md

@@ -301,6 +301,8 @@ Page queries that were queued up earlier from query extraction are run so the da
 
 With everything ready for the HTML pages in place, HTML is compiled and written out to files so it can be served up statically. Since HTML is being produced in a Node.js server context, [references to browser APIs like `window` can break the build](/docs/debugging-html-builds/) and must be conditionally applied.
 
+By default, Gatsby rebuilds static HTML for all pages on each build. There is an experimental feature flag `GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES` which enables [Page Build Optimizations for Incremental Data Changes](/docs/page-build-optimizations-for-incremental-data-changes/).
+
 ## What do you get from a successful build?
 
 When a Gatsby build is successfully completed, everything you need to deploy your site ends up in the `public` folder at the root of the site. The build includes minified files, transformed images, JSON files with information and data for each page, static HTML for each page, and more.

docs/docs/page-build-optimizations-for-incremental-data-changes.md

@@ -0,0 +1,63 @@
+---
+title: Experimental Page Build Optimizations for Incremental Data Changes
+---
+
+Building sites with large amounts of content (10,000s nodes upwards) is relatively fast with Gatsby. However, some projects might start to experience issues when adopting CI/CD principles - continuously building and deploying. Gatsby rebuilds the complete app on each `gatsby build` which means the complete app also needs to be deployed. Doing this each time a small data change occurs unnecessarily increases demand on CPU, memory, and bandwidth.
+
+One solution to these problems might be to use [Gatsby Cloud's Build features](https://www.gatsbyjs.com/cloud/).
+
+For projects that require self-hosted environments, where Gatsby Cloud would not be an option, deploying only the content that has changed or is new (incremental data changes, you might say) can help reduce build times, deployment times and demand on resources.
+
+For more info on the standard build process please see [overview of the gatsby build process](/docs/overview-of-the-gatsby-build-process/)
+
+## How to use
+
+To enable this enhancement, use the environment variable `GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES=true` in your `gatsby build` command, for example:
+
+`GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES=true gatsby build --log-pages`
+
+This will run the Gatsby build process, but only build pages that have data changes since your last build. If there are any changes to code (JS, CSS) the bundling process returns a new webpack compilation hash which causes all pages to be rebuilt.
+
+### Reporting what has been built
+
+You may want to retrieve a list of the pages that were built. For example, if you want to perform a sync action in your CI/CD pipeline.
+
+To list the paths in the build assets (`public`) folder, you can use one (or both) of the following arguments in your `build` command.
+
+- `--log-pages` parameter will output all the file paths that were updated or deleted at the end of the build stage.
+
+```bash
+success Building production JavaScript and CSS bundles - 82.198s
+success run queries - 82.762s - 4/4 0.05/s
+success Building static HTML for pages - 19.386s - 2/2 0.10/s
++ success Delete previous page data - 1.512s
+info Done building in 152.084 sec
++ info Built pages:
++ Updated page: /about
++ Updated page: /accounts/example
++ info Deleted pages:
++ Deleted page: /test
+
+Done in 154.501 sec
+```
+
+- `--write-to-file` creates two files in the `.cache` folder, with lists of the changed paths in the build assets (`public`) folder.
+
+  - `newPages.txt` will contain a list of new or changed paths
+  - `deletedPages.txt` will contain a list of deleted paths
+
+If there are no changed or deleted paths, then the relevant files will not be created in the `.cache` folder.
+
+## More information
+
+- This enhancement works by comparing the page data from the previous build to the new page data. This creates a list of page directories that are passed to the static build process.
+
+- To enable this build option you will need to set an environment variable, which requires access to do so in your build environment.
+
+- This feature is not available with `gatsby develop`.
+
+* At the end of each build, gatsby creates a `redux.state` file in `/.cache` that contains previous build data. You will need to persist the `.cache/redux.state` between builds, allowing for comparison. If there is no `redux.state` file located in the `/.cache` folder then a full build will be triggered.
+
+* Any code or static query changes (templates, components, source handling, new plugins etc) will prompt the creation of a new webpack compilation hash and trigger a full build.
+
+Note: When using the `GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES` flag it is important to do so consistently when building your project. Otherwise, the cache will be cleared and the necessary data for comparison will no longer be available, removing the ability to check for incremental data changes.

packages/gatsby/src/bootstrap/index.js

@@ -190,7 +190,10 @@ module.exports = async (args: BootstrapArgs) => {
 
   // During builds, delete html and css files from the public directory as we don't want
   // deleted pages and styles from previous builds to stick around.
-  if (process.env.NODE_ENV === `production`) {
+  if (
+    !process.env.GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES &&
+    process.env.NODE_ENV === `production`
+  ) {
     activity = report.activityTimer(
       `delete html and css files from previous builds`,
       {
@@ -221,6 +224,7 @@ module.exports = async (args: BootstrapArgs) => {
   // logic in there e.g. generating slugs for custom pages.
   const pluginVersions = flattenedPlugins.map(p => p.version)
   const hashes = await Promise.all([
+    !!process.env.GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES,
     md5File(`package.json`),
     Promise.resolve(
       md5File(`${program.directory}/gatsby-config.js`).catch(() => {})

packages/gatsby/src/commands/build-utils.js

@@ -0,0 +1,90 @@
+const fs = require(`fs-extra`)
+const path = require(`path`)
+const {
+  remove: removePageHtmlFile,
+  getPageHtmlFilePath,
+} = require(`../utils/page-html`)
+const {
+  remove: removePageDataFile,
+  fixedPagePath,
+} = require(`../utils/page-data`)
+
+const getChangedPageDataKeys = (state, cachedPageData) => {
+  if (cachedPageData && state.pageData) {
+    const pageKeys = []
+    state.pageData.forEach((newPageDataHash, key) => {
+      if (!cachedPageData.has(key)) {
+        pageKeys.push(key)
+      } else {
+        const previousPageDataHash = cachedPageData.get(key)
+        if (newPageDataHash !== previousPageDataHash) {
+          pageKeys.push(key)
+        }
+      }
+    })
+    return pageKeys
+  }
+
+  return [...state.pages.keys()]
+}
+
+const collectRemovedPageData = (state, cachedPageData) => {
+  if (cachedPageData && state.pageData) {
+    const deletedPageKeys = []
+    cachedPageData.forEach((_value, key) => {
+      if (!state.pageData.has(key)) {
+        deletedPageKeys.push(key)
+      }
+    })
+    return deletedPageKeys
+  }
+  return []
+}
+
+const checkAndRemoveEmptyDir = (publicDir, pagePath) => {
+  const pageHtmlDirectory = path.dirname(
+    getPageHtmlFilePath(publicDir, pagePath)
+  )
+  const pageDataDirectory = path.join(
+    publicDir,
+    `page-data`,
+    fixedPagePath(pagePath)
+  )
+  const hasFiles = fs.readdirSync(pageHtmlDirectory)
+
+  // if page's html folder is empty also remove matching page-data folder
+  if (!hasFiles.length) {
+    fs.removeSync(pageHtmlDirectory)
+    fs.removeSync(pageDataDirectory)
+  }
+}
+
+const sortedPageKeysByNestedLevel = pageKeys =>
+  pageKeys.sort((a, b) => {
+    const currentPagePathValue = a.split(`/`).length
+    const previousPagePathValue = b.split(`/`).length
+    return previousPagePathValue - currentPagePathValue
+  })
+
+const removePageFiles = ({ publicDir }, pageKeys) => {
+  const removePages = pageKeys.map(pagePath =>
+    removePageHtmlFile({ publicDir }, pagePath)
+  )
+
+  const removePageData = pageKeys.map(pagePath =>
+    removePageDataFile({ publicDir }, pagePath)
+  )
+
+  return Promise.all([...removePages, ...removePageData]).then(() => {
+    // Sort removed pageKeys by nested directories and remove if empty.
+    sortedPageKeysByNestedLevel(pageKeys).forEach(pagePath => {
+      checkAndRemoveEmptyDir(publicDir, pagePath)
+    })
+  })
+}
+
+module.exports = {
+  getChangedPageDataKeys,
+  collectRemovedPageData,
+  removePageFiles,
+}

packages/gatsby/src/commands/build.js

@@ -2,6 +2,7 @@
 
 const path = require(`path`)
 const report = require(`gatsby-cli/lib/reporter`)
+const fs = require(`fs-extra`)
 import { buildHTML } from "./build-html"
 const buildProductionBundle = require(`./build-javascript`)
 const bootstrap = require(`../bootstrap`)
@@ -11,14 +12,25 @@ const { initTracer, stopTracer } = require(`../utils/tracer`)
 const db = require(`../db`)
 const signalExit = require(`signal-exit`)
 const telemetry = require(`gatsby-telemetry`)
-const { store, emitter } = require(`../redux`)
+const { store, emitter, readState } = require(`../redux`)
 const queryUtil = require(`../query`)
 const appDataUtil = require(`../utils/app-data`)
 const WorkerPool = require(`../utils/worker/pool`)
 const { structureWebpackErrors } = require(`../utils/webpack-error-utils`)
 const {
   waitUntilAllJobsComplete: waitUntilAllJobsV2Complete,
 } = require(`../utils/jobs-manager`)
+const buildUtils = require(`../commands/build-utils`)
+const { boundActionCreators } = require(`../redux/actions`)
+
+let cachedPageData
+let cachedWebpackCompilationHash
+if (process.env.GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES) {
+  const { pageData, webpackCompilationHash } = readState()
+  // extract only data that we need to reuse and let v8 garbage collect rest of state
+  cachedPageData = pageData
+  cachedWebpackCompilationHash = webpackCompilationHash
+}
 
 type BuildArgs = {
   directory: string,
@@ -119,6 +131,19 @@ module.exports = async function build(program: BuildArgs) {
 
   await processPageQueries()
 
+  if (process.env.GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES) {
+    const { pages } = store.getState()
+    if (cachedPageData) {
+      cachedPageData.forEach((_value, key) => {
+        if (!pages.has(key)) {
+          boundActionCreators.removePageData({
+            id: key,
+          })
+        }
+      })
+    }
+  }
+
   if (telemetry.isTrackingEnabled()) {
     // transform asset size to kB (from bytes) to fit 64 bit to numbers
     const bundleSizes = stats
@@ -144,7 +169,20 @@ module.exports = async function build(program: BuildArgs) {
   // we need to save it again to make sure our latest state has been saved
   await db.saveState()
 
-  const pagePaths = [...store.getState().pages.keys()]
+  let pagePaths = [...store.getState().pages.keys()]
+
+  // Rebuild subset of pages if user opt into GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES
+  // if there were no source files (for example components, static queries, etc) changes since last build, otherwise rebuild all pages
+  if (
+    process.env.GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES &&
+    cachedWebpackCompilationHash === store.getState().webpackCompilationHash
+  ) {
+    pagePaths = buildUtils.getChangedPageDataKeys(
+      store.getState(),
+      cachedPageData
+    )
+  }
+
   activity = report.createProgress(
     `Building static HTML for pages`,
     pagePaths.length,
@@ -184,6 +222,19 @@ module.exports = async function build(program: BuildArgs) {
   }
   activity.done()
 
+  let deletedPageKeys = []
+  if (process.env.GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES) {
+    activity = report.activityTimer(`Delete previous page data`)
+    activity.start()
+    deletedPageKeys = buildUtils.collectRemovedPageData(
+      store.getState(),
+      cachedPageData
+    )
+    await buildUtils.removePageFiles({ publicDir }, deletedPageKeys)
+
+    activity.end()
+  }
+
   activity = report.activityTimer(`onPostBuild`, { parentSpan: buildSpan })
   activity.start()
   await apiRunnerNode(`onPostBuild`, {
@@ -201,4 +252,52 @@ module.exports = async function build(program: BuildArgs) {
   await stopTracer()
   workerPool.end()
   buildActivity.end()
+
+  if (
+    process.env.GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES &&
+    process.argv.includes(`--log-pages`)
+  ) {
+    if (pagePaths.length) {
+      report.info(
+        `Built pages:\n${pagePaths
+          .map(path => `Updated page: ${path}`)
+          .join(`\n`)}`
+      )
+    }
+
+    if (deletedPageKeys.length) {
+      report.info(
+        `Deleted pages:\n${deletedPageKeys
+          .map(path => `Deleted page: ${path}`)
+          .join(`\n`)}`
+      )
+    }
+  }
+
+  if (
+    process.env.GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES &&
+    process.argv.includes(`--write-to-file`)
+  ) {
+    const createdFilesPath = path.resolve(
+      `${program.directory}/.cache`,
+      `newPages.txt`
+    )
+    const deletedFilesPath = path.resolve(
+      `${program.directory}/.cache`,
+      `deletedPages.txt`
+    )
+
+    if (pagePaths.length) {
+      await fs.writeFile(createdFilesPath, `${pagePaths.join(`\n`)}\n`, `utf8`)
+      report.info(`.cache/newPages.txt created`)
+    }
+    if (deletedPageKeys.length) {
+      await fs.writeFile(
+        deletedFilesPath,
+        `${deletedPageKeys.join(`\n`)}\n`,
+        `utf8`
+      )
+      report.info(`.cache/deletedPages.txt created`)
+    }
+  }
 }

packages/gatsby/src/commands/develop.ts

@@ -350,6 +350,15 @@ async function startServer(program: IProgram): Promise<IServer> {
 }
 
 module.exports = async (program: IProgram): Promise<void> => {
+  if (process.env.GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES) {
+    report.panic(
+      `The flag ${chalk.yellow(
+        `GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES`
+      )} is not available with ${chalk.cyan(
+        `gatsby develop`
+      )}, please retry using ${chalk.cyan(`gatsby build`)}`
+    )
+  }
   initTracer(program.openTracingConfigFile)
   report.pendingActivity({ id: `webpack-develop` })
   telemetry.trackCli(`DEVELOP_START`)
@@ -407,7 +416,6 @@ module.exports = async (program: IProgram): Promise<void> => {
   require(`../redux/actions`).boundActionCreators.setProgramStatus(
     `BOOTSTRAP_QUERY_RUNNING_FINISHED`
   )
-
   await db.saveState()
 
   await waitUntilAllJobsComplete()

packages/gatsby/src/query/query-runner.js

@@ -98,7 +98,6 @@ module.exports = async (graphqlRunner, queryJob: QueryJob) => {
     .createHash(`sha1`)
     .update(resultJSON)
     .digest(`base64`)
-
   if (resultHash !== resultHashes.get(queryJob.id)) {
     resultHashes.set(queryJob.id, resultHash)
 
@@ -117,7 +116,6 @@ module.exports = async (graphqlRunner, queryJob: QueryJob) => {
         `d`,
         `${queryJob.hash}.json`
       )
-
       await fs.outputFile(resultPath, resultJSON)
     }
   }
@@ -128,5 +126,15 @@ module.exports = async (graphqlRunner, queryJob: QueryJob) => {
     isPage: queryJob.isPage,
   })
 
+  // Sets pageData to the store, here for easier access to the resultHash
+  if (
+    process.env.GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES &&
+    queryJob.isPage
+  ) {
+    boundActionCreators.setPageData({
+      id: queryJob.id,
+      resultHash,
+    })
+  }
   return result
 }

packages/gatsby/src/redux/__tests__/__snapshots__/index.js.snap

@@ -20,6 +20,7 @@ Object {
     "complete": Map {},
     "incomplete": Map {},
   },
+  "pageData": Map {},
   "pageDataStats": Map {},
   "staticQueryComponents": Map {},
   "status": Object {