feat(gatsby-source-contentful): Support storing assets locally

packages/gatsby-source-contentful/README.md

@@ -52,6 +52,70 @@ module.exports = {
 }
 ```
 
+### Download assets for static distribution
+
+Downloads and caches Contentful Assets to the local filesystem. Useful for reduced data usage in development or projects where you want the assets copied locally with builds for deploying without links to Contentful's CDN.
+
+Enable this feature with the `downloadLocal: true` option.
+
+```javascript
+// In your gatsby-config.js
+module.exports = {
+  plugins: [
+    {
+      resolve: `gatsby-source-contentful`,
+      options: {
+        spaceId: `your_space_id`,
+        // Learn about environment variables: https://gatsby.app/env-vars
+        accessToken: process.env.CONTENTFUL_ACCESS_TOKEN,
+        downloadLocal: true,
+      },
+    },
+  ],
+}
+```
+
+Query a `ContentfulAsset`'s `localFile` field in GraphQL to gain access to the common fields of the `gatsby-source-filesystem` `File` node. This is not a Contentful node, so usage for `gatsby-image` is different:
+
+```GraphQL
+graphql`
+  query MyQuery {
+    # Example is for a `ContentType` with a `ContenfulAsset` field
+    # You could also query an asset directly via
+    # `allContentfulAsset { edges{ node { } } }`
+    # or `contentfulAsset(contentful_id: { eq: "contentful_id here" } ) { }`
+    contentfulMyContentType {
+      myContentfulAssetField {
+        # Direct URL to Contentful CDN for this asset
+        file { url }
+
+        # Query for a fluid image resource on this `ContentfulAsset` node
+        fluid(maxWidth: 500){
+          ...GatsbyContentfulFluid_withWebp
+        }
+
+        # Query for locally stored file(eg An image) - `File` node
+        localFile {
+          # Where the asset is downloaded into cache, don't use this
+          absolutePath
+          # Where the asset is copied to for distribution, equivalent to using ContentfulAsset `file {url}`
+          publicURL
+          # Use `gatsby-image` to create fluid image resource
+          childImageSharp {
+            fluid(maxWidth: 500) {
+              ...GatsbyImageSharpFluid
+          }
+        }
+      }
+    }
+  }
+`
+```
+
+Note: This feature downloads any file from a `ContentfulAsset` node that `gatsby-source-contentful` provides. They are all copied over from `./cache/gatsby-source-filesystem/` to the sites build location `./public/static/`.
+
+For any troubleshooting related to this feature, first try clearing your `./cache/` directory. `gatsby-source-contentful` will acquire fresh data, and all `ContentfulAsset`s will be downloaded and cached again.
+
 ### Offline
 
 If you don't have internet connection you can add `export GATSBY_CONTENTFUL_OFFLINE=true` to tell the plugin to fallback to the cached data, if there is any.

packages/gatsby-source-contentful/src/download-contentful-assets.js

@@ -0,0 +1,86 @@
+const ProgressBar = require(`progress`)
+const { createRemoteFileNode } = require(`gatsby-source-filesystem`)
+
+const bar = new ProgressBar(
+  `Downloading Contentful Assets [:bar] :current/:total :elapsed secs :percent`,
+  {
+    total: 0,
+    width: 30,
+  }
+)
+
+let totalJobs = 0
+
+/**
+ * @name downloadContentfulAssets
+ * @description Downloads Contentful assets to the local filesystem.
+ * The asset files will be downloaded and cached. Use `localFile` to link to them
+ * @param gatsbyFunctions - Gatsby's internal helper functions
+ */
+
+const downloadContentfulAssets = async gatsbyFunctions => {
+  const {
+    actions: { createNode, touchNode },
+    createNodeId,
+    store,
+    cache,
+    getNodes,
+  } = gatsbyFunctions
+
+  // Any ContentfulAsset nodes will be downloaded, cached and copied to public/static
+  // regardless of if you use `localFile` to link an asset or not.
+  const contentfulAssetNodes = getNodes().filter(
+    n =>
+      n.internal.owner === `gatsby-source-contentful` &&
+      n.internal.type === `ContentfulAsset`
+  )
+
+  await Promise.all(
+    contentfulAssetNodes.map(async node => {
+      totalJobs += 1
+      bar.total = totalJobs
+
+      let fileNodeID
+      const remoteDataCacheKey = `contentful-asset-${node.contentful_id}`
+      const cacheRemoteData = await cache.get(remoteDataCacheKey)
+      const url = `http://${node.file.url.slice(2)}`
+
+      // Avoid downloading the asset again if it's been cached
+      // Note: Contentful Assets do not provide useful metadata
+      // to compare a modified asset to a cached version?
+      if (cacheRemoteData) {
+        fileNodeID = cacheRemoteData.fileNodeID // eslint-disable-line prefer-destructuring
+        touchNode({ nodeId: cacheRemoteData.fileNodeID })
+      }
+
+      // If we don't have cached data, download the file
+      if (!fileNodeID) {
+        try {
+          const fileNode = await createRemoteFileNode({
+            url,
+            store,
+            cache,
+            createNode,
+            createNodeId,
+          })
+
+          if (fileNode) {
+            bar.tick()
+            fileNodeID = fileNode.id
+
+            await cache.set(remoteDataCacheKey, { fileNodeID })
+          }
+        } catch (err) {
+          // Ignore
+        }
+      }
+
+      if (fileNodeID) {
+        node.localFile___NODE = fileNodeID
+      }
+
+      return node
+    })
+  )
+}
+exports.downloadContentfulAssets = downloadContentfulAssets

packages/gatsby-source-contentful/src/gatsby-node.js

@@ -5,6 +5,7 @@ const fs = require(`fs-extra`)
 
 const normalize = require(`./normalize`)
 const fetchData = require(`./fetch`)
+const { downloadContentfulAssets } = require(`./download-contentful-assets`)
 
 const conflictFieldPrefix = `contentful`
 
@@ -32,7 +33,7 @@ exports.setFieldsOnGraphQLNodeType = require(`./extend-node-type`).extendNodeTyp
  */
 
 exports.sourceNodes = async (
-  { actions, getNode, getNodes, createNodeId, hasNodeChanged, store },
+  { actions, getNode, getNodes, createNodeId, hasNodeChanged, store, cache },
   options
 ) => {
   const { createNode, deleteNode, touchNode, setPluginStatus } = actions
@@ -210,6 +211,16 @@ exports.sourceNodes = async (
     })
   })
 
+  if (options.downloadLocal) {
+    await downloadContentfulAssets({
+      actions,
+      createNodeId,
+      store,
+      cache,
+      getNodes,
+    })
+  }
+
   return
 }