From 99158a90b6049fa140be5f84ebbb68f802440880 Mon Sep 17 00:00:00 2001 From: polarathene Date: Thu, 27 Dec 2018 22:30:45 +0800 Subject: [PATCH 1/4] feat(gatsby-source-contentful): Local Assets Downloads and caches Contentful Assets to the local filesystem. Useful for reduced data usage or projects where you want the assets copied locally with builds for deploying without links to Contentful CDN. --- .../src/download-contentful-assets.js | 82 +++++++++++++++++++ .../src/gatsby-node.js | 13 ++- 2 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 packages/gatsby-source-contentful/src/download-contentful-assets.js diff --git a/packages/gatsby-source-contentful/src/download-contentful-assets.js b/packages/gatsby-source-contentful/src/download-contentful-assets.js new file mode 100644 index 0000000000000..3698ca642d776 --- /dev/null +++ b/packages/gatsby-source-contentful/src/download-contentful-assets.js @@ -0,0 +1,82 @@ +const ProgressBar = require('progress') +const { createRemoteFileNode } = require('gatsby-source-filesystem') + +const bar = new ProgressBar(`Downloading Contentful Assets [:bar] :current/:total :elapsed secs :percent`, { + total: 0, + width: 30, +}) + +let totalJobs = 0 + +/** + * @name downloadContentfulAssets + * @description Downloads Contentful assets to the local filesystem. + * The asset files will be downloaded and cached. Use `localFile` to link to them + * @param gatsbyFunctions - Gatsby's internal helper functions + */ + +const downloadContentfulAssets = async (gatsbyFunctions) => { + const { + actions: { createNode, touchNode }, + createNodeId, + store, + cache, + getNodes, + } = gatsbyFunctions + + // Any ContentfulAsset nodes will be downloaded, cached and copied to public/static + // regardless of if you use `localFile` to link an asset or not. + const contentfulAssetNodes = getNodes() + .filter(n => + (n.internal.owner === `gatsby-source-contentful`) && + (n.internal.type === `ContentfulAsset`) + ) + + await Promise.all(contentfulAssetNodes.map(async node => { + + totalJobs += 1 + bar.total = totalJobs + + let fileNodeID + const remoteDataCacheKey = `contentful-asset-${node.contentful_id}` + const cacheRemoteData = await cache.get(remoteDataCacheKey) + const url = "http://" + node.file.url.slice(2) + + // Avoid downloading the asset again if it's been cached + // Note: Contentful Assets do not provide useful metadata + // to compare a modified asset to a cached version? + if (cacheRemoteData) { + fileNodeID = cacheRemoteData.fileNodeID // eslint-disable-line prefer-destructuring + touchNode({ nodeId: cacheRemoteData.fileNodeID }) + } + + // If we don't have cached data, download the file + if (!fileNodeID) { + try { + const fileNode = await createRemoteFileNode({ + url, + store, + cache, + createNode, + createNodeId, + }) + + if (fileNode) { + bar.tick() + fileNodeID = fileNode.id + + await cache.set(remoteDataCacheKey, { fileNodeID }) + } + } catch (err) { + // Ignore + } + } + + if (fileNodeID) { + node.localFile___NODE = fileNodeID + } + + return node + })) +} +exports.downloadContentfulAssets = downloadContentfulAssets diff --git a/packages/gatsby-source-contentful/src/gatsby-node.js b/packages/gatsby-source-contentful/src/gatsby-node.js index ac425b77f332c..37356e477ea79 100644 --- a/packages/gatsby-source-contentful/src/gatsby-node.js +++ b/packages/gatsby-source-contentful/src/gatsby-node.js @@ -5,6 +5,7 @@ const fs = require(`fs-extra`) const normalize = require(`./normalize`) const fetchData = require(`./fetch`) +const { downloadContentfulAssets } = require(`./downloadContentfulAssets`) const conflictFieldPrefix = `contentful` @@ -32,7 +33,7 @@ exports.setFieldsOnGraphQLNodeType = require(`./extend-node-type`).extendNodeTyp */ exports.sourceNodes = async ( - { actions, getNode, getNodes, createNodeId, hasNodeChanged, store }, + { actions, getNode, getNodes, createNodeId, hasNodeChanged, store, cache }, options ) => { const { createNode, deleteNode, touchNode, setPluginStatus } = actions @@ -210,6 +211,16 @@ exports.sourceNodes = async ( }) }) + if (options.downloadLocal) { + await downloadContentfulAssets({ + actions, + createNodeId, + store, + cache, + getNodes, + }) + } + return } From 6afbb5487396a75d72c1fc1d3f6060d1e3ac690f Mon Sep 17 00:00:00 2001 From: Brennan Kinney Date: Thu, 27 Dec 2018 23:31:18 +0800 Subject: [PATCH 2/4] Typo on import Filename was updated to match naming convention, missed updating it's import statement in `gatsby-node.js`. --- packages/gatsby-source-contentful/src/gatsby-node.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gatsby-source-contentful/src/gatsby-node.js b/packages/gatsby-source-contentful/src/gatsby-node.js index 37356e477ea79..75f39acb2b0b9 100644 --- a/packages/gatsby-source-contentful/src/gatsby-node.js +++ b/packages/gatsby-source-contentful/src/gatsby-node.js @@ -5,7 +5,7 @@ const fs = require(`fs-extra`) const normalize = require(`./normalize`) const fetchData = require(`./fetch`) -const { downloadContentfulAssets } = require(`./downloadContentfulAssets`) +const { downloadContentfulAssets } = require(`./download-contentful-assets`) const conflictFieldPrefix = `contentful` From c24051d67af18efbcf84325f45b49f13f17dbdf5 Mon Sep 17 00:00:00 2001 From: polarathene Date: Fri, 28 Dec 2018 01:51:34 +0800 Subject: [PATCH 3/4] Appease CI lint errors --- .../src/download-contentful-assets.js | 104 +++++++++--------- 1 file changed, 54 insertions(+), 50 deletions(-) diff --git a/packages/gatsby-source-contentful/src/download-contentful-assets.js b/packages/gatsby-source-contentful/src/download-contentful-assets.js index 3698ca642d776..9695e74f318ec 100644 --- a/packages/gatsby-source-contentful/src/download-contentful-assets.js +++ b/packages/gatsby-source-contentful/src/download-contentful-assets.js @@ -1,10 +1,13 @@ -const ProgressBar = require('progress') -const { createRemoteFileNode } = require('gatsby-source-filesystem') +const ProgressBar = require(`progress`) +const { createRemoteFileNode } = require(`gatsby-source-filesystem`) -const bar = new ProgressBar(`Downloading Contentful Assets [:bar] :current/:total :elapsed secs :percent`, { - total: 0, - width: 30, -}) +const bar = new ProgressBar( + `Downloading Contentful Assets [:bar] :current/:total :elapsed secs :percent`, + { + total: 0, + width: 30, + } +) let totalJobs = 0 @@ -15,7 +18,7 @@ let totalJobs = 0 * @param gatsbyFunctions - Gatsby's internal helper functions */ -const downloadContentfulAssets = async (gatsbyFunctions) => { +const downloadContentfulAssets = async gatsbyFunctions => { const { actions: { createNode, touchNode }, createNodeId, @@ -26,57 +29,58 @@ const downloadContentfulAssets = async (gatsbyFunctions) => { // Any ContentfulAsset nodes will be downloaded, cached and copied to public/static // regardless of if you use `localFile` to link an asset or not. - const contentfulAssetNodes = getNodes() - .filter(n => - (n.internal.owner === `gatsby-source-contentful`) && - (n.internal.type === `ContentfulAsset`) - ) - - await Promise.all(contentfulAssetNodes.map(async node => { + const contentfulAssetNodes = getNodes().filter( + n => + n.internal.owner === `gatsby-source-contentful` && + n.internal.type === `ContentfulAsset` + ) - totalJobs += 1 - bar.total = totalJobs + await Promise.all( + contentfulAssetNodes.map(async node => { + totalJobs += 1 + bar.total = totalJobs - let fileNodeID - const remoteDataCacheKey = `contentful-asset-${node.contentful_id}` - const cacheRemoteData = await cache.get(remoteDataCacheKey) - const url = "http://" + node.file.url.slice(2) + let fileNodeID + const remoteDataCacheKey = `contentful-asset-${node.contentful_id}` + const cacheRemoteData = await cache.get(remoteDataCacheKey) + const url = `http://${node.file.url.slice(2)}` - // Avoid downloading the asset again if it's been cached - // Note: Contentful Assets do not provide useful metadata - // to compare a modified asset to a cached version? - if (cacheRemoteData) { - fileNodeID = cacheRemoteData.fileNodeID // eslint-disable-line prefer-destructuring - touchNode({ nodeId: cacheRemoteData.fileNodeID }) - } + // Avoid downloading the asset again if it's been cached + // Note: Contentful Assets do not provide useful metadata + // to compare a modified asset to a cached version? + if (cacheRemoteData) { + fileNodeID = cacheRemoteData.fileNodeID // eslint-disable-line prefer-destructuring + touchNode({ nodeId: cacheRemoteData.fileNodeID }) + } - // If we don't have cached data, download the file - if (!fileNodeID) { - try { - const fileNode = await createRemoteFileNode({ - url, - store, - cache, - createNode, - createNodeId, - }) + // If we don't have cached data, download the file + if (!fileNodeID) { + try { + const fileNode = await createRemoteFileNode({ + url, + store, + cache, + createNode, + createNodeId, + }) - if (fileNode) { - bar.tick() - fileNodeID = fileNode.id + if (fileNode) { + bar.tick() + fileNodeID = fileNode.id - await cache.set(remoteDataCacheKey, { fileNodeID }) + await cache.set(remoteDataCacheKey, { fileNodeID }) + } + } catch (err) { + // Ignore + } } - } catch (err) { - // Ignore - } - } - if (fileNodeID) { - node.localFile___NODE = fileNodeID - } + if (fileNodeID) { + node.localFile___NODE = fileNodeID + } - return node - })) + return node + }) + ) } exports.downloadContentfulAssets = downloadContentfulAssets From 3e39670c4c083ecb54ba8490d5d40e16a0f2ee59 Mon Sep 17 00:00:00 2001 From: polarathene Date: Fri, 28 Dec 2018 13:38:17 +0800 Subject: [PATCH 4/4] Add Documentation to README.md --- packages/gatsby-source-contentful/README.md | 64 +++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/packages/gatsby-source-contentful/README.md b/packages/gatsby-source-contentful/README.md index c526b82108e39..a483a666ffb8e 100644 --- a/packages/gatsby-source-contentful/README.md +++ b/packages/gatsby-source-contentful/README.md @@ -52,6 +52,70 @@ module.exports = { } ``` +### Download assets for static distribution + +Downloads and caches Contentful Assets to the local filesystem. Useful for reduced data usage in development or projects where you want the assets copied locally with builds for deploying without links to Contentful's CDN. + +Enable this feature with the `downloadLocal: true` option. + +```javascript +// In your gatsby-config.js +module.exports = { + plugins: [ + { + resolve: `gatsby-source-contentful`, + options: { + spaceId: `your_space_id`, + // Learn about environment variables: https://gatsby.app/env-vars + accessToken: process.env.CONTENTFUL_ACCESS_TOKEN, + downloadLocal: true, + }, + }, + ], +} +``` + +Query a `ContentfulAsset`'s `localFile` field in GraphQL to gain access to the common fields of the `gatsby-source-filesystem` `File` node. This is not a Contentful node, so usage for `gatsby-image` is different: + +```GraphQL +graphql` + query MyQuery { + # Example is for a `ContentType` with a `ContenfulAsset` field + # You could also query an asset directly via + # `allContentfulAsset { edges{ node { } } }` + # or `contentfulAsset(contentful_id: { eq: "contentful_id here" } ) { }` + contentfulMyContentType { + myContentfulAssetField { + # Direct URL to Contentful CDN for this asset + file { url } + + # Query for a fluid image resource on this `ContentfulAsset` node + fluid(maxWidth: 500){ + ...GatsbyContentfulFluid_withWebp + } + + # Query for locally stored file(eg An image) - `File` node + localFile { + # Where the asset is downloaded into cache, don't use this + absolutePath + # Where the asset is copied to for distribution, equivalent to using ContentfulAsset `file {url}` + publicURL + # Use `gatsby-image` to create fluid image resource + childImageSharp { + fluid(maxWidth: 500) { + ...GatsbyImageSharpFluid + } + } + } + } + } +` +``` + +Note: This feature downloads any file from a `ContentfulAsset` node that `gatsby-source-contentful` provides. They are all copied over from `./cache/gatsby-source-filesystem/` to the sites build location `./public/static/`. + +For any troubleshooting related to this feature, first try clearing your `./cache/` directory. `gatsby-source-contentful` will acquire fresh data, and all `ContentfulAsset`s will be downloaded and cached again. + ### Offline If you don't have internet connection you can add `export GATSBY_CONTENTFUL_OFFLINE=true` to tell the plugin to fallback to the cached data, if there is any.