Skip to content

Commit

Permalink
chore(gatsby-source-contentful): move to pluginOptionsSchema (#27322)
Browse files Browse the repository at this point in the history 
* Move gatsby-source-contentful to pluginOptionsSchema

* Remove old validation options tests

* Better error message in case of failed external Contentful validation

* Restore old validation, wrap new one in env var

* use preinit and overwrite defaults

* remove console.log

* fix fetch

Co-authored-by: Abhi Aiyer <abhiaiyer91@gmail.com>
Co-authored-by: Ward Peeters <ward@coding-tech.com>
  • Loading branch information
@mxstbr @abhiaiyer91 @wardpeet
3 people authored Oct 14, 2020
1 parent d6f033e commit ae9f9a9
Show file tree
Hide file tree
Showing 4 changed files with 143 additions and 202 deletions.
2 changes: 2 additions & 0 deletions packages/gatsby-source-contentful/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,12 @@
"contentful": "^7.14.7",
"fs-extra": "^9.0.1",
"gatsby-core-utils": "^1.3.23",
"gatsby-plugin-utils": "^0.2.27",
"gatsby-source-filesystem": "^2.3.34",
"is-online": "^8.5.0",
"json-stringify-safe": "^5.0.1",
"lodash": "^4.17.20",
"node-fetch": "^2.6.1",
"progress": "^2.0.3",
"qs": "^6.9.4"
},
Expand Down
159 changes: 1 addition & 158 deletions packages/gatsby-source-contentful/src/__tests__/plugin-options.js
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,7 @@
// disable output coloring for tests
process.env.FORCE_COLOR = 0

const {
maskText,
validateOptions,
formatPluginOptionsForCLI,
} = require(`../plugin-options`)
const { maskText, formatPluginOptionsForCLI } = require(`../plugin-options`)

const maskedCharacterCount = input =>
input.split(``).filter(char => char === `*`).length
Expand Down Expand Up @@ -78,156 +74,3 @@ describe(`Formatting plugin options for CLI`, () => {
)
})
})

describe(`Options validation`, () => {
const reporter = {
panic: jest.fn(),
}

beforeEach(() => {
reporter.panic.mockClear()
})

it(`Passes with valid options`, () => {
validateOptions(
{
reporter,
},
{
spaceId: `spaceId`,
accessToken: `accessToken`,
localeFilter: locale => locale.code === `de`,
downloadLocal: false,
}
)

expect(reporter.panic).not.toBeCalled()
})

it(`Fails with missing required options`, () => {
validateOptions(
{
reporter,
},
{}
)

expect(reporter.panic).toBeCalledWith(
expect.stringContaining(
`Problems with gatsby-source-contentful plugin options`
)
)
expect(reporter.panic).toBeCalledWith(
expect.stringContaining(`"accessToken" is required`)
)
expect(reporter.panic).toBeCalledWith(
expect.stringContaining(`"accessToken" is required`)
)
})

it(`Fails with empty options`, () => {
validateOptions(
{
reporter,
},
{
environment: ``,
host: ``,
accessToken: ``,
spaceId: ``,
}
)

expect(reporter.panic).toBeCalledWith(
expect.stringContaining(
`Problems with gatsby-source-contentful plugin options`
)
)
expect(reporter.panic).toBeCalledWith(
expect.stringContaining(`"environment" is not allowed to be empty`)
)
expect(reporter.panic).toBeCalledWith(
expect.stringContaining(`"host" is not allowed to be empty`)
)
expect(reporter.panic).toBeCalledWith(
expect.stringContaining(`"accessToken" is not allowed to be empty`)
)
expect(reporter.panic).toBeCalledWith(
expect.stringContaining(`"spaceId" is not allowed to be empty`)
)
})

it(`Fails with options of wrong types`, () => {
validateOptions(
{
reporter,
},
{
environment: 1,
host: [],
accessToken: true,
spaceId: {},
localeFilter: `yup`,
downloadLocal: 5,
useNameForId: 5,
pageLimit: `fifty`,
richText: true,
}
)

expect(reporter.panic).toBeCalledWith(
expect.stringContaining(
`Problems with gatsby-source-contentful plugin options`
)
)
expect(reporter.panic).toBeCalledWith(
expect.stringContaining(`"environment" must be a string`)
)
expect(reporter.panic).toBeCalledWith(
expect.stringContaining(`"host" must be a string`)
)
expect(reporter.panic).toBeCalledWith(
expect.stringContaining(`"accessToken" must be a string`)
)
expect(reporter.panic).toBeCalledWith(
expect.stringContaining(`"spaceId" must be a string`)
)
expect(reporter.panic).toBeCalledWith(
expect.stringContaining(`"localeFilter" must be a Function`)
)
expect(reporter.panic).toBeCalledWith(
expect.stringContaining(`"downloadLocal" must be a boolean`)
)
expect(reporter.panic).toBeCalledWith(
expect.stringContaining(`"useNameForId" must be a boolean`)
)
expect(reporter.panic).toBeCalledWith(
expect.stringContaining(`"pageLimit" must be a number`)
)
expect(reporter.panic).toBeCalledWith(
expect.stringContaining(`"richText" must be an object`)
)
})

it(`Fails with undefined option keys`, () => {
validateOptions(
{
reporter,
},
{
spaceId: `spaceId`,
accessToken: `accessToken`,
wat: true,
}
)

expect(reporter.panic).toBeCalledWith(
expect.stringContaining(
`Problems with gatsby-source-contentful plugin options`
)
)
expect(reporter.panic).toBeCalledWith(
expect.stringContaining(`"wat" is not allowed`)
)
})
})
143 changes: 140 additions & 3 deletions packages/gatsby-source-contentful/src/gatsby-node.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,16 @@ const _ = require(`lodash`)
const fs = require(`fs-extra`)
const { createClient } = require(`contentful`)
const v8 = require(`v8`)
const fetch = require(`node-fetch`)
const { Joi } = require(`gatsby-plugin-utils`)

const normalize = require(`./normalize`)
const fetchData = require(`./fetch`)
const { createPluginConfig, validateOptions } = require(`./plugin-options`)
const {
createPluginConfig,
maskText,
formatPluginOptionsForCLI,
} = require(`./plugin-options`)
const { downloadContentfulAssets } = require(`./download-contentful-assets`)

const conflictFieldPrefix = `contentful`
Expand All @@ -24,7 +30,138 @@ const restrictedNodeFields = [

exports.setFieldsOnGraphQLNodeType = require(`./extend-node-type`).extendNodeType

exports.onPreBootstrap = validateOptions
// TODO: Remove once pluginOptionsSchema is stable
exports.onPreInit = ({ reporter }, options) => {
const result = pluginOptionsSchema({ Joi }).validate(options, {
abortEarly: false,
externals: false,
})
if (result.error) {
const errors = {}
result.error.details.forEach(detail => {
errors[detail.path[0]] = detail.message
})
reporter.panic(`Problems with gatsby-source-contentful plugin options:
${formatPluginOptionsForCLI(options, errors)}`)
}

options = result.value
}

const validateContentfulAccess = async pluginOptions => {
if (process.env.NODE_ENV === `test`) return undefined

await fetch(`https://${pluginOptions.host}/spaces/${pluginOptions.spaceId}`, {
headers: {
Authorization: `Bearer ${pluginOptions.accessToken}`,
"Content-Type": `application/json`,
},
})
.then(res => res.ok)
.then(ok => {
if (!ok)
throw new Error(
`Cannot access Contentful space "${maskText(
pluginOptions.spaceId
)}" with access token "${maskText(
pluginOptions.accessToken
)}". Make sure to double check them!`
)
})

return undefined
}

const pluginOptionsSchema = ({ Joi }) =>
Joi.object()
.keys({
accessToken: Joi.string()
.description(
`Contentful delivery api key, when using the Preview API use your Preview API key`
)
.required()
.empty(),
spaceId: Joi.string()
.description(`Contentful spaceId`)
.required()
.empty(),
host: Joi.string()
.description(
`The base host for all the API requests, by default it's 'cdn.contentful.com', if you want to use the Preview API set it to 'preview.contentful.com'. You can use your own host for debugging/testing purposes as long as you respect the same Contentful JSON structure.`
)
.default(`cdn.contentful.com`)
.empty(),
environment: Joi.string()
.description(
`The environment to pull the content from, for more info on environments check out this [Guide](https://www.contentful.com/developers/docs/concepts/multiple-environments/).`
)
.default(`master`)
.empty(),
downloadLocal: Joi.boolean()
.description(
`Downloads and caches ContentfulAsset's to the local filesystem. Allows you to query a ContentfulAsset's localFile field, which is not linked to Contentful's CDN. Useful for reducing data usage.
You can pass in any other options available in the [contentful.js SDK](https://github.com/contentful/contentful.js#configuration).`
)
.default(false),
localeFilter: Joi.func()
.description(
`Possibility to limit how many locales/nodes are created in GraphQL. This can limit the memory usage by reducing the amount of nodes created. Useful if you have a large space in contentful and only want to get the data from one selected locale.
For example, to filter locales on only germany \`localeFilter: locale => locale.code === 'de-DE'\`
List of locales and their codes can be found in Contentful app -> Settings -> Locales`
)
.default(() => true),
forceFullSync: Joi.boolean()
.description(
`Prevents the use of sync tokens when accessing the Contentful API.`
)
.default(false),
pageLimit: Joi.number()
.integer()
.description(
`Number of entries to retrieve from Contentful at a time. Due to some technical limitations, the response payload should not be greater than 7MB when pulling content from Contentful. If you encounter this issue you can set this param to a lower number than 100, e.g 50.`
)
.default(100),
proxy: Joi.object()
.keys({
host: Joi.string().required(),
port: Joi.number().required(),
auth: Joi.object().keys({
username: Joi.string(),
password: Joi.string(),
}),
})
.description(
`Axios proxy configuration. See the [axios request config documentation](https://github.com/mzabriskie/axios#request-config) for further information about the supported values.`
),
useNameForId: Joi.boolean()
.description(
`Use the content's \`name\` when generating the GraphQL schema e.g. a Content Type called \`[Component] Navigation bar\` will be named \`contentfulComponentNavigationBar\`.
When set to \`false\`, the content's internal ID will be used instead e.g. a Content Type with the ID \`navigationBar\` will be called \`contentfulNavigationBar\`.
Using the ID is a much more stable property to work with as it will change less often. However, in some scenarios, Content Types' IDs will be auto-generated (e.g. when creating a new Content Type without specifying an ID) which means the name in the GraphQL schema will be something like \`contentfulC6XwpTaSiiI2Ak2Ww0oi6qa\`. This won't change and will still function perfectly as a valid field name but it is obviously pretty ugly to work with.
If you are confident your Content Types will have natural-language IDs (e.g. \`blogPost\`), then you should set this option to \`false\`. If you are unable to ensure this, then you should leave this option set to \`true\` (the default).`
)
.default(true),
// default plugins passed by gatsby
plugins: Joi.array(),
richText: Joi.object()
.keys({
resolveFieldLocales: Joi.boolean()
.description(
`If you want to resolve the locales in fields of assets and entries that are referenced by rich text (e.g., via embedded entries or entry hyperlinks), set this to \`true\`. Otherwise, fields of referenced assets or entries will be objects keyed by locale.`
)
.default(false),
})
.default({}),
})
.external(validateContentfulAccess)

if (process.env.GATSBY_EXPERIMENTAL_PLUGIN_OPTION_VALIDATION) {
exports.pluginOptionsSchema = pluginOptionsSchema
}

/***
* Localization algorithm
*
Expand Down Expand Up @@ -151,7 +288,7 @@ exports.sourceNodes = async (
parentSpan,
}
)
console.log(`runs through this part anyways`)

fetchActivity.start()
;({
currentSyncData,
Expand Down
Loading

0 comments on commit ae9f9a9

Please sign in to comment.