From c1cf1ce0017078df5ae41e8c1581dcd1037870ce Mon Sep 17 00:00:00 2001 From: Jairo <68893868+jairo-bc@users.noreply.github.com> Date: Fri, 19 Nov 2021 14:17:25 +0200 Subject: [PATCH] fix: strf-4307 Frontmatter/yaml validation and trailing symbols checks (#798) --- lib/bundle-validator.js | 64 +++++++++++ lib/bundle-validator.spec.js | 15 ++- lib/utils/frontmatter.js | 33 ++++++ package-lock.json | 81 +++++++++++-- package.json | 1 + server/plugins/renderer/renderer.module.js | 22 ++-- .../themes/invalid-frontmatter/config.json | 106 ++++++++++++++++++ .../themes/invalid-frontmatter/lang/en.json | 20 ++++ .../themes/invalid-frontmatter/schema.json | 103 +++++++++++++++++ .../schemaTranslations.json | 5 + .../templates/components/a.html | 1 + .../templates/components/b.html | 1 + .../templates/pages/page.html | 18 +++ .../templates/pages/page2.html | 12 ++ 14 files changed, 460 insertions(+), 22 deletions(-) create mode 100644 lib/utils/frontmatter.js create mode 100644 test/_mocks/themes/invalid-frontmatter/config.json create mode 100644 test/_mocks/themes/invalid-frontmatter/lang/en.json create mode 100644 test/_mocks/themes/invalid-frontmatter/schema.json create mode 100644 test/_mocks/themes/invalid-frontmatter/schemaTranslations.json create mode 100644 test/_mocks/themes/invalid-frontmatter/templates/components/a.html create mode 100644 test/_mocks/themes/invalid-frontmatter/templates/components/b.html create mode 100644 test/_mocks/themes/invalid-frontmatter/templates/pages/page.html create mode 100644 test/_mocks/themes/invalid-frontmatter/templates/pages/page2.html diff --git a/lib/bundle-validator.js b/lib/bundle-validator.js index 14155f24..94efff45 100644 --- a/lib/bundle-validator.js +++ b/lib/bundle-validator.js @@ -6,7 +6,10 @@ const fs = require('fs'); const sizeOf = require('image-size'); const path = require('path'); const Validator = require('ajv'); +const yamlValidator = require('js-yaml'); +const { recursiveReadDir } = require('./utils/fsUtils'); +const { getFrontmatterContent, interpolateThemeSettings } = require('./utils/frontmatter'); const ValidatorSchemaTranslations = require('./validator/schema-translations'); const privateThemeConfigValidationSchema = require('./schemas/privateThemeConfig.json'); const themeConfigValidationSchema = require('./schemas/themeConfig.json'); @@ -43,6 +46,7 @@ class BundleValidator { this._validateThemeConfiguration.bind(this), this._validateThemeSchema.bind(this), this._validateSchemaTranslations.bind(this), + this._validateTemplatesFrontmatter.bind(this), ]; if (!this.isPrivate) { @@ -371,6 +375,66 @@ class BundleValidator { cb(null, true); }); } + + async _validateTemplatesFrontmatter() { + const config = await this.themeConfig.getRawConfig(); + const filePaths = await recursiveReadDir(path.join(this.themePath, 'templates'), [ + '!*.html', + ]); + + for await (const filePath of filePaths) { + const fileContent = await fs.promises.readFile(filePath, { encoding: 'utf-8' }); + const frontmatter = getFrontmatterContent(fileContent); + if (frontmatter) { + const yaml = interpolateThemeSettings(frontmatter, config.settings); + + try { + const result = yamlValidator.loadAll(yaml); + this.validateTrailingSymbols(result[0]); + if (filePath.includes('home.html')) { + console.log(filePath); + console.log(yaml); + console.log(result[0].products.new.limit); + } + } catch (e) { + throw new Error( + `Error: ${e.message}, while parsing frontmatter at "${filePath}".`.red, + ); + } + } + } + + return true; + } + + validateTrailingSymbols(data) { + if (_.isObject(data)) { + return _.every(data, (value) => this.validateTrailingSymbols(value)); + } + if (_.isArray(data)) { + return data.every((row) => this.validateTrailingSymbols(row)); + } + + return data ? this.hasFrontmatterValidValue(data) : true; + } + + hasFrontmatterValidValue(value) { + if (this.hasUnallowedTrailingSymbol(value)) { + throw new Error(`Found unallowed trailing symbol in: "${value}"`); + } + + return true; + } + + getUnallowedTrailingSymbols() { + return [',', ';']; + } + + hasUnallowedTrailingSymbol(value) { + const symbols = this.getUnallowedTrailingSymbols(); + const trailingSymbol = value.toString().trim().slice(-1); + return symbols.includes(trailingSymbol); + } } module.exports = BundleValidator; diff --git a/lib/bundle-validator.spec.js b/lib/bundle-validator.spec.js index 057460d5..b663c96e 100644 --- a/lib/bundle-validator.spec.js +++ b/lib/bundle-validator.spec.js @@ -96,12 +96,12 @@ describe('BundleValidator', () => { ).rejects.toThrow('Missing required objects/properties: footer.scripts'); }); - it('should validate theme schema successfully', async () => { + it('should validate theme schema and frontmatter successfully', async () => { const validator = new BundleValidator(themePath, themeConfig, false); const res = await promisify(validator.validateTheme.bind(validator))(); - expect(res).toHaveLength(4); // 4 validation tasks + expect(res).toHaveLength(5); // 5 validation tasks expect(res).not.toContain(false); }); @@ -115,4 +115,15 @@ describe('BundleValidator', () => { "schema[0].settings[0] should have required property 'content'", ); }); + + it('should validate theme frontmatter and throw an error on invalid frontmatter', async () => { + const themePath2 = path.join(process.cwd(), 'test/_mocks/themes/invalid-frontmatter'); + themeConfig = ThemeConfig.getInstance(themePath2); + + const validator = new BundleValidator(themePath2, themeConfig, false); + + await expect(promisify(validator.validateTheme.bind(validator))()).rejects.toThrow( + 'while parsing frontmatter', + ); + }); }); diff --git a/lib/utils/frontmatter.js b/lib/utils/frontmatter.js new file mode 100644 index 00000000..38a8c531 --- /dev/null +++ b/lib/utils/frontmatter.js @@ -0,0 +1,33 @@ +const frontmatterRegex = /---\r?\n(?:.|\s)*?\r?\n---\r?\n/g; + +/** + * + * @param {String} file + * @returns {String|null} + */ +function getFrontmatterContent(file) { + const frontmatterMatch = file.match(frontmatterRegex); + return frontmatterMatch !== null ? frontmatterMatch[0] : null; +} + +/** + * + * @param {String} frontmatter + * @param {Object} settings + * @returns {String} + */ +function interpolateThemeSettings(frontmatter, settings) { + for (const [key, val] of Object.entries(settings)) { + const regex = `{{\\s*?theme_settings\\.${key}\\s*?}}`; + // eslint-disable-next-line no-param-reassign + frontmatter = frontmatter.replace(new RegExp(regex, 'g'), val); + } + + return frontmatter; +} + +module.exports = { + frontmatterRegex, + getFrontmatterContent, + interpolateThemeSettings, +}; diff --git a/package-lock.json b/package-lock.json index 02688928..ec8fad7d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -962,6 +962,22 @@ "ms": "2.1.2" } }, + "esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true + }, + "js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "dev": true, + "requires": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + } + }, "ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -1520,6 +1536,12 @@ "resolve-from": "^5.0.0" }, "dependencies": { + "esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true + }, "find-up": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", @@ -1530,6 +1552,16 @@ "path-exists": "^4.0.0" } }, + "js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "dev": true, + "requires": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + } + }, "locate-path": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", @@ -6110,6 +6142,22 @@ "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", "dev": true }, + "esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true + }, + "js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "dev": true, + "requires": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + } + }, "ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -7327,6 +7375,22 @@ "integrity": "sha512-I8ZuJ/qG92NWX8i5x1Y8qyj3vizhXS31OxjKDu3LKP+7/qBgfIKValiZIEwoVoJKUHlhWtYrktkxV1XsX+pPlg==", "requires": { "js-yaml": "^3.13.1" + }, + "dependencies": { + "esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==" + }, + "js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "requires": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + } + } } }, "fs-constants": { @@ -12174,18 +12238,17 @@ "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==" }, "js-yaml": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", - "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", "requires": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" + "argparse": "^2.0.1" }, "dependencies": { - "esprima": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", - "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==" + "argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" } } }, diff --git a/package.json b/package.json index 438be9c4..d85d5a1c 100644 --- a/package.json +++ b/package.json @@ -67,6 +67,7 @@ "graceful-fs": "^4.2.4", "image-size": "^0.9.1", "inquirer": "^8.1.5", + "js-yaml": "^4.1.0", "lodash": "^4.17.20", "memory-cache": "^0.2.0", "npm-which": "^3.0.1", diff --git a/server/plugins/renderer/renderer.module.js b/server/plugins/renderer/renderer.module.js index 5d1ca34a..efc73878 100644 --- a/server/plugins/renderer/renderer.module.js +++ b/server/plugins/renderer/renderer.module.js @@ -14,6 +14,11 @@ const { readFromStream } = require('../../../lib/utils/asyncUtils'); const NetworkUtils = require('../../../lib/utils/NetworkUtils'); const contentApiClient = require('../../../lib/content-api-client'); const { getPageType } = require('../../lib/page-type-util'); +const { + frontmatterRegex, + getFrontmatterContent, + interpolateThemeSettings, +} = require('../../../lib/utils/frontmatter'); const networkUtils = new NetworkUtils(); @@ -262,7 +267,6 @@ internals.parseResponse = async (bcAppData, request, response, responseArgs) => * @returns {Object} */ internals.getResourceConfig = (data, request, configuration) => { - const frontmatterRegex = /---\r?\n(?:.|\s)*?\r?\n---\r?\n/g; const missingThemeSettingsRegex = /{{\\s*?theme_settings\\..+?\\s*?}}/g; let resourcesConfig = {}; const templatePath = data.template_file; @@ -276,16 +280,12 @@ internals.getResourceConfig = (data, request, configuration) => { templatePath, ); - const frontmatterMatch = rawTemplate.match(frontmatterRegex); - if (frontmatterMatch !== null) { - let frontmatterContent = frontmatterMatch[0]; - // Interpolate theme settings for frontmatter - for (const [key, val] of Object.entries(configuration.settings)) { - const regex = `{{\\s*?theme_settings\\.${key}\\s*?}}`; - - frontmatterContent = frontmatterContent.replace(new RegExp(regex, 'g'), val); - } - + let frontmatterContent = getFrontmatterContent(rawTemplate); + if (frontmatterContent !== null) { + frontmatterContent = interpolateThemeSettings( + frontmatterContent, + configuration.settings, + ); // Remove any handlebars tags that weren't interpolated because there was no setting for it frontmatterContent = frontmatterContent.replace(missingThemeSettingsRegex, ''); // Replace the frontmatter with the newly interpolated version diff --git a/test/_mocks/themes/invalid-frontmatter/config.json b/test/_mocks/themes/invalid-frontmatter/config.json new file mode 100644 index 00000000..ea49d27a --- /dev/null +++ b/test/_mocks/themes/invalid-frontmatter/config.json @@ -0,0 +1,106 @@ +{ + "name": "Stencil", + "version": "1.0", + "css_compiler": "scss", + "autoprefixer_cascade": true, + "autoprefixer_browsers": [ + "> 5% in US" + ], + "meta": { + "price": 0, + "documentation_url": "https://stencil.bigcommerce.com/docs/what-is-stencil", + "author_name": "BigCommerce", + "author_email": "support@bigcommerce.com", + "author_support_url": "https://www.bigcommerce.com/support", + "composed_image": "composed.jpg", + "features": [ + "fully_responsive" + ] + }, + "settings": { + "color": "#ffffff", + "font": "Sans Something", + "select": "first", + "checkbox": true, + "radio": "first" + }, + "images": { + "logo": { + "width": 100, + "height": 100 + }, + "thumb": { + "width": 10, + "height": 10 + } + }, + "variations": [ + { + "id": "first", + "name": "First", + "meta": { + "desktop_screenshot": "desktop_light.jpg", + "mobile_screenshot": "mobile_light.jpg", + "description": "First variation", + "demo_url": "https://stencil-first.example.com", + "optimized_for": [ + "arts_crafts" + ], + "industries": [] + }, + "settings": { + "color": "#000000", + "select": "first" + }, + "images": {} + }, + { + "id": "second", + "name": "Second", + "meta": { + "desktop_screenshot": "desktop_bold.jpg", + "mobile_screenshot": "mobile_bold.jpg", + "description": "Second variation", + "demo_url": "https://stencil-second.example.com", + "optimized_for": [ + "arts_crafts" + ], + "industries": [] + }, + "settings": { + "color": "#000000", + "select": "second" + }, + "images": { + "logo": { + "width": 200, + "height": 200 + } + } + }, + { + "id": "third", + "name": "Third", + "meta": { + "desktop_screenshot": "desktop_warm.jpg", + "mobile_screenshot": "mobile_warm.jpg", + "description": "Third variation", + "demo_url": "https://stencil-third.example.com", + "optimized_for": [ + "arts_crafts" + ], + "industries": [] + }, + "settings": { + "color": "#FFFFFF", + "select": "third" + }, + "images": { + "logo": { + "width": 150, + "height": 150 + } + } + } + ] +} diff --git a/test/_mocks/themes/invalid-frontmatter/lang/en.json b/test/_mocks/themes/invalid-frontmatter/lang/en.json new file mode 100644 index 00000000..4fabf988 --- /dev/null +++ b/test/_mocks/themes/invalid-frontmatter/lang/en.json @@ -0,0 +1,20 @@ +{ + "header": { + "welcome_back": "Welcome back, {name}" + }, + "footer": { + "brands": "Popular Brands", + "navigate": "Navigate", + "info": "Info", + "categories": "Categories", + "call_us": "Call us at {phone_number}" + }, + "home": { + "heading": "Home" + }, + "blog": { + "recent_posts": "Recent Posts", + "label": "Blog", + "posted_by": "Posted by {name}" + } +} diff --git a/test/_mocks/themes/invalid-frontmatter/schema.json b/test/_mocks/themes/invalid-frontmatter/schema.json new file mode 100644 index 00000000..0f44ad26 --- /dev/null +++ b/test/_mocks/themes/invalid-frontmatter/schema.json @@ -0,0 +1,103 @@ +[ + { + "name": "i18n.Test", + "settings": [ + { + "type": "heading", + "content": "Heading" + }, + { + "type": "paragraph", + "content": "Paragraph" + }, + { + "type": "color", + "label": "Color", + "id": "color" + }, + { + "type": "font", + "label": "Font", + "id": "font", + "options": [ + { + "group": "Karla", + "label": "Karla", + "value": "Google_Karla_400" + }, + { + "group": "Roboto", + "label": "Roboto", + "value": "Google_Roboto_400" + }, + { + "group": "Source Sans Pro", + "label": "Source Sans Pro", + "value": "Google_Source+Sans+Pro_400" + } + ] + }, + { + "type": "select", + "label": "Select", + "id": "select", + "force_reload": true, + "options": [ + { + "value": "first", + "label": "First" + }, + { + "value": "second", + "label": "Second" + }, + { + "value": "third", + "label": "Third" + } + ] + }, + { + "type": "checkbox", + "label": "Checkbox", + "id": "checkbox" + }, + { + "type": "select", + "label": "Select", + "id": "select-two", + "options": [ + { + "value": "first", + "label": "First" + }, + { + "value": "second", + "label": "Second" + } + ] + } + ] + }, + { + "name": "Page", + "settings": [ + { + "type": "paragraph", + "label": "A Title", + "id": "customizable_title", + "content": "hello world" + }, + { + "type": "checkbox", + "label": "Fetch Data", + "id": "front_matter_value" + }, + { + "type": "input", + "label": "Display That", + "id": "display_that" + } + ] + } +] diff --git a/test/_mocks/themes/invalid-frontmatter/schemaTranslations.json b/test/_mocks/themes/invalid-frontmatter/schemaTranslations.json new file mode 100644 index 00000000..e300c7e0 --- /dev/null +++ b/test/_mocks/themes/invalid-frontmatter/schemaTranslations.json @@ -0,0 +1,5 @@ +{ + "i18n.Test": { + "default": "Test" + } +} diff --git a/test/_mocks/themes/invalid-frontmatter/templates/components/a.html b/test/_mocks/themes/invalid-frontmatter/templates/components/a.html new file mode 100644 index 00000000..78981922 --- /dev/null +++ b/test/_mocks/themes/invalid-frontmatter/templates/components/a.html @@ -0,0 +1 @@ +a diff --git a/test/_mocks/themes/invalid-frontmatter/templates/components/b.html b/test/_mocks/themes/invalid-frontmatter/templates/components/b.html new file mode 100644 index 00000000..61780798 --- /dev/null +++ b/test/_mocks/themes/invalid-frontmatter/templates/components/b.html @@ -0,0 +1 @@ +b diff --git a/test/_mocks/themes/invalid-frontmatter/templates/pages/page.html b/test/_mocks/themes/invalid-frontmatter/templates/pages/page.html new file mode 100644 index 00000000..626c9140 --- /dev/null +++ b/test/_mocks/themes/invalid-frontmatter/templates/pages/page.html @@ -0,0 +1,18 @@ +--- +front_matter_options: + setting_x: {{theme_settings.front_matter_value}}, +--- + + + + page.html + {{head.scripts}} + + + {{#if theme_settings.display_that}} +
That
+ {{> components/a}} + {{/if}} + {{footer.scripts}} + + diff --git a/test/_mocks/themes/invalid-frontmatter/templates/pages/page2.html b/test/_mocks/themes/invalid-frontmatter/templates/pages/page2.html new file mode 100644 index 00000000..cd952e0f --- /dev/null +++ b/test/_mocks/themes/invalid-frontmatter/templates/pages/page2.html @@ -0,0 +1,12 @@ + + + + page2.html + {{head.scripts}} + + +

{{theme_settings.customizable_title}}

+ {{> components/b}} + {{footer.scripts}} + +