From db0187aac25d89d3a4d03df35af32477a6335b0b Mon Sep 17 00:00:00 2001 From: bluwy Date: Wed, 25 Sep 2024 15:51:16 +0100 Subject: [PATCH 1/4] Parse frontmatter ourselves --- .changeset/sweet-timers-smash.md | 8 +++ packages/astro/package.json | 1 - packages/astro/src/content/utils.ts | 4 +- .../content-entry-type.ts | 6 +- .../astro/src/vite-plugin-markdown/index.ts | 2 +- packages/integrations/markdoc/package.json | 1 - .../markdoc/src/content-entry-type.ts | 17 +++--- packages/integrations/mdx/package.json | 1 - packages/integrations/mdx/src/index.ts | 10 +-- packages/integrations/mdx/src/utils.ts | 6 +- .../integrations/mdx/src/vite-plugin-mdx.ts | 7 +-- packages/markdown/remark/package.json | 2 + packages/markdown/remark/src/frontmatter.ts | 30 +++++++++ packages/markdown/remark/src/index.ts | 7 ++- pnpm-lock.yaml | 61 ++----------------- 15 files changed, 76 insertions(+), 87 deletions(-) create mode 100644 .changeset/sweet-timers-smash.md diff --git a/.changeset/sweet-timers-smash.md b/.changeset/sweet-timers-smash.md new file mode 100644 index 000000000000..5726071a99e8 --- /dev/null +++ b/.changeset/sweet-timers-smash.md @@ -0,0 +1,8 @@ +--- +'@astrojs/markdoc': patch +'@astrojs/mdx': patch +'@astrojs/markdown-remark': patch +'astro': patch +--- + +Parses frontmatter ourselves diff --git a/packages/astro/package.json b/packages/astro/package.json index 31bb6bcf5857..947c1714b3bd 100644 --- a/packages/astro/package.json +++ b/packages/astro/package.json @@ -152,7 +152,6 @@ "fastq": "^1.17.1", "flattie": "^1.1.1", "github-slugger": "^2.0.0", - "gray-matter": "^4.0.3", "html-escaper": "^3.0.3", "http-cache-semantics": "^4.1.1", "js-yaml": "^4.1.0", diff --git a/packages/astro/src/content/utils.ts b/packages/astro/src/content/utils.ts index 0fa7bed06ef0..a4bedd1a586b 100644 --- a/packages/astro/src/content/utils.ts +++ b/packages/astro/src/content/utils.ts @@ -1,8 +1,8 @@ import fsMod from 'node:fs'; import path from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; +import { parseFrontmatter } from '@astrojs/markdown-remark'; import { slug as githubSlug } from 'github-slugger'; -import matter from 'gray-matter'; import type { PluginContext } from 'rollup'; import type { ViteDevServer } from 'vite'; import xxhash from 'xxhash-wasm'; @@ -455,7 +455,7 @@ function getYAMLErrorLine(rawData: string | undefined, objectKey: string) { export function safeParseFrontmatter(source: string, id?: string) { try { - return matter(source); + return parseFrontmatter(source); } catch (err: any) { const markdownError = new MarkdownError({ name: 'MarkdownError', diff --git a/packages/astro/src/vite-plugin-markdown/content-entry-type.ts b/packages/astro/src/vite-plugin-markdown/content-entry-type.ts index 841789b22a63..c49f971e9754 100644 --- a/packages/astro/src/vite-plugin-markdown/content-entry-type.ts +++ b/packages/astro/src/vite-plugin-markdown/content-entry-type.ts @@ -8,10 +8,10 @@ export const markdownContentEntryType: ContentEntryType = { async getEntryInfo({ contents, fileUrl }: { contents: string; fileUrl: URL }) { const parsed = safeParseFrontmatter(contents, fileURLToPath(fileUrl)); return { - data: parsed.data, + data: parsed.frontmatter, body: parsed.content, - slug: parsed.data.slug, - rawData: parsed.matter, + slug: parsed.frontmatter.slug, + rawData: parsed.rawFrontmatter, }; }, // We need to handle propagation for Markdown because they support layouts which will bring in styles. diff --git a/packages/astro/src/vite-plugin-markdown/index.ts b/packages/astro/src/vite-plugin-markdown/index.ts index ddedef9738f9..b728ffb7048d 100644 --- a/packages/astro/src/vite-plugin-markdown/index.ts +++ b/packages/astro/src/vite-plugin-markdown/index.ts @@ -66,7 +66,7 @@ export default function markdown({ settings, logger }: AstroPluginOptions): Plug const renderResult = await (await processor).render(raw.content, { // @ts-expect-error passing internal prop fileURL, - frontmatter: raw.data, + frontmatter: raw.frontmatter, }); // Improve error message for invalid astro frontmatter diff --git a/packages/integrations/markdoc/package.json b/packages/integrations/markdoc/package.json index c92b5049dd74..35deff065458 100644 --- a/packages/integrations/markdoc/package.json +++ b/packages/integrations/markdoc/package.json @@ -68,7 +68,6 @@ "@markdoc/markdoc": "^0.4.0", "esbuild": "^0.21.5", "github-slugger": "^2.0.0", - "gray-matter": "^4.0.3", "htmlparser2": "^9.1.0" }, "peerDependencies": { diff --git a/packages/integrations/markdoc/src/content-entry-type.ts b/packages/integrations/markdoc/src/content-entry-type.ts index 303251d66b54..ea6ebfbfbf83 100644 --- a/packages/integrations/markdoc/src/content-entry-type.ts +++ b/packages/integrations/markdoc/src/content-entry-type.ts @@ -1,11 +1,11 @@ import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; +import { parseFrontmatter } from '@astrojs/markdown-remark'; import type { Config as MarkdocConfig, Node } from '@markdoc/markdoc'; import Markdoc from '@markdoc/markdoc'; import type { AstroConfig, ContentEntryType } from 'astro'; import { emitESMImage } from 'astro/assets/utils'; -import matter from 'gray-matter'; import type { Rollup, ErrorPayload as ViteErrorPayload } from 'vite'; import type { ComponentConfig } from './config.js'; import { htmlTokenTransform } from './html/transform/html-token-transform.js'; @@ -283,12 +283,12 @@ function getUsedTags(markdocAst: Node) { } function getEntryInfo({ fileUrl, contents }: { fileUrl: URL; contents: string }) { - const parsed = parseFrontmatter(contents, fileURLToPath(fileUrl)); + const parsed = safeParseFrontmatter(contents, fileURLToPath(fileUrl)); return { - data: parsed.data, + data: parsed.frontmatter, body: parsed.content, - slug: parsed.data.slug, - rawData: parsed.matter, + slug: parsed.frontmatter.slug, + rawData: parsed.rawFrontmatter, }; } @@ -410,12 +410,9 @@ function getStringifiedMap( * Match YAML exception handling from Astro core errors * @see 'astro/src/core/errors.ts' */ -function parseFrontmatter(fileContents: string, filePath: string) { +function safeParseFrontmatter(fileContents: string, filePath: string) { try { - // `matter` is empty string on cache results - // clear cache to prevent this - (matter as any).clearCache(); - return matter(fileContents); + return parseFrontmatter(fileContents); } catch (e: any) { if (e.name === 'YAMLException') { const err: Error & ViteErrorPayload['err'] = e; diff --git a/packages/integrations/mdx/package.json b/packages/integrations/mdx/package.json index bcb55ede0565..ec02ebf12621 100644 --- a/packages/integrations/mdx/package.json +++ b/packages/integrations/mdx/package.json @@ -39,7 +39,6 @@ "acorn": "^8.12.1", "es-module-lexer": "^1.5.4", "estree-util-visit": "^2.0.0", - "gray-matter": "^4.0.3", "hast-util-to-html": "^9.0.2", "kleur": "^4.1.5", "rehype-raw": "^7.0.0", diff --git a/packages/integrations/mdx/src/index.ts b/packages/integrations/mdx/src/index.ts index 3ebfc5f31f9f..eeb774111ffa 100644 --- a/packages/integrations/mdx/src/index.ts +++ b/packages/integrations/mdx/src/index.ts @@ -11,7 +11,7 @@ import type { import type { Options as RemarkRehypeOptions } from 'remark-rehype'; import type { PluggableList } from 'unified'; import type { OptimizeOptions } from './rehype-optimize-static.js'; -import { ignoreStringPlugins, parseFrontmatter } from './utils.js'; +import { ignoreStringPlugins, safeParseFrontmatter } from './utils.js'; import { vitePluginMdxPostprocess } from './vite-plugin-mdx-postprocess.js'; import { vitePluginMdx } from './vite-plugin-mdx.js'; @@ -60,12 +60,12 @@ export default function mdx(partialMdxOptions: Partial = {}): AstroI addContentEntryType({ extensions: ['.mdx'], async getEntryInfo({ fileUrl, contents }: { fileUrl: URL; contents: string }) { - const parsed = parseFrontmatter(contents, fileURLToPath(fileUrl)); + const parsed = safeParseFrontmatter(contents, fileURLToPath(fileUrl)); return { - data: parsed.data, + data: parsed.frontmatter, body: parsed.content, - slug: parsed.data.slug, - rawData: parsed.matter, + slug: parsed.frontmatter.slug, + rawData: parsed.rawFrontmatter, }; }, contentModuleTypes: await fs.readFile( diff --git a/packages/integrations/mdx/src/utils.ts b/packages/integrations/mdx/src/utils.ts index ad98abb9ef68..a1f2683856e7 100644 --- a/packages/integrations/mdx/src/utils.ts +++ b/packages/integrations/mdx/src/utils.ts @@ -1,7 +1,7 @@ +import { parseFrontmatter } from '@astrojs/markdown-remark'; import type { Options as AcornOpts } from 'acorn'; import { parse } from 'acorn'; import type { AstroConfig, AstroIntegrationLogger, SSRError } from 'astro'; -import matter from 'gray-matter'; import { bold } from 'kleur/colors'; import type { MdxjsEsm } from 'mdast-util-mdx'; import type { PluggableList } from 'unified'; @@ -48,9 +48,9 @@ export function getFileInfo(id: string, config: AstroConfig): FileInfo { * Match YAML exception handling from Astro core errors * @see 'astro/src/core/errors.ts' */ -export function parseFrontmatter(code: string, id: string) { +export function safeParseFrontmatter(code: string, id: string) { try { - return matter(code); + return parseFrontmatter(code); } catch (e: any) { if (e.name === 'YAMLException') { const err: SSRError = e; diff --git a/packages/integrations/mdx/src/vite-plugin-mdx.ts b/packages/integrations/mdx/src/vite-plugin-mdx.ts index 23828620266a..eea530c1c31b 100644 --- a/packages/integrations/mdx/src/vite-plugin-mdx.ts +++ b/packages/integrations/mdx/src/vite-plugin-mdx.ts @@ -4,7 +4,7 @@ import { VFile } from 'vfile'; import type { Plugin } from 'vite'; import type { MdxOptions } from './index.js'; import { createMdxProcessor } from './plugins.js'; -import { parseFrontmatter } from './utils.js'; +import { safeParseFrontmatter } from './utils.js'; export function vitePluginMdx(mdxOptions: MdxOptions): Plugin { let processor: ReturnType | undefined; @@ -38,11 +38,10 @@ export function vitePluginMdx(mdxOptions: MdxOptions): Plugin { async transform(code, id) { if (!id.endsWith('.mdx')) return; - const { data: frontmatter, content: pageContent, matter } = parseFrontmatter(code, id); - const frontmatterLines = matter ? matter.match(/\n/g)?.join('') + '\n\n' : ''; + const { frontmatter, content } = safeParseFrontmatter(code, id); const vfile = new VFile({ - value: frontmatterLines + pageContent, + value: content, path: id, data: { astro: { diff --git a/packages/markdown/remark/package.json b/packages/markdown/remark/package.json index f385e4dff0ad..e0b86e981f69 100644 --- a/packages/markdown/remark/package.json +++ b/packages/markdown/remark/package.json @@ -37,6 +37,7 @@ "hast-util-from-html": "^2.0.2", "hast-util-to-text": "^4.0.2", "import-meta-resolve": "^4.1.0", + "js-yaml": "^4.1.0", "mdast-util-definitions": "^6.0.0", "rehype-raw": "^7.0.0", "rehype-stringify": "^10.0.0", @@ -54,6 +55,7 @@ "devDependencies": { "@types/estree": "^1.0.5", "@types/hast": "^3.0.4", + "@types/js-yaml": "^4.0.9", "@types/mdast": "^4.0.4", "@types/unist": "^3.0.3", "astro-scripts": "workspace:*", diff --git a/packages/markdown/remark/src/frontmatter.ts b/packages/markdown/remark/src/frontmatter.ts index d8828a6fe2eb..751be187434a 100644 --- a/packages/markdown/remark/src/frontmatter.ts +++ b/packages/markdown/remark/src/frontmatter.ts @@ -1,3 +1,5 @@ +import yaml from 'js-yaml'; + export function isFrontmatterValid(frontmatter: Record) { try { // ensure frontmatter is JSON-serializable @@ -7,3 +9,31 @@ export function isFrontmatterValid(frontmatter: Record) { } return typeof frontmatter === 'object' && frontmatter !== null; } + +const frontmatterRE = /^---(.*?)^---/ms; +export function extractFrontmatter(code: string): string | undefined { + return frontmatterRE.exec(code)?.[1]; +} + +export interface ParseFrontmatterResult { + frontmatter: Record; + rawFrontmatter: string; + content: string; +} + +export function parseFrontmatter(code: string): ParseFrontmatterResult { + const rawFrontmatter = extractFrontmatter(code); + + if (rawFrontmatter == null) { + return { frontmatter: {}, rawFrontmatter: '', content: code }; + } + + const parsed = yaml.load(rawFrontmatter); + const frontmatter = (parsed && typeof parsed === 'object' ? parsed : {}) as Record; + + return { + frontmatter, + rawFrontmatter, + content: code.replace(rawFrontmatter, rawFrontmatter.replace(/[^\r\n]/g, ' ')), + }; +} diff --git a/packages/markdown/remark/src/index.ts b/packages/markdown/remark/src/index.ts index ad52dc16fe94..4d6eeb83fb07 100644 --- a/packages/markdown/remark/src/index.ts +++ b/packages/markdown/remark/src/index.ts @@ -20,7 +20,12 @@ export { rehypeHeadingIds } from './rehype-collect-headings.js'; export { remarkCollectImages } from './remark-collect-images.js'; export { rehypePrism } from './rehype-prism.js'; export { rehypeShiki } from './rehype-shiki.js'; -export { isFrontmatterValid } from './frontmatter.js'; +export { + isFrontmatterValid, + extractFrontmatter, + parseFrontmatter, + type ParseFrontmatterResult, +} from './frontmatter.js'; export { createShikiHighlighter, type ShikiHighlighter, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1eabbc7f7e09..17852471ca10 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -627,9 +627,6 @@ importers: github-slugger: specifier: ^2.0.0 version: 2.0.0 - gray-matter: - specifier: ^4.0.3 - version: 4.0.3 html-escaper: specifier: ^3.0.3 version: 3.0.3 @@ -4538,9 +4535,6 @@ importers: github-slugger: specifier: ^2.0.0 version: 2.0.0 - gray-matter: - specifier: ^4.0.3 - version: 4.0.3 htmlparser2: specifier: ^9.1.0 version: 9.1.0 @@ -4731,9 +4725,6 @@ importers: estree-util-visit: specifier: ^2.0.0 version: 2.0.0 - gray-matter: - specifier: ^4.0.3 - version: 4.0.3 hast-util-to-html: specifier: ^9.0.2 version: 9.0.2 @@ -5389,6 +5380,9 @@ importers: import-meta-resolve: specifier: ^4.1.0 version: 4.1.0 + js-yaml: + specifier: ^4.1.0 + version: 4.1.0 mdast-util-definitions: specifier: ^6.0.0 version: 6.0.0 @@ -5435,6 +5429,9 @@ importers: '@types/hast': specifier: ^3.0.4 version: 3.0.4 + '@types/js-yaml': + specifier: ^4.0.9 + version: 4.0.9 '@types/mdast': specifier: ^4.0.4 version: 4.0.4 @@ -8223,10 +8220,6 @@ packages: resolution: {integrity: sha512-uHaC9LYNv6BcW+8SvXcwUUDCrrUxt3GSa61DFvTHj8JC+M0hekMFBwMlCarLQDk5bbpZ2vStpnQPIwRuV98YMw==} engines: {node: '>=12.0.0'} - extend-shallow@2.0.1: - resolution: {integrity: sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==} - engines: {node: '>=0.10.0'} - extend@3.0.2: resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==} @@ -8415,10 +8408,6 @@ packages: graphemer@1.4.0: resolution: {integrity: sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==} - gray-matter@4.0.3: - resolution: {integrity: sha512-5v6yZd4JK3eMI3FqqCouswVqwugaA9r4dNZB1wwcmrD02QkV5H0y7XBQW8QwQqEaZY1pM9aqORSORhJRdNK44Q==} - engines: {node: '>=6.0'} - has-async-hooks@1.0.0: resolution: {integrity: sha512-YF0VPGjkxr7AyyQQNykX8zK4PvtEDsUJAPqwu06UFz1lb6EvI53sPh5H1kWxg8NXI5LsfRCZ8uX9NkYDZBb/mw==} @@ -8621,10 +8610,6 @@ packages: engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} hasBin: true - is-extendable@0.1.1: - resolution: {integrity: sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==} - engines: {node: '>=0.10.0'} - is-extglob@2.1.1: resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==} engines: {node: '>=0.10.0'} @@ -8781,10 +8766,6 @@ packages: keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} - kind-of@6.0.3: - resolution: {integrity: sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==} - engines: {node: '>=0.10.0'} - kleur@3.0.3: resolution: {integrity: sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==} engines: {node: '>=6'} @@ -10044,10 +10025,6 @@ packages: resolution: {integrity: sha512-3A6sD0WYP7+QrjbfNA2FN3FsOaGGFoekCVgTyypy53gPxhbkCIjtO6YWgdrfM+n/8sI8JeXZOIxsHjMTNxQ4nQ==} engines: {node: ^14.0.0 || >=16.0.0} - section-matter@1.0.0: - resolution: {integrity: sha512-vfD3pmTzGpufjScBh50YHKzEu2lxBWhVEHsNGoEXmCmn2hKGfeNLYMzCJpe8cD7gqX7TJluOVpBkAequ6dgMmA==} - engines: {node: '>=4'} - semver@6.3.1: resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==} hasBin: true @@ -10237,10 +10214,6 @@ packages: resolution: {integrity: sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==} engines: {node: '>=12'} - strip-bom-string@1.0.0: - resolution: {integrity: sha512-uCC2VHvQRYu+lMh4My/sFNmF2klFymLX1wHJeXnbEJERpV/ZsVuonzerjfrGpIGF7LBVa1O7i9kjiWvJiFck8g==} - engines: {node: '>=0.10.0'} - strip-bom@3.0.0: resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==} engines: {node: '>=4'} @@ -13907,10 +13880,6 @@ snapshots: expect-type@0.20.0: {} - extend-shallow@2.0.1: - dependencies: - is-extendable: 0.1.1 - extend@3.0.2: {} extendable-error@0.1.7: {} @@ -14106,13 +14075,6 @@ snapshots: graphemer@1.4.0: {} - gray-matter@4.0.3: - dependencies: - js-yaml: 3.14.1 - kind-of: 6.0.3 - section-matter: 1.0.0 - strip-bom-string: 1.0.0 - has-async-hooks@1.0.0: {} has-flag@3.0.0: {} @@ -14424,8 +14386,6 @@ snapshots: is-docker@3.0.0: {} - is-extendable@0.1.1: {} - is-extglob@2.1.1: {} is-fullwidth-code-point@3.0.0: {} @@ -14563,8 +14523,6 @@ snapshots: dependencies: json-buffer: 3.0.1 - kind-of@6.0.3: {} - kleur@3.0.3: {} kleur@4.1.5: {} @@ -16234,11 +16192,6 @@ snapshots: refa: 0.12.1 regexp-ast-analysis: 0.7.1 - section-matter@1.0.0: - dependencies: - extend-shallow: 2.0.1 - kind-of: 6.0.3 - semver@6.3.1: {} semver@7.6.3: {} @@ -16457,8 +16410,6 @@ snapshots: dependencies: ansi-regex: 6.0.1 - strip-bom-string@1.0.0: {} - strip-bom@3.0.0: {} strip-final-newline@3.0.0: {} From 3f8a6f631613c3a83b846d6dde8edcb7dc122274 Mon Sep 17 00:00:00 2001 From: bluwy Date: Wed, 25 Sep 2024 16:16:01 +0100 Subject: [PATCH 2/4] fix bug --- packages/markdown/remark/src/frontmatter.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/markdown/remark/src/frontmatter.ts b/packages/markdown/remark/src/frontmatter.ts index 751be187434a..026783043b4b 100644 --- a/packages/markdown/remark/src/frontmatter.ts +++ b/packages/markdown/remark/src/frontmatter.ts @@ -34,6 +34,9 @@ export function parseFrontmatter(code: string): ParseFrontmatterResult { return { frontmatter, rawFrontmatter, - content: code.replace(rawFrontmatter, rawFrontmatter.replace(/[^\r\n]/g, ' ')), + content: code.replace( + `---${rawFrontmatter}---`, + ` ${rawFrontmatter.replace(/[^\r\n]/g, ' ')} `, + ), }; } From c3566d505d57e942bf1f5e48b80b56108f332071 Mon Sep 17 00:00:00 2001 From: bluwy Date: Thu, 26 Sep 2024 10:25:58 +0100 Subject: [PATCH 3/4] fix everything --- packages/astro/src/content/utils.ts | 2 +- .../content-entry-type.ts | 2 +- .../markdoc/src/content-entry-type.ts | 36 +++++++--------- .../markdoc/test/content-collections.test.js | 6 +-- packages/integrations/mdx/src/index.ts | 2 +- packages/integrations/mdx/src/utils.ts | 2 +- packages/markdown/remark/src/frontmatter.ts | 42 ++++++++++++++++--- packages/markdown/remark/src/index.ts | 1 + 8 files changed, 60 insertions(+), 33 deletions(-) diff --git a/packages/astro/src/content/utils.ts b/packages/astro/src/content/utils.ts index a4bedd1a586b..7b5f74a6dd60 100644 --- a/packages/astro/src/content/utils.ts +++ b/packages/astro/src/content/utils.ts @@ -455,7 +455,7 @@ function getYAMLErrorLine(rawData: string | undefined, objectKey: string) { export function safeParseFrontmatter(source: string, id?: string) { try { - return parseFrontmatter(source); + return parseFrontmatter(source, { frontmatter: 'empty-with-spaces' }); } catch (err: any) { const markdownError = new MarkdownError({ name: 'MarkdownError', diff --git a/packages/astro/src/vite-plugin-markdown/content-entry-type.ts b/packages/astro/src/vite-plugin-markdown/content-entry-type.ts index c49f971e9754..3ca85a35655c 100644 --- a/packages/astro/src/vite-plugin-markdown/content-entry-type.ts +++ b/packages/astro/src/vite-plugin-markdown/content-entry-type.ts @@ -9,7 +9,7 @@ export const markdownContentEntryType: ContentEntryType = { const parsed = safeParseFrontmatter(contents, fileURLToPath(fileUrl)); return { data: parsed.frontmatter, - body: parsed.content, + body: parsed.content.trim(), slug: parsed.frontmatter.slug, rawData: parsed.rawFrontmatter, }; diff --git a/packages/integrations/markdoc/src/content-entry-type.ts b/packages/integrations/markdoc/src/content-entry-type.ts index ea6ebfbfbf83..67a8be531740 100644 --- a/packages/integrations/markdoc/src/content-entry-type.ts +++ b/packages/integrations/markdoc/src/content-entry-type.ts @@ -26,12 +26,20 @@ export async function getContentEntryType({ }): Promise { return { extensions: ['.mdoc'], - getEntryInfo, + getEntryInfo({ fileUrl, contents }) { + const parsed = safeParseFrontmatter(contents, fileURLToPath(fileUrl)); + return { + data: parsed.frontmatter, + body: parsed.content.trim(), + slug: parsed.frontmatter.slug, + rawData: parsed.rawFrontmatter, + }; + }, handlePropagation: true, async getRenderModule({ contents, fileUrl, viteId }) { - const entry = getEntryInfo({ contents, fileUrl }); + const parsed = safeParseFrontmatter(contents, fileURLToPath(fileUrl)); const tokenizer = getMarkdocTokenizer(options); - let tokens = tokenizer.tokenize(entry.body); + let tokens = tokenizer.tokenize(parsed.content); if (options?.allowHTML) { tokens = htmlTokenTransform(tokenizer, tokens); @@ -47,7 +55,6 @@ export async function getContentEntryType({ ast, /* Raised generics issue with Markdoc core https://github.com/markdoc/markdoc/discussions/400 */ markdocConfig: markdocConfig as MarkdocConfig, - entry, viteId, astroConfig, filePath, @@ -64,7 +71,6 @@ export async function getContentEntryType({ raiseValidationErrors({ ast: partialAst, markdocConfig: markdocConfig as MarkdocConfig, - entry, viteId, astroConfig, filePath: partialPath, @@ -224,14 +230,12 @@ async function resolvePartials({ function raiseValidationErrors({ ast, markdocConfig, - entry, viteId, astroConfig, filePath, }: { ast: Node; markdocConfig: MarkdocConfig; - entry: ReturnType; viteId: string; astroConfig: AstroConfig; filePath: string; @@ -250,8 +254,6 @@ function raiseValidationErrors({ }); if (validationErrors.length) { - // Heuristic: take number of newlines for `rawData` and add 2 for the `---` fences - const frontmatterBlockOffset = entry.rawData.split('\n').length + 2; const rootRelativePath = path.relative(fileURLToPath(astroConfig.root), filePath); throw new MarkdocError({ message: [ @@ -261,7 +263,7 @@ function raiseValidationErrors({ location: { // Error overlay does not support multi-line or ranges. // Just point to the first line. - line: frontmatterBlockOffset + validationErrors[0].lines[0], + line: validationErrors[0].lines[0], file: viteId, }, }); @@ -282,16 +284,6 @@ function getUsedTags(markdocAst: Node) { return tags; } -function getEntryInfo({ fileUrl, contents }: { fileUrl: URL; contents: string }) { - const parsed = safeParseFrontmatter(contents, fileURLToPath(fileUrl)); - return { - data: parsed.frontmatter, - body: parsed.content, - slug: parsed.frontmatter.slug, - rawData: parsed.rawFrontmatter, - }; -} - /** * Emits optimized images, and appends the generated `src` to each AST node * via the `__optimizedSrc` attribute. @@ -412,7 +404,9 @@ function getStringifiedMap( */ function safeParseFrontmatter(fileContents: string, filePath: string) { try { - return parseFrontmatter(fileContents); + // empty with lines to preserve sourcemap location, but not `empty-with-spaces` + // because markdoc struggles with spaces + return parseFrontmatter(fileContents, { frontmatter: 'empty-with-lines' }); } catch (e: any) { if (e.name === 'YAMLException') { const err: Error & ViteErrorPayload['err'] = e; diff --git a/packages/integrations/markdoc/test/content-collections.test.js b/packages/integrations/markdoc/test/content-collections.test.js index 5417f297d011..16032ce585ca 100644 --- a/packages/integrations/markdoc/test/content-collections.test.js +++ b/packages/integrations/markdoc/test/content-collections.test.js @@ -83,7 +83,7 @@ const post1Entry = { schemaWorks: true, title: 'Post 1', }, - body: '\n## Post 1\n\nThis is the contents of post 1.\n', + body: '## Post 1\n\nThis is the contents of post 1.', }; const post2Entry = { @@ -94,7 +94,7 @@ const post2Entry = { schemaWorks: true, title: 'Post 2', }, - body: '\n## Post 2\n\nThis is the contents of post 2.\n', + body: '## Post 2\n\nThis is the contents of post 2.', }; const post3Entry = { @@ -105,5 +105,5 @@ const post3Entry = { schemaWorks: true, title: 'Post 3', }, - body: '\n## Post 3\n\nThis is the contents of post 3.\n', + body: '## Post 3\n\nThis is the contents of post 3.', }; diff --git a/packages/integrations/mdx/src/index.ts b/packages/integrations/mdx/src/index.ts index eeb774111ffa..dcb13bc62f70 100644 --- a/packages/integrations/mdx/src/index.ts +++ b/packages/integrations/mdx/src/index.ts @@ -63,7 +63,7 @@ export default function mdx(partialMdxOptions: Partial = {}): AstroI const parsed = safeParseFrontmatter(contents, fileURLToPath(fileUrl)); return { data: parsed.frontmatter, - body: parsed.content, + body: parsed.content.trim(), slug: parsed.frontmatter.slug, rawData: parsed.rawFrontmatter, }; diff --git a/packages/integrations/mdx/src/utils.ts b/packages/integrations/mdx/src/utils.ts index a1f2683856e7..7dcd4a14cacf 100644 --- a/packages/integrations/mdx/src/utils.ts +++ b/packages/integrations/mdx/src/utils.ts @@ -50,7 +50,7 @@ export function getFileInfo(id: string, config: AstroConfig): FileInfo { */ export function safeParseFrontmatter(code: string, id: string) { try { - return parseFrontmatter(code); + return parseFrontmatter(code, { frontmatter: 'empty-with-spaces' }); } catch (e: any) { if (e.name === 'YAMLException') { const err: SSRError = e; diff --git a/packages/markdown/remark/src/frontmatter.ts b/packages/markdown/remark/src/frontmatter.ts index 026783043b4b..702509046381 100644 --- a/packages/markdown/remark/src/frontmatter.ts +++ b/packages/markdown/remark/src/frontmatter.ts @@ -15,13 +15,29 @@ export function extractFrontmatter(code: string): string | undefined { return frontmatterRE.exec(code)?.[1]; } +export interface ParseFrontmatterOptions { + /** + * How the frontmatter should be handled in the returned `content` string. + * - `preserve`: Keep the frontmatter in the returned `content` string. + * - `strip`: Completely remove the frontmatter from the returned `content` string. + * - `empty-with-spaces`: Replace the frontmatter with spaces in the returned `content` string. (preserves sourcemap offset/line/col) + * - `empty-with-lines`: Replace the frontmatter with newlines in the returned `content` string. (preserves sourcemap line/col) + * + * @default 'strip' + */ + frontmatter: 'preserve' | 'strip' | 'empty-with-spaces' | 'empty-with-lines'; +} + export interface ParseFrontmatterResult { frontmatter: Record; rawFrontmatter: string; content: string; } -export function parseFrontmatter(code: string): ParseFrontmatterResult { +export function parseFrontmatter( + code: string, + options?: ParseFrontmatterOptions, +): ParseFrontmatterResult { const rawFrontmatter = extractFrontmatter(code); if (rawFrontmatter == null) { @@ -31,12 +47,28 @@ export function parseFrontmatter(code: string): ParseFrontmatterResult { const parsed = yaml.load(rawFrontmatter); const frontmatter = (parsed && typeof parsed === 'object' ? parsed : {}) as Record; + let content: string; + switch (options?.frontmatter ?? 'strip') { + case 'preserve': + content = code; + break; + case 'strip': + content = code.replace(`---${rawFrontmatter}---`, ''); + break; + case 'empty-with-spaces': + content = code.replace( + `---${rawFrontmatter}---`, + ` ${rawFrontmatter.replace(/[^\r\n]/g, ' ')} `, + ); + break; + case 'empty-with-lines': + content = code.replace(`---${rawFrontmatter}---`, rawFrontmatter.replace(/[^\r\n]/g, '')); + break; + } + return { frontmatter, rawFrontmatter, - content: code.replace( - `---${rawFrontmatter}---`, - ` ${rawFrontmatter.replace(/[^\r\n]/g, ' ')} `, - ), + content, }; } diff --git a/packages/markdown/remark/src/index.ts b/packages/markdown/remark/src/index.ts index 4d6eeb83fb07..dd7d0444a6a7 100644 --- a/packages/markdown/remark/src/index.ts +++ b/packages/markdown/remark/src/index.ts @@ -24,6 +24,7 @@ export { isFrontmatterValid, extractFrontmatter, parseFrontmatter, + type ParseFrontmatterOptions, type ParseFrontmatterResult, } from './frontmatter.js'; export { From 813d03d83cea1f4ac0dbbb3220ad6770630e075b Mon Sep 17 00:00:00 2001 From: bluwy Date: Thu, 26 Sep 2024 13:52:21 +0100 Subject: [PATCH 4/4] fix wording --- packages/markdown/remark/src/frontmatter.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/markdown/remark/src/frontmatter.ts b/packages/markdown/remark/src/frontmatter.ts index 702509046381..60ae0b5da0aa 100644 --- a/packages/markdown/remark/src/frontmatter.ts +++ b/packages/markdown/remark/src/frontmatter.ts @@ -18,14 +18,14 @@ export function extractFrontmatter(code: string): string | undefined { export interface ParseFrontmatterOptions { /** * How the frontmatter should be handled in the returned `content` string. - * - `preserve`: Keep the frontmatter in the returned `content` string. - * - `strip`: Completely remove the frontmatter from the returned `content` string. - * - `empty-with-spaces`: Replace the frontmatter with spaces in the returned `content` string. (preserves sourcemap offset/line/col) - * - `empty-with-lines`: Replace the frontmatter with newlines in the returned `content` string. (preserves sourcemap line/col) + * - `preserve`: Keep the frontmatter. + * - `remove`: Remove the frontmatter. + * - `empty-with-spaces`: Replace the frontmatter with empty spaces. (preserves sourcemap line/col/offset) + * - `empty-with-lines`: Replace the frontmatter with empty line breaks. (preserves sourcemap line/col) * - * @default 'strip' + * @default 'remove' */ - frontmatter: 'preserve' | 'strip' | 'empty-with-spaces' | 'empty-with-lines'; + frontmatter: 'preserve' | 'remove' | 'empty-with-spaces' | 'empty-with-lines'; } export interface ParseFrontmatterResult { @@ -48,11 +48,11 @@ export function parseFrontmatter( const frontmatter = (parsed && typeof parsed === 'object' ? parsed : {}) as Record; let content: string; - switch (options?.frontmatter ?? 'strip') { + switch (options?.frontmatter ?? 'remove') { case 'preserve': content = code; break; - case 'strip': + case 'remove': content = code.replace(`---${rawFrontmatter}---`, ''); break; case 'empty-with-spaces':