From f9da56e63fbea33b464cc7b3cec963b439468e4a Mon Sep 17 00:00:00 2001 From: secustor Date: Wed, 11 Sep 2024 16:09:41 +0200 Subject: [PATCH] refactor: use yaml instead of js-yaml for parsing YAML files --- lib/modules/datasource/bitrise/index.ts | 5 +- lib/modules/datasource/bitrise/schema.ts | 2 +- lib/modules/datasource/conan/index.ts | 4 +- lib/modules/datasource/helm/common.spec.ts | 4 +- lib/modules/datasource/helm/index.ts | 4 +- lib/modules/manager/argocd/extract.ts | 16 +-- .../manager/crossplane/extract.spec.ts | 6 +- lib/modules/manager/crossplane/extract.ts | 20 +-- lib/modules/manager/docker-compose/extract.ts | 1 - lib/modules/manager/fleet/extract.ts | 37 +++-- lib/modules/manager/flux/extract.ts | 23 ++-- lib/modules/manager/github-actions/extract.ts | 2 +- .../manager/gitlabci-include/extract.ts | 4 +- .../gitlabci/__fixtures__/gitlab-ci.4.yaml | 2 +- lib/modules/manager/gitlabci/extract.ts | 10 +- lib/modules/manager/glasskube/extract.ts | 1 - .../manager/helm-requirements/extract.ts | 2 +- lib/modules/manager/helm-values/extract.ts | 2 +- lib/modules/manager/helmfile/extract.ts | 20 +-- lib/modules/manager/helmsman/extract.ts | 4 +- lib/modules/manager/helmv3/extract.ts | 2 +- lib/modules/manager/jenkins/extract.ts | 2 +- lib/modules/manager/kubernetes/extract.ts | 2 - lib/modules/manager/kustomize/extract.ts | 2 +- lib/modules/manager/npm/extract/pnpm.ts | 3 - lib/modules/manager/pre-commit/extract.ts | 2 +- lib/modules/manager/travis/extract.ts | 4 +- lib/modules/manager/velaci/extract.ts | 2 +- lib/modules/manager/woodpecker/extract.ts | 2 +- lib/util/schema-utils.ts | 4 +- lib/util/yaml.spec.ts | 2 +- lib/util/yaml.ts | 127 ++++++++++++++---- lib/workers/global/config/parse/file.ts | 4 +- package.json | 2 +- pnpm-lock.yaml | 14 +- 35 files changed, 180 insertions(+), 163 deletions(-) diff --git a/lib/modules/datasource/bitrise/index.ts b/lib/modules/datasource/bitrise/index.ts index dfdc2a1f2ab830..f3b2b66b838e14 100644 --- a/lib/modules/datasource/bitrise/index.ts +++ b/lib/modules/datasource/bitrise/index.ts @@ -115,12 +115,9 @@ export class BitriseDatasource extends Datasource { customSchema: BitriseStepFile, }); - const releaseTimestamp = is.string(published_at) - ? published_at - : published_at.toISOString(); result.releases.push({ version: versionDir.name, - releaseTimestamp, + releaseTimestamp: published_at, sourceUrl: source_code_url, }); } diff --git a/lib/modules/datasource/bitrise/schema.ts b/lib/modules/datasource/bitrise/schema.ts index 8b04b3cbe7ae96..b06b5d3c105cd0 100644 --- a/lib/modules/datasource/bitrise/schema.ts +++ b/lib/modules/datasource/bitrise/schema.ts @@ -1,6 +1,6 @@ import { z } from 'zod'; export const BitriseStepFile = z.object({ - published_at: z.date().or(z.string()), + published_at: z.string(), source_code_url: z.string().optional(), }); diff --git a/lib/modules/datasource/conan/index.ts b/lib/modules/datasource/conan/index.ts index a25d36280cc408..8ecb40124792bb 100644 --- a/lib/modules/datasource/conan/index.ts +++ b/lib/modules/datasource/conan/index.ts @@ -63,9 +63,7 @@ export class ConanDatasource extends Datasource { headers: { accept: 'application/vnd.github.v3.raw' }, }); // TODO: use schema (#9610) - const doc = parseSingleYaml(res.body, { - json: true, - }); + const doc = parseSingleYaml(res.body); return { releases: Object.keys(doc?.versions ?? {}).map((version) => ({ version, diff --git a/lib/modules/datasource/helm/common.spec.ts b/lib/modules/datasource/helm/common.spec.ts index 678ab118816022..84b88120db135a 100644 --- a/lib/modules/datasource/helm/common.spec.ts +++ b/lib/modules/datasource/helm/common.spec.ts @@ -4,9 +4,7 @@ import { findSourceUrl } from './common'; import type { HelmRepository } from './types'; // Truncated index.yaml file -const repo = parseSingleYaml(Fixtures.get('sample.yaml'), { - json: true, -}); +const repo = parseSingleYaml(Fixtures.get('sample.yaml')); describe('modules/datasource/helm/common', () => { describe('findSourceUrl', () => { diff --git a/lib/modules/datasource/helm/index.ts b/lib/modules/datasource/helm/index.ts index e5ccc32b5113d1..6ca3681c98103e 100644 --- a/lib/modules/datasource/helm/index.ts +++ b/lib/modules/datasource/helm/index.ts @@ -56,9 +56,7 @@ export class HelmDatasource extends Datasource { } try { // TODO: use schema (#9610) - const doc = parseSingleYaml(res.body, { - json: true, - }); + const doc = parseSingleYaml(res.body); if (!is.plainObject(doc)) { logger.warn( { helmRepository }, diff --git a/lib/modules/manager/argocd/extract.ts b/lib/modules/manager/argocd/extract.ts index de58802200dcbc..520415366891b0 100644 --- a/lib/modules/manager/argocd/extract.ts +++ b/lib/modules/manager/argocd/extract.ts @@ -36,17 +36,11 @@ export function extractPackageFile( return null; } - let definitions: ApplicationDefinition[]; - try { - definitions = parseYaml(content, { - customSchema: ApplicationDefinition, - failureBehaviour: 'filter', - removeTemplates: true, - }); - } catch (err) { - logger.debug({ err, packageFile }, 'Failed to parse ArgoCD definition.'); - return null; - } + const definitions = parseYaml(content, { + customSchema: ApplicationDefinition, + failureBehaviour: 'filter', + removeTemplates: true, + }); const deps = definitions.flatMap(processAppSpec); diff --git a/lib/modules/manager/crossplane/extract.spec.ts b/lib/modules/manager/crossplane/extract.spec.ts index dff7b0e7ddc276..dc16cb1962d430 100644 --- a/lib/modules/manager/crossplane/extract.spec.ts +++ b/lib/modules/manager/crossplane/extract.spec.ts @@ -13,10 +13,8 @@ describe('modules/manager/crossplane/extract', () => { expect(extractPackageFile('nothing here', 'packages.yml')).toBeNull(); }); - it('returns null for invalid', () => { - expect( - extractPackageFile(`${malformedPackages}\n123`, 'packages.yml'), - ).toBeNull(); + it('strips invalid templates', () => { + expect(extractPackageFile(`test: test: 123`, 'packages.yml')).toBeNull(); }); it('return null for kubernetes manifest', () => { diff --git a/lib/modules/manager/crossplane/extract.ts b/lib/modules/manager/crossplane/extract.ts index 154dfa2d0fb24d..9620dd5cedd272 100644 --- a/lib/modules/manager/crossplane/extract.ts +++ b/lib/modules/manager/crossplane/extract.ts @@ -6,7 +6,7 @@ import type { PackageDependency, PackageFileContent, } from '../types'; -import { type XPKG, XPKGSchema } from './schema'; +import { XPKGSchema } from './schema'; export function extractPackageFile( content: string, @@ -19,19 +19,11 @@ export function extractPackageFile( return null; } - let list: XPKG[] = []; - try { - list = parseYaml(content, { - customSchema: XPKGSchema, - failureBehaviour: 'filter', - }); - } catch (err) { - logger.debug( - { err, packageFile }, - 'Failed to parse Crossplane package file.', - ); - return null; - } + // not try and catching this as failureBehaviour is set to filter and therefore it will not throw + const list = parseYaml(content, { + customSchema: XPKGSchema, + failureBehaviour: 'filter', + }); const deps: PackageDependency[] = []; for (const xpkg of list) { diff --git a/lib/modules/manager/docker-compose/extract.ts b/lib/modules/manager/docker-compose/extract.ts index 255d47ae7d35a9..1f1932e0461e39 100644 --- a/lib/modules/manager/docker-compose/extract.ts +++ b/lib/modules/manager/docker-compose/extract.ts @@ -37,7 +37,6 @@ export function extractPackageFile( let config: DockerComposeFile; try { config = parseSingleYaml(content, { - json: true, customSchema: DockerComposeFile, }); } catch (err) { diff --git a/lib/modules/manager/fleet/extract.ts b/lib/modules/manager/fleet/extract.ts index 80f1cefa13526f..a84dfa3a04756b 100644 --- a/lib/modules/manager/fleet/extract.ts +++ b/lib/modules/manager/fleet/extract.ts @@ -1,5 +1,4 @@ import is from '@sindresorhus/is'; -import { logger } from '../../../logger'; import { regEx } from '../../../util/regex'; import { parseYaml } from '../../../util/yaml'; import { GitTagsDatasource } from '../../datasource/git-tags'; @@ -137,27 +136,21 @@ export function extractPackageFile( } const deps: PackageDependency[] = []; - try { - if (regEx('fleet.ya?ml').test(packageFile)) { - const docs = parseYaml(content, { - json: true, - customSchema: FleetFile, - failureBehaviour: 'filter', - }); - const fleetDeps = docs.flatMap(extractFleetFile); - - deps.push(...fleetDeps); - } else { - const docs = parseYaml(content, { - json: true, - customSchema: GitRepo, - failureBehaviour: 'filter', - }); - const gitRepoDeps = docs.flatMap(extractGitRepo); - deps.push(...gitRepoDeps); - } - } catch (err) { - logger.debug({ error: err, packageFile }, 'Failed to parse fleet YAML'); + if (regEx('fleet.ya?ml').test(packageFile)) { + const docs = parseYaml(content, { + customSchema: FleetFile, + failureBehaviour: 'filter', + }); + const fleetDeps = docs.flatMap(extractFleetFile); + + deps.push(...fleetDeps); + } else { + const docs = parseYaml(content, { + customSchema: GitRepo, + failureBehaviour: 'filter', + }); + const gitRepoDeps = docs.flatMap(extractGitRepo); + deps.push(...gitRepoDeps); } return deps.length ? { deps } : null; diff --git a/lib/modules/manager/flux/extract.ts b/lib/modules/manager/flux/extract.ts index 9bdf9670fcb152..c6c57a4ee8021f 100644 --- a/lib/modules/manager/flux/extract.ts +++ b/lib/modules/manager/flux/extract.ts @@ -47,21 +47,14 @@ function readManifest( }; } - try { - const manifest: FluxManifest = { - kind: 'resource', - file: packageFile, - resources: parseYaml(content, { - json: true, - customSchema: FluxResource, - failureBehaviour: 'filter', - }), - }; - return manifest; - } catch (err) { - logger.debug({ err, packageFile }, 'Failed to parse Flux manifest'); - return null; - } + return { + kind: 'resource', + file: packageFile, + resources: parseYaml(content, { + customSchema: FluxResource, + failureBehaviour: 'filter', + }), + }; } const githubUrlRegex = regEx( diff --git a/lib/modules/manager/github-actions/extract.ts b/lib/modules/manager/github-actions/extract.ts index c147df401c9180..0839912373120a 100644 --- a/lib/modules/manager/github-actions/extract.ts +++ b/lib/modules/manager/github-actions/extract.ts @@ -187,7 +187,7 @@ function extractWithYAMLParser( let pkg: Workflow; try { // TODO: use schema (#9610) - pkg = parseSingleYaml(content, { json: true }); + pkg = parseSingleYaml(content); } catch (err) { logger.debug( { packageFile, err }, diff --git a/lib/modules/manager/gitlabci-include/extract.ts b/lib/modules/manager/gitlabci-include/extract.ts index 2cc369e78c513f..9de7408d4a4ed5 100644 --- a/lib/modules/manager/gitlabci-include/extract.ts +++ b/lib/modules/manager/gitlabci-include/extract.ts @@ -72,9 +72,7 @@ export function extractPackageFile( const endpoint = GlobalConfig.get('endpoint'); try { // TODO: use schema (#9610) - const doc = parseSingleYaml(replaceReferenceTags(content), { - json: true, - }); + const doc = parseSingleYaml(replaceReferenceTags(content)); const includes = getAllIncludeProjects(doc); for (const includeObj of includes) { const dep = extractDepFromIncludeFile(includeObj); diff --git a/lib/modules/manager/gitlabci/__fixtures__/gitlab-ci.4.yaml b/lib/modules/manager/gitlabci/__fixtures__/gitlab-ci.4.yaml index 2fa96465c28b87..845c234955a3c3 100644 --- a/lib/modules/manager/gitlabci/__fixtures__/gitlab-ci.4.yaml +++ b/lib/modules/manager/gitlabci/__fixtures__/gitlab-ci.4.yaml @@ -1,7 +1,7 @@ include: - local: 'lib/modules/manager/gitlabci/__fixtures__/include.yml' -test: +test: test: script: - !abc [.setup, script] - echo running my own command diff --git a/lib/modules/manager/gitlabci/extract.ts b/lib/modules/manager/gitlabci/extract.ts index 09e4ba195e5d0c..488b27a9171e7a 100644 --- a/lib/modules/manager/gitlabci/extract.ts +++ b/lib/modules/manager/gitlabci/extract.ts @@ -25,7 +25,7 @@ import type { Job, Services, } from './types'; -import { getGitlabDep, replaceReferenceTags } from './utils'; +import { getGitlabDep } from './utils'; // See https://docs.gitlab.com/ee/ci/components/index.html#use-a-component const componentReferenceRegex = regEx( @@ -170,8 +170,8 @@ export function extractPackageFile( let deps: PackageDependency[] = []; try { // TODO: use schema (#9610) - const docs = parseYaml(replaceReferenceTags(content), { - json: true, + const docs = parseYaml(content, { + uniqueKeys: false, }); for (const doc of docs) { if (is.object(doc)) { @@ -259,8 +259,8 @@ export async function extractAllPackageFiles( let docs: GitlabPipeline[]; try { // TODO: use schema (#9610) - docs = parseYaml(replaceReferenceTags(content), { - json: true, + docs = parseYaml(content, { + uniqueKeys: false, }); } catch (err) { logger.debug( diff --git a/lib/modules/manager/glasskube/extract.ts b/lib/modules/manager/glasskube/extract.ts index 38f89ceaccfb6e..708bcc394c996a 100644 --- a/lib/modules/manager/glasskube/extract.ts +++ b/lib/modules/manager/glasskube/extract.ts @@ -20,7 +20,6 @@ function parseResources( packageFile: string, ): GlasskubeResources { const resources: GlasskubeResource[] = parseYaml(content, { - json: true, customSchema: GlasskubeResource, failureBehaviour: 'filter', }); diff --git a/lib/modules/manager/helm-requirements/extract.ts b/lib/modules/manager/helm-requirements/extract.ts index 194394137cea3c..7b62818b4f7dfe 100644 --- a/lib/modules/manager/helm-requirements/extract.ts +++ b/lib/modules/manager/helm-requirements/extract.ts @@ -17,7 +17,7 @@ export function extractPackageFile( // TODO: fix type let doc: any; try { - doc = parseSingleYaml(content, { json: true }); // TODO #9610 + doc = parseSingleYaml(content); // TODO #9610 } catch { logger.debug({ packageFile }, `Failed to parse helm requirements.yaml`); return null; diff --git a/lib/modules/manager/helm-values/extract.ts b/lib/modules/manager/helm-values/extract.ts index 76e51740ad4af1..59e465c0f0000e 100644 --- a/lib/modules/manager/helm-values/extract.ts +++ b/lib/modules/manager/helm-values/extract.ts @@ -77,7 +77,7 @@ export function extractPackageFile( // a parser that allows extracting line numbers would be preferable, with // the current approach we need to match anything we find again during the update // TODO: fix me (#9610) - parsedContent = parseYaml(content, { json: true }) as any; + parsedContent = parseYaml(content) as any; } catch (err) { logger.debug({ err, packageFile }, 'Failed to parse helm-values YAML'); return null; diff --git a/lib/modules/manager/helmfile/extract.ts b/lib/modules/manager/helmfile/extract.ts index 07f0ce0868dfb8..eaff3133deee3b 100644 --- a/lib/modules/manager/helmfile/extract.ts +++ b/lib/modules/manager/helmfile/extract.ts @@ -33,24 +33,14 @@ export async function extractPackageFile( config: ExtractConfig, ): Promise { const deps: PackageDependency[] = []; - let docs: Doc[]; let registryData: Record = {}; // Record kustomization usage for all deps, since updating artifacts is run on the helmfile.yaml as a whole. let needKustomize = false; - try { - docs = parseYaml(content, { - customSchema: documentSchema, - failureBehaviour: 'filter', - removeTemplates: true, - json: true, - }); - } catch (err) { - logger.debug( - { err, packageFile }, - 'Failed to parse helmfile helmfile.yaml', - ); - return null; - } + const docs: Doc[] = parseYaml(content, { + customSchema: documentSchema, + failureBehaviour: 'filter', + removeTemplates: true, + }); for (const doc of docs) { // Always check for repositories in the current document and override the existing ones if any (as YAML does) diff --git a/lib/modules/manager/helmsman/extract.ts b/lib/modules/manager/helmsman/extract.ts index 786876a2403219..5087e3a891fcde 100644 --- a/lib/modules/manager/helmsman/extract.ts +++ b/lib/modules/manager/helmsman/extract.ts @@ -69,9 +69,7 @@ export function extractPackageFile( ): PackageFileContent | null { try { // TODO: use schema (#9610) - const doc = parseSingleYaml(content, { - json: true, - }); + const doc = parseSingleYaml(content); if (!doc.apps) { logger.debug({ packageFile }, `Missing apps keys`); return null; diff --git a/lib/modules/manager/helmv3/extract.ts b/lib/modules/manager/helmv3/extract.ts index 7541f9ec2c9250..c876ebb7d1375a 100644 --- a/lib/modules/manager/helmv3/extract.ts +++ b/lib/modules/manager/helmv3/extract.ts @@ -23,7 +23,7 @@ export async function extractPackageFile( }; try { // TODO: use schema (#9610) - chart = parseSingleYaml(content, { json: true }); + chart = parseSingleYaml(content); if (!(chart?.apiVersion && chart.name && chart.version)) { logger.debug( { packageFile }, diff --git a/lib/modules/manager/jenkins/extract.ts b/lib/modules/manager/jenkins/extract.ts index 0b88ea5dfa45fc..0ee7d0e5469b81 100644 --- a/lib/modules/manager/jenkins/extract.ts +++ b/lib/modules/manager/jenkins/extract.ts @@ -58,7 +58,7 @@ function extractYaml( try { // TODO: use schema (#9610) - const doc = parseSingleYaml(content, { json: true }); + const doc = parseSingleYaml(content); if (is.nonEmptyArray(doc?.plugins)) { for (const plugin of doc.plugins) { if (plugin.artifactId) { diff --git a/lib/modules/manager/kubernetes/extract.ts b/lib/modules/manager/kubernetes/extract.ts index 55142784ad5c52..5d376539afdfb1 100644 --- a/lib/modules/manager/kubernetes/extract.ts +++ b/lib/modules/manager/kubernetes/extract.ts @@ -72,9 +72,7 @@ function extractApis( try { // TODO: use schema (#9610) doc = parseYaml(content, { - filename: packageFile, removeTemplates: true, - json: true, }); } catch (err) { logger.debug({ err, packageFile }, 'Failed to parse Kubernetes manifest.'); diff --git a/lib/modules/manager/kustomize/extract.ts b/lib/modules/manager/kustomize/extract.ts index b1a0a8341150a6..5e2b21ca226096 100644 --- a/lib/modules/manager/kustomize/extract.ts +++ b/lib/modules/manager/kustomize/extract.ts @@ -178,7 +178,7 @@ export function parseKustomize( let pkg: Kustomize | null = null; try { // TODO: use schema (#9610) - pkg = parseSingleYaml(content, { json: true }); + pkg = parseSingleYaml(content); } catch /* istanbul ignore next */ { logger.debug({ packageFile }, 'Error parsing kustomize file'); return null; diff --git a/lib/modules/manager/npm/extract/pnpm.ts b/lib/modules/manager/npm/extract/pnpm.ts index c4fa0d79510bf6..4871ed0fc5baa4 100644 --- a/lib/modules/manager/npm/extract/pnpm.ts +++ b/lib/modules/manager/npm/extract/pnpm.ts @@ -26,9 +26,6 @@ export async function extractPnpmFilters( // TODO: use schema (#9610,#22198) const contents = parseSingleYaml( (await readLocalFile(fileName, 'utf8'))!, - { - json: true, - }, ); if ( !Array.isArray(contents.packages) || diff --git a/lib/modules/manager/pre-commit/extract.ts b/lib/modules/manager/pre-commit/extract.ts index fc493014e7cc04..8dc89e4e70a269 100644 --- a/lib/modules/manager/pre-commit/extract.ts +++ b/lib/modules/manager/pre-commit/extract.ts @@ -159,7 +159,7 @@ export function extractPackageFile( let parsedContent: ParsedContent; try { // TODO: use schema (#9610) - parsedContent = parseSingleYaml(content, { json: true }); + parsedContent = parseSingleYaml(content); } catch (err) { logger.debug( { filename: packageFile, err }, diff --git a/lib/modules/manager/travis/extract.ts b/lib/modules/manager/travis/extract.ts index 4916a823e5ac8a..e5867bf3f9f855 100644 --- a/lib/modules/manager/travis/extract.ts +++ b/lib/modules/manager/travis/extract.ts @@ -12,9 +12,7 @@ export function extractPackageFile( let doc: TravisYaml; try { // TODO: use schema (#9610) - doc = parseSingleYaml(content, { - json: true, - }); + doc = parseSingleYaml(content); } catch (err) { logger.debug({ err, packageFile }, 'Failed to parse .travis.yml file.'); return null; diff --git a/lib/modules/manager/velaci/extract.ts b/lib/modules/manager/velaci/extract.ts index dfdbf90656b7cf..b401ad2fb8f0a8 100644 --- a/lib/modules/manager/velaci/extract.ts +++ b/lib/modules/manager/velaci/extract.ts @@ -13,7 +13,7 @@ export function extractPackageFile( try { // TODO: use schema (#9610) - doc = parseSingleYaml(file, { json: true }); + doc = parseSingleYaml(file); } catch (err) { logger.debug({ err, packageFile }, 'Failed to parse Vela file.'); return null; diff --git a/lib/modules/manager/woodpecker/extract.ts b/lib/modules/manager/woodpecker/extract.ts index 0877a3c81cd3c0..ba7f25c6123ec8 100644 --- a/lib/modules/manager/woodpecker/extract.ts +++ b/lib/modules/manager/woodpecker/extract.ts @@ -23,7 +23,7 @@ export function extractPackageFile( let config: WoodpeckerConfig; try { // TODO: use schema (#9610) - config = parseSingleYaml(content, { json: true }); + config = parseSingleYaml(content); if (!config) { logger.debug( { packageFile }, diff --git a/lib/util/schema-utils.ts b/lib/util/schema-utils.ts index d6cd33172a0c3b..4a1c35d98d6d3f 100644 --- a/lib/util/schema-utils.ts +++ b/lib/util/schema-utils.ts @@ -239,7 +239,7 @@ export const UtcDate = z export const Yaml = z.string().transform((str, ctx): JsonValue => { try { - return parseSingleYaml(str, { json: true }); + return parseSingleYaml(str); } catch { ctx.addIssue({ code: 'custom', message: 'Invalid YAML' }); return z.NEVER; @@ -248,7 +248,7 @@ export const Yaml = z.string().transform((str, ctx): JsonValue => { export const MultidocYaml = z.string().transform((str, ctx): JsonArray => { try { - return parseYaml(str, { json: true }) as JsonArray; + return parseYaml(str) as JsonArray; } catch { ctx.addIssue({ code: 'custom', message: 'Invalid YAML' }); return z.NEVER; diff --git a/lib/util/yaml.spec.ts b/lib/util/yaml.spec.ts index 448b0670b5abaf..de9491f458054a 100644 --- a/lib/util/yaml.spec.ts +++ b/lib/util/yaml.spec.ts @@ -193,7 +193,7 @@ describe('util/yaml', () => { describe('load', () => { it('should return undefined', () => { - expect(parseSingleYaml(``)).toBeUndefined(); + expect(parseSingleYaml(``)).toBeNull(); }); it('should parse content with single document', () => { diff --git a/lib/util/yaml.ts b/lib/util/yaml.ts index 67098d72b2c72c..122e25fa4aed90 100644 --- a/lib/util/yaml.ts +++ b/lib/util/yaml.ts @@ -1,9 +1,11 @@ -import type { DumpOptions, LoadOptions } from 'js-yaml'; -import { - loadAll as multiple, - load as single, - dump as upstreamDump, -} from 'js-yaml'; +import type { + CreateNodeOptions, + DocumentOptions, + ParseOptions, + SchemaOptions, + ToStringOptions, +} from 'yaml'; +import { parseAllDocuments, parseDocument, stringify } from 'yaml'; import type { ZodType } from 'zod'; import { logger } from '../logger'; import { regEx } from './regex'; @@ -11,7 +13,9 @@ import { regEx } from './regex'; interface YamlOptions< ResT = unknown, Schema extends ZodType = ZodType, -> extends LoadOptions { +> extends ParseOptions, + DocumentOptions, + SchemaOptions { customSchema?: Schema; removeTemplates?: boolean; } @@ -23,55 +27,117 @@ interface YamlOptionsMultiple< failureBehaviour?: 'throw' | 'filter'; } +export type DumpOptions = DocumentOptions & + SchemaOptions & + ParseOptions & + CreateNodeOptions & + ToStringOptions; + +/** + * Parse a YAML string into a JavaScript object. + * + * Multiple documents are supported. + * + * If a schema is provided, the parsed object will be validated against it. + * + * If failureBehaviour is set to 'filter', + * the function will return an empty array if the YAML parsing or schema validation fails and therefore will not throw an error. + * + * If failureBehaviour is set to 'throw', + * the function will throw an error if the YAML parsing or schema validation fails for ANY document. + * @param content + * @param options + */ export function parseYaml( content: string, options?: YamlOptionsMultiple, ): ResT[] { const massagedContent = massageContent(content, options); - const rawDocuments = multiple(massagedContent, null, options); + const rawDocuments = parseAllDocuments( + massagedContent, + prepareParseOption(options), + ); const schema = options?.customSchema; - if (!schema) { - return rawDocuments as ResT[]; - } - const parsed: ResT[] = []; - for (const element of rawDocuments) { - const result = schema.safeParse(element); + const results: ResT[] = []; + for (const rawDocument of rawDocuments) { + const document = rawDocument.toJS(); + + const errors = rawDocument.errors; + // handle YAML parse errors + if (errors?.length) { + const error = new AggregateError(errors, 'Failed to parse YAML file'); + if (options?.failureBehaviour === 'filter') { + logger.debug({ error, document }, 'Failed to parse YAML'); + continue; + } + throw error; + } + + // skip schema validation if no schema is provided + if (!schema) { + results.push(document as ResT); + continue; + } + + const result = schema.safeParse(document); if (result.success) { - parsed.push(result.data); + results.push(result.data); continue; } - if (options?.failureBehaviour !== 'filter') { - throw new Error('Failed to parse YAML file', { cause: result.error }); + // handle schema validation errors + if (options?.failureBehaviour === 'filter') { + logger.trace( + { error: result.error, document }, + 'Failed to parse schema for YAML', + ); + continue; } - logger.trace( - { error: result.error, document: element }, - 'Failed to parse schema for YAML', - ); + throw new Error('Failed to parse YAML file', { cause: result.error }); } - return parsed; + + return results; } +/** + * Parse a YAML string into a JavaScript object. + * + * Only a single document is supported. + * + * If a schema is provided, the parsed object will be validated against it. + * Should the YAML parsing or schemata validation fail, an error will be thrown. + * + * @param content + * @param options + */ export function parseSingleYaml( content: string, options?: YamlOptions, ): ResT { const massagedContent = massageContent(content, options); - const rawDocument = single(massagedContent, options); + const rawDocument = parseDocument( + massagedContent, + prepareParseOption(options), + ); + if (rawDocument?.errors?.length) { + throw new AggregateError(rawDocument.errors, 'Failed to parse YAML file'); + } + + const document = rawDocument.toJS(); const schema = options?.customSchema; if (!schema) { - return rawDocument as ResT; + return document as ResT; } - return schema.parse(rawDocument); + return schema.parse(document); } export function dump(obj: any, opts?: DumpOptions | undefined): string { - return upstreamDump(obj, opts); + return stringify(obj, opts); } function massageContent(content: string, options?: YamlOptions): string { @@ -86,3 +152,12 @@ function massageContent(content: string, options?: YamlOptions): string { return content; } + +function prepareParseOption(options: YamlOptions | undefined): YamlOptions { + return { + prettyErrors: true, + // if we're removing templates, we can run into the situation where we have duplicate keys + uniqueKeys: !options?.removeTemplates, + ...options, + }; +} diff --git a/lib/workers/global/config/parse/file.ts b/lib/workers/global/config/parse/file.ts index 9d7c815c59ef04..bdc39481f16708 100644 --- a/lib/workers/global/config/parse/file.ts +++ b/lib/workers/global/config/parse/file.ts @@ -16,9 +16,7 @@ export async function getParsedContent(file: string): Promise { switch (upath.extname(file)) { case '.yaml': case '.yml': - return parseSingleYaml(await readSystemFile(file, 'utf8'), { - json: true, - }); + return parseSingleYaml(await readSystemFile(file, 'utf8')); case '.json5': case '.json': return parseJson( diff --git a/package.json b/package.json index 3a1fdc97565520..87992f728d53c6 100644 --- a/package.json +++ b/package.json @@ -205,7 +205,6 @@ "handlebars": "4.7.8", "ignore": "5.3.2", "ini": "4.1.3", - "js-yaml": "4.1.0", "json-dup-key-validator": "1.0.3", "json-stringify-pretty-compact": "3.0.0", "json5": "2.2.3", @@ -245,6 +244,7 @@ "validate-npm-package-name": "5.0.1", "vuln-vects": "1.1.0", "xmldoc": "1.3.0", + "yaml": "2.5.1", "zod": "3.23.8" }, "optionalDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4316a663ed6120..243a738f77de76 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -209,9 +209,6 @@ importers: ini: specifier: 4.1.3 version: 4.1.3 - js-yaml: - specifier: 4.1.0 - version: 4.1.0 json-dup-key-validator: specifier: 1.0.3 version: 1.0.3 @@ -329,6 +326,9 @@ importers: xmldoc: specifier: 1.3.0 version: 1.3.0 + yaml: + specifier: 2.5.1 + version: 2.5.1 zod: specifier: 3.23.8 version: 3.23.8 @@ -1186,7 +1186,6 @@ packages: '@ls-lint/ls-lint@2.2.3': resolution: {integrity: sha512-ekM12jNm/7O2I/hsRv9HvYkRdfrHpiV1epVuI2NP+eTIcEgdIdKkKCs9KgQydu/8R5YXTov9aHdOgplmCHLupw==} - cpu: [x64, arm64, s390x] os: [darwin, linux, win32] hasBin: true @@ -6150,6 +6149,11 @@ packages: resolution: {integrity: sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==} engines: {node: '>=18'} + yaml@2.5.1: + resolution: {integrity: sha512-bLQOjaX/ADgQ20isPJRvF0iRUHIxVhYvr53Of7wGcWlO2jvtUlH5m87DsmulFVxRpNLOnI4tB6p/oh8D7kpn9Q==} + engines: {node: '>= 14'} + hasBin: true + yargs-parser@18.1.3: resolution: {integrity: sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==} engines: {node: '>=6'} @@ -13240,6 +13244,8 @@ snapshots: yallist@5.0.0: {} + yaml@2.5.1: {} + yargs-parser@18.1.3: dependencies: camelcase: 5.3.1