From b0ce30b59a2c31a58b2f71972b89ad13832cde9f Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Sun, 13 Feb 2022 16:55:03 +0300 Subject: [PATCH] refactor(datasource/nuget): Convert to class (#14140) * refactor(datasource/nuget): Convert to class * Fix strict nulls and obsolete URL * Fixes * Fix mutability Co-authored-by: Rhys Arkins Co-authored-by: Michael Kriese --- lib/datasource/api.ts | 4 +- lib/datasource/nuget/common.ts | 32 ++++++++- lib/datasource/nuget/index.spec.ts | 11 +-- lib/datasource/nuget/index.ts | 70 ++++++++------------ lib/datasource/nuget/types.ts | 5 ++ lib/datasource/nuget/v2.ts | 7 +- lib/datasource/nuget/v3.ts | 18 ++--- lib/manager/cake/index.ts | 10 ++- lib/manager/nuget/artifacts.ts | 5 +- lib/manager/nuget/extract.ts | 6 +- lib/manager/nuget/extract/global-manifest.ts | 4 +- lib/manager/nuget/index.ts | 4 +- lib/manager/nuget/util.ts | 13 ++-- lib/util/host-rules.spec.ts | 48 +++++++------- lib/workers/repository/init/vulnerability.ts | 4 +- 15 files changed, 131 insertions(+), 110 deletions(-) diff --git a/lib/datasource/api.ts b/lib/datasource/api.ts index 0d23685404fcab..66f4f321820c44 100644 --- a/lib/datasource/api.ts +++ b/lib/datasource/api.ts @@ -26,7 +26,7 @@ import { JenkinsPluginsDatasource } from './jenkins-plugins'; import * as maven from './maven'; import { NodeDatasource } from './node'; import * as npm from './npm'; -import * as nuget from './nuget'; +import { NugetDatasource } from './nuget'; import { OrbDatasource } from './orb'; import { PackagistDatasource } from './packagist'; import { PodDatasource } from './pod'; @@ -71,7 +71,7 @@ api.set(JenkinsPluginsDatasource.id, new JenkinsPluginsDatasource()); api.set('maven', maven); api.set(NodeDatasource.id, new NodeDatasource()); api.set('npm', npm); -api.set('nuget', nuget); +api.set(NugetDatasource.id, new NugetDatasource()); api.set(OrbDatasource.id, new OrbDatasource()); api.set(PackagistDatasource.id, new PackagistDatasource()); api.set(PodDatasource.id, new PodDatasource()); diff --git a/lib/datasource/nuget/common.ts b/lib/datasource/nuget/common.ts index b961f477a84f2c..10d4073b13c230 100644 --- a/lib/datasource/nuget/common.ts +++ b/lib/datasource/nuget/common.ts @@ -1,9 +1,37 @@ +import { logger } from '../../logger'; import { regEx } from '../../util/regex'; - -export const id = 'nuget'; +import { parseUrl } from '../../util/url'; +import type { ParsedRegistryUrl } from './types'; const buildMetaRe = regEx(/\+.+$/g); export function removeBuildMeta(version: string): string { return version?.replace(buildMetaRe, ''); } + +const protocolVersionRegExp = regEx(/#protocolVersion=(?2|3)/); + +export function parseRegistryUrl(registryUrl: string): ParsedRegistryUrl { + const parsedUrl = parseUrl(registryUrl); + if (!parsedUrl) { + logger.debug( + { urL: registryUrl }, + `nuget registry failure: can't parse ${registryUrl}` + ); + return { feedUrl: registryUrl, protocolVersion: null }; + } + let protocolVersion = 2; + const protocolVersionMatch = protocolVersionRegExp.exec( + parsedUrl.hash + )?.groups; + if (protocolVersionMatch) { + const { protocol } = protocolVersionMatch; + parsedUrl.hash = ''; + protocolVersion = Number.parseInt(protocol, 10); + } else if (parsedUrl.pathname.endsWith('.json')) { + protocolVersion = 3; + } + + const feedUrl = parsedUrl.href; + return { feedUrl, protocolVersion }; +} diff --git a/lib/datasource/nuget/index.spec.ts b/lib/datasource/nuget/index.spec.ts index 926de848f3ed56..4bb87a56045564 100644 --- a/lib/datasource/nuget/index.spec.ts +++ b/lib/datasource/nuget/index.spec.ts @@ -3,7 +3,10 @@ import * as httpMock from '../../../test/http-mock'; import { loadFixture } from '../../../test/util'; import * as _hostRules from '../../util/host-rules'; import { id as versioning } from '../../versioning/nuget'; -import { id as datasource, parseRegistryUrl } from '.'; +import { parseRegistryUrl } from './common'; +import { NugetDatasource } from '.'; + +const datasource = NugetDatasource.id; const hostRules: any = _hostRules; @@ -118,11 +121,9 @@ describe('datasource/nuget/index', () => { }); it('returns null for unparseable', () => { - const parsed = parseRegistryUrl( - 'https://test:malfor%5Med@test.example.com' - ); + const parsed = parseRegistryUrl('https://test.example.com:abc'); - expect(parsed.feedUrl).toBe('https://test:malfor%5Med@test.example.com'); + expect(parsed.feedUrl).toBe('https://test.example.com:abc'); expect(parsed.protocolVersion).toBeNull(); }); }); diff --git a/lib/datasource/nuget/index.ts b/lib/datasource/nuget/index.ts index a9d97e6ce2cf54..6da18fc6054d0f 100644 --- a/lib/datasource/nuget/index.ts +++ b/lib/datasource/nuget/index.ts @@ -1,54 +1,42 @@ -import urlApi from 'url'; import { logger } from '../../logger'; -import { regEx } from '../../util/regex'; import * as nugetVersioning from '../../versioning/nuget'; +import { Datasource } from '../datasource'; import type { GetReleasesConfig, ReleaseResult } from '../types'; +import { parseRegistryUrl } from './common'; import * as v2 from './v2'; import * as v3 from './v3'; -export { id } from './common'; +// https://api.nuget.org/v3/index.json is a default official nuget feed +export const defaultRegistryUrls = ['https://api.nuget.org/v3/index.json']; -export const customRegistrySupport = true; -export const defaultRegistryUrls = [v3.getDefaultFeed()]; -export const defaultVersioning = nugetVersioning.id; -export const registryStrategy = 'merge'; +export class NugetDatasource extends Datasource { + static readonly id = 'nuget'; -export function parseRegistryUrl(registryUrl: string): { - feedUrl: string; - protocolVersion: number; -} { - try { - const parsedUrl = urlApi.parse(registryUrl); - let protocolVersion = 2; - const protocolVersionRegExp = regEx(/#protocolVersion=(2|3)/); - const protocolVersionMatch = protocolVersionRegExp.exec(parsedUrl.hash); - if (protocolVersionMatch) { - parsedUrl.hash = ''; - protocolVersion = Number.parseInt(protocolVersionMatch[1], 10); - } else if (parsedUrl.pathname.endsWith('.json')) { - protocolVersion = 3; - } - return { feedUrl: urlApi.format(parsedUrl), protocolVersion }; - } catch (err) { - logger.debug({ err }, `nuget registry failure: can't parse ${registryUrl}`); - return { feedUrl: registryUrl, protocolVersion: null }; - } -} + override readonly defaultRegistryUrls = defaultRegistryUrls; + + override readonly defaultVersioning = nugetVersioning.id; -export async function getReleases({ - lookupName, - registryUrl, -}: GetReleasesConfig): Promise { - logger.trace(`nuget.getReleases(${lookupName})`); - const { feedUrl, protocolVersion } = parseRegistryUrl(registryUrl); - if (protocolVersion === 2) { - return v2.getReleases(feedUrl, lookupName); + override readonly registryStrategy = 'merge'; + + constructor() { + super(NugetDatasource.id); } - if (protocolVersion === 3) { - const queryUrl = await v3.getResourceUrl(feedUrl); - if (queryUrl) { - return v3.getReleases(feedUrl, queryUrl, lookupName); + + async getReleases({ + lookupName, + registryUrl, + }: GetReleasesConfig): Promise { + logger.trace(`nuget.getReleases(${lookupName})`); + const { feedUrl, protocolVersion } = parseRegistryUrl(registryUrl); + if (protocolVersion === 2) { + return v2.getReleases(this.http, feedUrl, lookupName); + } + if (protocolVersion === 3) { + const queryUrl = await v3.getResourceUrl(this.http, feedUrl); + if (queryUrl) { + return v3.getReleases(this.http, feedUrl, queryUrl, lookupName); + } } + return null; } - return null; } diff --git a/lib/datasource/nuget/types.ts b/lib/datasource/nuget/types.ts index d8a4d9815d6307..29aba2a5c6b4a5 100644 --- a/lib/datasource/nuget/types.ts +++ b/lib/datasource/nuget/types.ts @@ -22,3 +22,8 @@ export interface CatalogPage { export interface PackageRegistration { items: CatalogPage[]; } + +export interface ParsedRegistryUrl { + feedUrl: string; + protocolVersion: number | null; +} diff --git a/lib/datasource/nuget/v2.ts b/lib/datasource/nuget/v2.ts index 4e8cfb4d544f8d..81875f8596ecc2 100644 --- a/lib/datasource/nuget/v2.ts +++ b/lib/datasource/nuget/v2.ts @@ -1,17 +1,16 @@ import { XmlDocument, XmlElement } from 'xmldoc'; import { logger } from '../../logger'; -import { Http } from '../../util/http'; +import type { Http } from '../../util/http'; import { regEx } from '../../util/regex'; import type { ReleaseResult } from '../types'; -import { id, removeBuildMeta } from './common'; - -const http = new Http(id); +import { removeBuildMeta } from './common'; function getPkgProp(pkgInfo: XmlElement, propName: string): string { return pkgInfo.childNamed('m:properties').childNamed(`d:${propName}`)?.val; } export async function getReleases( + http: Http, feedUrl: string, pkgName: string ): Promise { diff --git a/lib/datasource/nuget/v3.ts b/lib/datasource/nuget/v3.ts index c3a98e6e40834b..3dec8d0fae0b72 100644 --- a/lib/datasource/nuget/v3.ts +++ b/lib/datasource/nuget/v3.ts @@ -5,12 +5,12 @@ import { XmlDocument } from 'xmldoc'; import { logger } from '../../logger'; import { ExternalHostError } from '../../types/errors/external-host-error'; import * as packageCache from '../../util/cache/package'; -import { Http } from '../../util/http'; +import type { Http } from '../../util/http'; import { HttpError } from '../../util/http/types'; import { regEx } from '../../util/regex'; import { ensureTrailingSlash } from '../../util/url'; import type { Release, ReleaseResult } from '../types'; -import { id, removeBuildMeta } from './common'; +import { removeBuildMeta } from './common'; import type { CatalogEntry, CatalogPage, @@ -18,17 +18,10 @@ import type { ServicesIndexRaw, } from './types'; -const http = new Http(id); - -// https://api.nuget.org/v3/index.json is a default official nuget feed -const defaultNugetFeed = 'https://api.nuget.org/v3/index.json'; const cacheNamespace = 'datasource-nuget'; -export function getDefaultFeed(): string { - return defaultNugetFeed; -} - export async function getResourceUrl( + http: Http, url: string, resourceType = 'RegistrationsBaseUrl' ): Promise { @@ -101,6 +94,7 @@ export async function getResourceUrl( } async function getCatalogEntry( + http: Http, catalogPage: CatalogPage ): Promise { let items = catalogPage.items; @@ -113,6 +107,7 @@ async function getCatalogEntry( } export async function getReleases( + http: Http, registryUrl: string, feedUrl: string, pkgName: string @@ -122,7 +117,7 @@ export async function getReleases( const packageRegistration = await http.getJson(url); const catalogPages = packageRegistration.body.items || []; const catalogPagesQueue = catalogPages.map( - (page) => (): Promise => getCatalogEntry(page) + (page) => (): Promise => getCatalogEntry(http, page) ); const catalogEntries = ( await pAll(catalogPagesQueue, { concurrency: 5 }) @@ -164,6 +159,7 @@ export async function getReleases( try { const packageBaseAddress = await getResourceUrl( + http, registryUrl, 'PackageBaseAddress' ); diff --git a/lib/manager/cake/index.ts b/lib/manager/cake/index.ts index 43130e954f5dd8..f92270d3fc64c2 100644 --- a/lib/manager/cake/index.ts +++ b/lib/manager/cake/index.ts @@ -1,6 +1,6 @@ import moo from 'moo'; import { ProgrammingLanguage } from '../../constants'; -import { id as datasource } from '../../datasource/nuget'; +import { NugetDatasource } from '../../datasource/nuget'; import { regEx } from '../../util/regex'; import type { PackageDependency, PackageFile } from '../types'; @@ -36,7 +36,11 @@ function parseDependencyLine(line: string): PackageDependency | null { const depName = searchParams.get('package'); const currentValue = searchParams.get('version'); - const result: PackageDependency = { datasource, depName, currentValue }; + const result: PackageDependency = { + datasource: NugetDatasource.id, + depName, + currentValue, + }; if (!isEmptyHost) { if (protocol.startsWith('http')) { @@ -69,4 +73,4 @@ export function extractPackageFile(content: string): PackageFile { return { deps }; } -export const supportedDatasources = [datasource]; +export const supportedDatasources = [NugetDatasource.id]; diff --git a/lib/manager/nuget/artifacts.ts b/lib/manager/nuget/artifacts.ts index f245301ea54469..ede2e3b0081cd3 100644 --- a/lib/manager/nuget/artifacts.ts +++ b/lib/manager/nuget/artifacts.ts @@ -2,7 +2,8 @@ import { join } from 'path'; import { quote } from 'shlex'; import { GlobalConfig } from '../../config/global'; import { TEMPORARY_ERROR } from '../../constants/error-messages'; -import { id, parseRegistryUrl } from '../../datasource/nuget'; +import { NugetDatasource } from '../../datasource/nuget'; +import { parseRegistryUrl } from '../../datasource/nuget/common'; import { logger } from '../../logger'; import { exec } from '../../util/exec'; import type { ExecOptions } from '../../util/exec/types'; @@ -39,7 +40,7 @@ async function addSourceCmds( const result = []; for (const registry of registries) { const { username, password } = hostRules.find({ - hostType: id, + hostType: NugetDatasource.id, url: registry.url, }); const registryInfo = parseRegistryUrl(registry.url); diff --git a/lib/manager/nuget/extract.ts b/lib/manager/nuget/extract.ts index 5c14279e91993f..df4cd40ee5faf8 100644 --- a/lib/manager/nuget/extract.ts +++ b/lib/manager/nuget/extract.ts @@ -1,6 +1,6 @@ import { XmlDocument, XmlElement, XmlNode } from 'xmldoc'; import { GlobalConfig } from '../../config/global'; -import * as datasourceNuget from '../../datasource/nuget'; +import { NugetDatasource } from '../../datasource/nuget'; import { logger } from '../../logger'; import { getSiblingFileName, localPathExists } from '../../util/fs'; import { hasKey } from '../../util/object'; @@ -54,7 +54,7 @@ function extractDepsFromXml(xmlNode: XmlDocument): PackageDependency[] { ?.groups?.currentValue?.trim(); if (depName && currentValue) { results.push({ - datasource: datasourceNuget.id, + datasource: NugetDatasource.id, depType: 'nuget', depName, currentValue, @@ -103,7 +103,7 @@ export async function extractPackageFile( depType: 'nuget', depName, currentValue, - datasource: datasourceNuget.id, + datasource: NugetDatasource.id, }; if (registryUrls) { dep.registryUrls = registryUrls; diff --git a/lib/manager/nuget/extract/global-manifest.ts b/lib/manager/nuget/extract/global-manifest.ts index acbd6cbf350029..9048b1d90552b5 100644 --- a/lib/manager/nuget/extract/global-manifest.ts +++ b/lib/manager/nuget/extract/global-manifest.ts @@ -1,4 +1,4 @@ -import * as datasourceNuget from '../../../datasource/nuget'; +import { NugetDatasource } from '../../../datasource/nuget'; import { logger } from '../../../logger'; import type { PackageDependency, PackageFile } from '../../types'; import type { MsbuildGlobalManifest } from '../types'; @@ -41,7 +41,7 @@ export function extractMsbuildGlobalManifest( depType: 'msbuild-sdk', depName, currentValue, - datasource: datasourceNuget.id, + datasource: NugetDatasource.id, }; deps.push(dep); diff --git a/lib/manager/nuget/index.ts b/lib/manager/nuget/index.ts index 0fbdfe63c41d7d..adbac59113ee17 100644 --- a/lib/manager/nuget/index.ts +++ b/lib/manager/nuget/index.ts @@ -1,5 +1,5 @@ import { ProgrammingLanguage } from '../../constants'; -import * as datasourceNuget from '../../datasource/nuget'; +import { NugetDatasource } from '../../datasource/nuget'; export { extractPackageFile } from './extract'; export { updateArtifacts } from './artifacts'; @@ -15,4 +15,4 @@ export const defaultConfig = { ], }; -export const supportedDatasources = [datasourceNuget.id]; +export const supportedDatasources = [NugetDatasource.id]; diff --git a/lib/manager/nuget/util.ts b/lib/manager/nuget/util.ts index 1f7365b8ecbd3f..d33f255dbad8d6 100644 --- a/lib/manager/nuget/util.ts +++ b/lib/manager/nuget/util.ts @@ -2,7 +2,7 @@ import cryptoRandomString from 'crypto-random-string'; import findUp from 'find-up'; import upath from 'upath'; import { XmlDocument } from 'xmldoc'; -import * as datasourceNuget from '../../datasource/nuget'; +import { defaultRegistryUrls } from '../../datasource/nuget'; import { logger } from '../../logger'; import { readFile } from '../../util/fs'; import { regEx } from '../../util/regex'; @@ -22,13 +22,12 @@ export function getRandomString(): string { return cryptoRandomString({ length: 16 }); } +const defaultRegistries = defaultRegistryUrls.map( + (registryUrl) => ({ url: registryUrl } as Registry) +); + export function getDefaultRegistries(): Registry[] { - return datasourceNuget.defaultRegistryUrls.map( - (registryUrl) => - ({ - url: registryUrl, - } as Registry) - ); + return [...defaultRegistries]; } export async function getConfiguredRegistries( diff --git a/lib/util/host-rules.spec.ts b/lib/util/host-rules.spec.ts index 4ad62e4a3efd0d..449b7337705495 100644 --- a/lib/util/host-rules.spec.ts +++ b/lib/util/host-rules.spec.ts @@ -1,5 +1,5 @@ import { PlatformId } from '../constants'; -import * as datasourceNuget from '../datasource/nuget'; +import { NugetDatasource } from '../datasource/nuget'; import { add, clear, find, findAll, getAll, hosts } from './host-rules'; describe('util/host-rules', () => { @@ -55,21 +55,21 @@ describe('util/host-rules', () => { }); it('needs exact host matches', () => { add({ - hostType: datasourceNuget.id, + hostType: NugetDatasource.id, hostName: 'nuget.org', username: 'root', password: 'p4$$w0rd', token: undefined, } as any); - expect(find({ hostType: datasourceNuget.id })).toEqual({}); + expect(find({ hostType: NugetDatasource.id })).toEqual({}); expect( - find({ hostType: datasourceNuget.id, url: 'https://nuget.org' }) + find({ hostType: NugetDatasource.id, url: 'https://nuget.org' }) ).not.toEqual({}); expect( - find({ hostType: datasourceNuget.id, url: 'https://not.nuget.org' }) + find({ hostType: NugetDatasource.id, url: 'https://not.nuget.org' }) ).not.toEqual({}); expect( - find({ hostType: datasourceNuget.id, url: 'https://not-nuget.org' }) + find({ hostType: NugetDatasource.id, url: 'https://not-nuget.org' }) ).toEqual({}); }); it('matches on empty rules', () => { @@ -77,16 +77,16 @@ describe('util/host-rules', () => { enabled: true, }); expect( - find({ hostType: datasourceNuget.id, url: 'https://api.github.com' }) + find({ hostType: NugetDatasource.id, url: 'https://api.github.com' }) ).toEqual({ enabled: true }); }); it('matches on hostType', () => { add({ - hostType: datasourceNuget.id, + hostType: NugetDatasource.id, token: 'abc', }); expect( - find({ hostType: datasourceNuget.id, url: 'https://nuget.local/api' }) + find({ hostType: NugetDatasource.id, url: 'https://nuget.local/api' }) ).toEqual({ token: 'abc' }); }); it('matches on domainName', () => { @@ -95,14 +95,14 @@ describe('util/host-rules', () => { token: 'def', } as any); expect( - find({ hostType: datasourceNuget.id, url: 'https://api.github.com' }) + find({ hostType: NugetDatasource.id, url: 'https://api.github.com' }) .token ).toBe('def'); expect( - find({ hostType: datasourceNuget.id, url: 'https://github.com' }).token + find({ hostType: NugetDatasource.id, url: 'https://github.com' }).token ).toBe('def'); expect( - find({ hostType: datasourceNuget.id, url: 'https://apigithub.com' }) + find({ hostType: NugetDatasource.id, url: 'https://apigithub.com' }) .token ).toBeUndefined(); }); @@ -178,7 +178,7 @@ describe('util/host-rules', () => { token: 'abc', } as any); expect( - find({ hostType: datasourceNuget.id, url: 'https://nuget.local/api' }) + find({ hostType: NugetDatasource.id, url: 'https://nuget.local/api' }) ).toEqual({ token: 'abc' }); }); it('matches on matchHost with protocol', () => { @@ -190,7 +190,7 @@ describe('util/host-rules', () => { expect(find({ url: 'https://domain.com' }).token).toBe('def'); expect( find({ - hostType: datasourceNuget.id, + hostType: NugetDatasource.id, url: 'https://domain.com/renovatebot', }).token ).toBe('def'); @@ -215,55 +215,55 @@ describe('util/host-rules', () => { }); it('matches on hostType and endpoint', () => { add({ - hostType: datasourceNuget.id, + hostType: NugetDatasource.id, matchHost: 'https://nuget.local/api', token: 'abc', } as any); expect( - find({ hostType: datasourceNuget.id, url: 'https://nuget.local/api' }) + find({ hostType: NugetDatasource.id, url: 'https://nuget.local/api' }) .token ).toBe('abc'); }); it('matches on endpoint subresource', () => { add({ - hostType: datasourceNuget.id, + hostType: NugetDatasource.id, matchHost: 'https://nuget.local/api', token: 'abc', } as any); expect( find({ - hostType: datasourceNuget.id, + hostType: NugetDatasource.id, url: 'https://nuget.local/api/sub-resource', }) ).toEqual({ token: 'abc' }); }); it('returns hosts', () => { add({ - hostType: datasourceNuget.id, + hostType: NugetDatasource.id, token: 'aaaaaa', }); add({ - hostType: datasourceNuget.id, + hostType: NugetDatasource.id, matchHost: 'https://nuget.local/api', token: 'abc', } as any); add({ - hostType: datasourceNuget.id, + hostType: NugetDatasource.id, hostName: 'my.local.registry', token: 'def', } as any); add({ - hostType: datasourceNuget.id, + hostType: NugetDatasource.id, matchHost: 'another.local.registry', token: 'xyz', }); add({ - hostType: datasourceNuget.id, + hostType: NugetDatasource.id, matchHost: 'https://yet.another.local.registry', token: '123', }); const res = hosts({ - hostType: datasourceNuget.id, + hostType: NugetDatasource.id, }); expect(res).toEqual([ 'nuget.local', diff --git a/lib/workers/repository/init/vulnerability.ts b/lib/workers/repository/init/vulnerability.ts index 439f1e944328ae..53549c420f35c5 100644 --- a/lib/workers/repository/init/vulnerability.ts +++ b/lib/workers/repository/init/vulnerability.ts @@ -2,7 +2,7 @@ import type { PackageRule, RenovateConfig } from '../../../config/types'; import { NO_VULNERABILITY_ALERTS } from '../../../constants/error-messages'; import * as datasourceMaven from '../../../datasource/maven'; import { id as npmId } from '../../../datasource/npm'; -import * as datasourceNuget from '../../../datasource/nuget'; +import { NugetDatasource } from '../../../datasource/nuget'; import { PypiDatasource } from '../../../datasource/pypi'; import { RubyGemsDatasource } from '../../../datasource/rubygems'; import { logger } from '../../../logger'; @@ -89,7 +89,7 @@ export async function detectVulnerabilityAlerts( const datasourceMapping: Record = { MAVEN: datasourceMaven.id, NPM: npmId, - NUGET: datasourceNuget.id, + NUGET: NugetDatasource.id, PIP: PypiDatasource.id, RUBYGEMS: RubyGemsDatasource.id, };