diff --git a/.eslintrc.js b/.eslintrc.js index c2c970e27d2e4bc..b21122cd9791c02 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -42,6 +42,7 @@ module.exports = { 'prefer-template': 0, 'no-underscore-dangle': 0, 'no-negated-condition': 'error', + 'sort-imports': [ 'error', { diff --git a/lib/datasource/api.ts b/lib/datasource/api.ts index c3b5100cc981e75..069d8ccd52c01fd 100644 --- a/lib/datasource/api.ts +++ b/lib/datasource/api.ts @@ -1,6 +1,6 @@ import * as bitbucketTags from './bitbucket-tags'; -import * as cdnjs from './cdnjs'; -import * as clojure from './clojure'; +import { CdnJsDatasource } from './cdnjs'; +import { ClojureDatasource } from './clojure'; import * as crate from './crate'; import * as dart from './dart'; import * as docker from './docker'; @@ -36,8 +36,8 @@ const api = new Map(); export default api; api.set('bitbucket-tags', bitbucketTags); -api.set('cdnjs', cdnjs); -api.set('clojure', clojure); +api.set('cdnjs', new CdnJsDatasource()); +api.set('clojure', new ClojureDatasource()); api.set('crate', crate); api.set('dart', dart); api.set('docker', docker); diff --git a/lib/datasource/cdnjs/__snapshots__/index.spec.ts.snap b/lib/datasource/cdnjs/__snapshots__/index.spec.ts.snap index f6b82cfb899b2ba..eafc2cec56c0f67 100644 --- a/lib/datasource/cdnjs/__snapshots__/index.spec.ts.snap +++ b/lib/datasource/cdnjs/__snapshots__/index.spec.ts.snap @@ -187,7 +187,7 @@ Array [ ] `; -exports[`datasource/cdnjs/index getReleases returns null for unknown error 1`] = ` +exports[`datasource/cdnjs/index getReleases throws for unknown error 1`] = ` Array [ Object { "headers": Object { diff --git a/lib/datasource/cdnjs/index.spec.ts b/lib/datasource/cdnjs/index.spec.ts index 7819d61c4f99476..d81e627fbc4d5c8 100644 --- a/lib/datasource/cdnjs/index.spec.ts +++ b/lib/datasource/cdnjs/index.spec.ts @@ -2,7 +2,7 @@ import { getPkgReleases } from '..'; import * as httpMock from '../../../test/http-mock'; import { getName, loadFixture } from '../../../test/util'; import { EXTERNAL_HOST_ERROR } from '../../constants/error-messages'; -import { id as datasource } from '.'; +import { CdnJsDatasource } from '.'; const res1 = loadFixture('d3-force.json'); const res2 = loadFixture('bulma.json'); @@ -26,21 +26,30 @@ describe(getName(), () => { it('throws for empty result', async () => { httpMock.scope(baseUrl).get(pathFor('foo/bar')).reply(200, null); await expect( - getPkgReleases({ datasource, depName: 'foo/bar' }) + getPkgReleases({ + datasource: CdnJsDatasource.id, + depName: 'foo/bar', + }) ).rejects.toThrow(EXTERNAL_HOST_ERROR); expect(httpMock.getTrace()).toMatchSnapshot(); }); it('throws for error', async () => { httpMock.scope(baseUrl).get(pathFor('foo/bar')).replyWithError('error'); await expect( - getPkgReleases({ datasource, depName: 'foo/bar' }) + getPkgReleases({ + datasource: CdnJsDatasource.id, + depName: 'foo/bar', + }) ).rejects.toThrow(EXTERNAL_HOST_ERROR); expect(httpMock.getTrace()).toMatchSnapshot(); }); it('returns null for 404', async () => { httpMock.scope(baseUrl).get(pathFor('foo/bar')).reply(404); expect( - await getPkgReleases({ datasource, depName: 'foo/bar' }) + await getPkgReleases({ + datasource: CdnJsDatasource.id, + depName: 'foo/bar', + }) ).toBeNull(); expect(httpMock.getTrace()).toMatchSnapshot(); }); @@ -51,7 +60,7 @@ describe(getName(), () => { .reply(200, {}); expect( await getPkgReleases({ - datasource, + datasource: CdnJsDatasource.id, depName: 'doesnotexist/doesnotexist', }) ).toBeNull(); @@ -60,28 +69,40 @@ describe(getName(), () => { it('throws for 401', async () => { httpMock.scope(baseUrl).get(pathFor('foo/bar')).reply(401); await expect( - getPkgReleases({ datasource, depName: 'foo/bar' }) + getPkgReleases({ + datasource: CdnJsDatasource.id, + depName: 'foo/bar', + }) ).rejects.toThrow(EXTERNAL_HOST_ERROR); expect(httpMock.getTrace()).toMatchSnapshot(); }); it('throws for 429', async () => { httpMock.scope(baseUrl).get(pathFor('foo/bar')).reply(429); await expect( - getPkgReleases({ datasource, depName: 'foo/bar' }) + getPkgReleases({ + datasource: CdnJsDatasource.id, + depName: 'foo/bar', + }) ).rejects.toThrow(EXTERNAL_HOST_ERROR); expect(httpMock.getTrace()).toMatchSnapshot(); }); it('throws for 5xx', async () => { httpMock.scope(baseUrl).get(pathFor('foo/bar')).reply(502); await expect( - getPkgReleases({ datasource, depName: 'foo/bar' }) + getPkgReleases({ + datasource: CdnJsDatasource.id, + depName: 'foo/bar', + }) ).rejects.toThrow(EXTERNAL_HOST_ERROR); expect(httpMock.getTrace()).toMatchSnapshot(); }); - it('returns null for unknown error', async () => { + it('throws for unknown error', async () => { httpMock.scope(baseUrl).get(pathFor('foo/bar')).replyWithError('error'); await expect( - getPkgReleases({ datasource, depName: 'foo/bar' }) + getPkgReleases({ + datasource: CdnJsDatasource.id, + depName: 'foo/bar', + }) ).rejects.toThrow(EXTERNAL_HOST_ERROR); expect(httpMock.getTrace()).toMatchSnapshot(); }); @@ -91,7 +112,7 @@ describe(getName(), () => { .get(pathFor('d3-force/d3-force.js')) .reply(200, res1); const res = await getPkgReleases({ - datasource, + datasource: CdnJsDatasource.id, depName: 'd3-force/d3-force.js', }); expect(res).toMatchSnapshot(); @@ -103,7 +124,7 @@ describe(getName(), () => { .get(pathFor('bulma/only/0.7.5/style.css')) .reply(200, res2); const res = await getPkgReleases({ - datasource, + datasource: CdnJsDatasource.id, depName: 'bulma/only/0.7.5/style.css', }); expect(res).toMatchSnapshot(); diff --git a/lib/datasource/cdnjs/index.ts b/lib/datasource/cdnjs/index.ts index e678e56ecd33a6b..0944d32d39d94c9 100644 --- a/lib/datasource/cdnjs/index.ts +++ b/lib/datasource/cdnjs/index.ts @@ -1,61 +1,56 @@ import { ExternalHostError } from '../../types/errors/external-host-error'; -import { Http } from '../../util/http'; +import { Datasource } from '../datasource'; import type { GetReleasesConfig, ReleaseResult } from '../types'; +import type { CdnjsResponse } from './types'; -export const id = 'cdnjs'; -export const customRegistrySupport = false; -export const defaultRegistryUrls = ['https://api.cdnjs.com/']; -export const caching = true; +export class CdnJsDatasource extends Datasource { + static readonly id = 'cdnjs'; -const http = new Http(id); + constructor() { + super(CdnJsDatasource.id); + } -interface CdnjsAsset { - version: string; - files: string[]; - sri?: Record; -} + customRegistrySupport = false; -interface CdnjsResponse { - homepage?: string; - repository?: { - type: 'git' | unknown; - url?: string; - }; - assets?: CdnjsAsset[]; -} + defaultRegistryUrls = ['https://api.cdnjs.com/']; -export async function getReleases({ - lookupName, - registryUrl, -}: GetReleasesConfig): Promise { - // Each library contains multiple assets, so we cache at the library level instead of per-asset - const library = lookupName.split('/')[0]; - const url = `${registryUrl}libraries/${library}?fields=homepage,repository,assets`; - try { - const { assets, homepage, repository } = ( - await http.getJson(url) - ).body; - if (!assets) { - return null; - } - const assetName = lookupName.replace(`${library}/`, ''); - const releases = assets - .filter(({ files }) => files.includes(assetName)) - .map(({ version, sri }) => ({ version, newDigest: sri[assetName] })); + caching = true; - const result: ReleaseResult = { releases }; + // this.handleErrors will always throw + // eslint-disable-next-line consistent-return + async getReleases({ + lookupName, + registryUrl, + }: GetReleasesConfig): Promise { + // Each library contains multiple assets, so we cache at the library level instead of per-asset + const library = lookupName.split('/')[0]; + const url = `${registryUrl}libraries/${library}?fields=homepage,repository,assets`; + try { + const { assets, homepage, repository } = ( + await this.http.getJson(url) + ).body; + if (!assets) { + return null; + } + const assetName = lookupName.replace(`${library}/`, ''); + const releases = assets + .filter(({ files }) => files.includes(assetName)) + .map(({ version, sri }) => ({ version, newDigest: sri[assetName] })); - if (homepage) { - result.homepage = homepage; - } - if (repository?.url) { - result.sourceUrl = repository.url; - } - return result; - } catch (err) { - if (err.statusCode !== 404) { - throw new ExternalHostError(err); + const result: ReleaseResult = { releases }; + + if (homepage) { + result.homepage = homepage; + } + if (repository?.url) { + result.sourceUrl = repository.url; + } + return result; + } catch (err) { + if (err.statusCode !== 404) { + throw new ExternalHostError(err); + } + this.handleGenericErrors(err); } - throw err; } } diff --git a/lib/datasource/cdnjs/types.ts b/lib/datasource/cdnjs/types.ts new file mode 100644 index 000000000000000..97ff84563dd3717 --- /dev/null +++ b/lib/datasource/cdnjs/types.ts @@ -0,0 +1,14 @@ +interface CdnjsAsset { + version: string; + files: string[]; + sri?: Record; +} + +export interface CdnjsResponse { + homepage?: string; + repository?: { + type: 'git' | unknown; + url?: string; + }; + assets?: CdnjsAsset[]; +} diff --git a/lib/datasource/clojure/index.spec.ts b/lib/datasource/clojure/index.spec.ts new file mode 100644 index 000000000000000..644e6e176ec690a --- /dev/null +++ b/lib/datasource/clojure/index.spec.ts @@ -0,0 +1,52 @@ +import { Release, getPkgReleases } from '..'; +import { getName } from '../../../test/util'; +import { id as mavenVersioning } from '../../versioning/maven'; +import { ClojureDatasource } from '.'; + +const config = { + versioning: mavenVersioning, + datasource: ClojureDatasource.id, +}; + +describe(getName(), () => { + describe('getReleases', () => { + function generateReleases(versions: string[]): Release[] { + return versions.map((v) => ({ version: v })); + } + + it('should return empty if library is not found', async () => { + const releases = await getPkgReleases({ + ...config, + depName: 'unknown:unknown', + registryUrls: [ + 's3://somewhere.s3.aws.amazon.com', + 'file://lib/datasource/maven/__fixtures__/repo1.maven.org/maven2/', + ], + }); + expect(releases).toBeNull(); + }); + + it('should simply return all versions of a specific library', async () => { + const releases = await getPkgReleases({ + ...config, + depName: 'org.hamcrest:hamcrest-core', + registryUrls: [ + 'file://lib/datasource/maven/__fixtures__/repo1.maven.org/maven2/', + 'file://lib/datasource/maven/__fixtures__/custom_maven_repo/maven2/', + 's3://somewhere.s3.aws.amazon.com', + ], + }); + expect(releases.releases).toEqual( + generateReleases([ + '1.1', + '1.2', + '1.2.1', + '1.3.RC2', + '1.3', + '2.1-rc2', + '2.1-rc3', + ]) + ); + }); + }); +}); diff --git a/lib/datasource/clojure/index.ts b/lib/datasource/clojure/index.ts index 10adb16fe33f932..de4f1e148944d73 100644 --- a/lib/datasource/clojure/index.ts +++ b/lib/datasource/clojure/index.ts @@ -1,8 +1,26 @@ +import { Datasource } from '../datasource'; +import { getReleases } from '../maven'; import { MAVEN_REPO } from '../maven/common'; +import type { GetReleasesConfig, ReleaseResult } from '../types'; -export const id = 'clojure'; -export const customRegistrySupport = true; -export const defaultRegistryUrls = ['https://clojars.org/repo', MAVEN_REPO]; -export const registryStrategy = 'merge'; +export class ClojureDatasource extends Datasource { + static readonly id = 'clojure'; -export { getReleases } from '../maven'; + constructor() { + super(ClojureDatasource.id); + } + + readonly registryStrategy = 'merge'; + + readonly customRegistrySupport = true; + + readonly defaultRegistryUrls = ['https://clojars.org/repo', MAVEN_REPO]; + + // eslint-disable-next-line class-methods-use-this + getReleases({ + lookupName, + registryUrl, + }: GetReleasesConfig): Promise { + return getReleases({ lookupName, registryUrl }); + } +} diff --git a/lib/datasource/datasource.ts b/lib/datasource/datasource.ts new file mode 100644 index 000000000000000..bce5754603ea002 --- /dev/null +++ b/lib/datasource/datasource.ts @@ -0,0 +1,51 @@ +import { ExternalHostError } from '../types/errors/external-host-error'; +import { Http } from '../util/http'; +import type { HttpError } from '../util/http/types'; +import type { + DatasourceApi, + DigestConfig, + GetReleasesConfig, + ReleaseResult, +} from './types'; + +export abstract class Datasource implements DatasourceApi { + protected constructor(public readonly id: string) { + this.http = new Http(id); + } + + caching: boolean; + + defaultConfig: Record; + + customRegistrySupport: boolean; + + defaultRegistryUrls: string[]; + + defaultVersioning: string; + + registryStrategy: 'first' | 'hunt' | 'merge'; + + protected http: Http; + + abstract getReleases( + getReleasesConfig: GetReleasesConfig + ): Promise; + + getDigest?(config: DigestConfig, newValue?: string): Promise; + + // eslint-disable-next-line class-methods-use-this + handleSpecificErrors(err: HttpError): void {} + + protected handleGenericErrors(err: HttpError): never { + this.handleSpecificErrors(err); + if (err.response?.statusCode !== undefined) { + if ( + err.response?.statusCode === 429 || + (err.response?.statusCode >= 500 && err.response?.statusCode < 600) + ) { + throw new ExternalHostError(err); + } + } + throw err; + } +} diff --git a/lib/datasource/index.spec.ts b/lib/datasource/index.spec.ts index 3ccc8fee67a914b..e8623fcea9da82a 100644 --- a/lib/datasource/index.spec.ts +++ b/lib/datasource/index.spec.ts @@ -5,6 +5,7 @@ import { } from '../constants/error-messages'; import { ExternalHostError } from '../types/errors/external-host-error'; import { loadModules } from '../util/modules'; +import { Datasource } from './datasource'; import * as datasourceDocker from './docker'; import * as datasourceGalaxy from './galaxy'; import * as datasourceGithubTags from './github-tags'; @@ -34,19 +35,27 @@ describe(getName(), () => { expect(datasource.getDatasources()).toBeDefined(); expect(datasource.getDatasourceList()).toBeDefined(); }); - it('validates dataource', () => { + it('validates datasource', () => { function validateDatasource(module: DatasourceApi, name: string): boolean { if (!module.getReleases) { return false; } - if (module.id !== name) { - return false; - } - return true; + return module.id === name; + } + function filterClassBasedDatasources(name: string): boolean { + return !['cdnjs', 'clojure'].includes(name); } const dss = datasource.getDatasources(); - const loadedDs = loadModules(__dirname, validateDatasource); + // class based datasources + dss.delete('cdnjs'); + dss.delete('clojure'); + + const loadedDs = loadModules( + __dirname, + validateDatasource, + filterClassBasedDatasources + ); expect(Array.from(dss.keys())).toEqual(Object.keys(loadedDs)); for (const dsName of dss.keys()) { @@ -83,6 +92,14 @@ describe(getName(), () => { }) ).toBeNull(); }); + it('returns class datasource', async () => { + expect( + await datasource.getPkgReleases({ + datasource: 'cdnjs', + depName: null, + }) + ).toBeNull(); + }); it('returns getDigest', async () => { expect( await datasource.getDigest({ diff --git a/lib/datasource/index.ts b/lib/datasource/index.ts index 0f18b8596e3f918..09d3488deddc68a 100644 --- a/lib/datasource/index.ts +++ b/lib/datasource/index.ts @@ -27,7 +27,7 @@ export const getDatasourceList = (): string[] => Array.from(datasources.keys()); const cacheNamespace = 'datasource-releases'; -function load(datasource: string): DatasourceApi { +function getDatasourceFor(datasource: string): DatasourceApi { return datasources.get(datasource); } @@ -204,7 +204,7 @@ function resolveRegistryUrls( } export function getDefaultVersioning(datasourceName: string): string { - const datasource = load(datasourceName); + const datasource = getDatasourceFor(datasourceName); return datasource.defaultVersioning || 'semver'; } @@ -212,11 +212,11 @@ async function fetchReleases( config: GetReleasesInternalConfig ): Promise { const { datasource: datasourceName } = config; - if (!datasourceName || !datasources.has(datasourceName)) { + if (!datasourceName || getDatasourceFor(datasourceName) === undefined) { logger.warn('Unknown datasource: ' + datasourceName); return null; } - const datasource = load(datasourceName); + const datasource = getDatasourceFor(datasourceName); const registryUrls = resolveRegistryUrls(datasource, config.registryUrls); let dep: ReleaseResult = null; const registryStrategy = datasource.registryStrategy || 'hunt'; @@ -358,14 +358,14 @@ export async function getPkgReleases( } export function supportsDigests(config: DigestConfig): boolean { - return 'getDigest' in load(config.datasource); + return 'getDigest' in getDatasourceFor(config.datasource); } export function getDigest( config: DigestConfig, value?: string ): Promise { - const datasource = load(config.datasource); + const datasource = getDatasourceFor(config.datasource); const lookupName = config.lookupName || config.depName; const registryUrls = resolveRegistryUrls(datasource, config.registryUrls); return datasource.getDigest( @@ -377,7 +377,7 @@ export function getDigest( export function getDefaultConfig( datasource: string ): Promise> { - const loadedDatasource = load(datasource); + const loadedDatasource = getDatasourceFor(datasource); return Promise.resolve>( loadedDatasource?.defaultConfig || Object.create({}) ); diff --git a/lib/manager/cdnurl/extract.ts b/lib/manager/cdnurl/extract.ts index b6fb95495681c5b..03f2d085c7a93bb 100644 --- a/lib/manager/cdnurl/extract.ts +++ b/lib/manager/cdnurl/extract.ts @@ -1,4 +1,4 @@ -import * as datasourceCdnjs from '../../datasource/cdnjs'; +import { CdnJsDatasource } from '../../datasource/cdnjs'; import type { PackageDependency, PackageFile } from '../types'; export const cloudflareUrlRegex = /\/\/cdnjs\.cloudflare\.com\/ajax\/libs\/(?[^/]+?)\/(?[^/]+?)\/(?[-/_.a-zA-Z0-9]+)/; @@ -17,7 +17,7 @@ export function extractPackageFile(content: string): PackageFile { match = cloudflareUrlRegex.exec(rest); deps.push({ - datasource: datasourceCdnjs.id, + datasource: CdnJsDatasource.id, depName, lookupName: `${depName}/${asset}`, currentValue, diff --git a/lib/manager/deps-edn/extract.ts b/lib/manager/deps-edn/extract.ts index 4d614145df34275..76aaadfacc97a8f 100644 --- a/lib/manager/deps-edn/extract.ts +++ b/lib/manager/deps-edn/extract.ts @@ -1,4 +1,4 @@ -import * as datasourceClojure from '../../datasource/clojure'; +import { ClojureDatasource } from '../../datasource/clojure'; import { expandDepName } from '../leiningen/extract'; import type { PackageDependency, PackageFile } from '../types'; @@ -16,7 +16,7 @@ export function extractPackageFile(content: string): PackageFile { match = regex.exec(rest); deps.push({ - datasource: datasourceClojure.id, + datasource: ClojureDatasource.id, depName: expandDepName(depName), currentValue, registryUrls: [], diff --git a/lib/manager/html/extract.ts b/lib/manager/html/extract.ts index 5fa618d68f921fe..b027836e78a9449 100644 --- a/lib/manager/html/extract.ts +++ b/lib/manager/html/extract.ts @@ -1,4 +1,4 @@ -import * as datasourceCdnjs from '../../datasource/cdnjs'; +import { CdnJsDatasource } from '../../datasource/cdnjs'; import { cloudflareUrlRegex } from '../cdnurl/extract'; import type { PackageDependency, PackageFile } from '../types'; @@ -13,7 +13,7 @@ export function extractDep(tag: string): PackageDependency | null { } const { depName, currentValue, asset } = match.groups; const dep: PackageDependency = { - datasource: datasourceCdnjs.id, + datasource: CdnJsDatasource.id, depName, lookupName: `${depName}/${asset}`, currentValue, diff --git a/lib/manager/leiningen/extract.spec.ts b/lib/manager/leiningen/extract.spec.ts index 9dd83c4028a05a6..219fac2d98fc5f3 100644 --- a/lib/manager/leiningen/extract.spec.ts +++ b/lib/manager/leiningen/extract.spec.ts @@ -1,5 +1,5 @@ import { getName, loadFixture } from '../../../test/util'; -import * as datasourceClojure from '../../datasource/clojure'; +import { ClojureDatasource } from '../../datasource/clojure'; import { extractFromVectors, extractPackageFile, trimAtKey } from './extract'; const leinProjectClj = loadFixture(`project.clj`); @@ -18,7 +18,7 @@ describe(getName(), () => { expect(extractFromVectors('[[]]')).toEqual([]); expect(extractFromVectors('[[foo/bar "1.2.3"]]')).toEqual([ { - datasource: datasourceClojure.id, + datasource: ClojureDatasource.id, depName: 'foo:bar', currentValue: '1.2.3', }, @@ -27,12 +27,12 @@ describe(getName(), () => { extractFromVectors('[\t[foo/bar "1.2.3"]\n["foo/baz" "4.5.6"] ]') ).toEqual([ { - datasource: datasourceClojure.id, + datasource: ClojureDatasource.id, depName: 'foo:bar', currentValue: '1.2.3', }, { - datasource: datasourceClojure.id, + datasource: ClojureDatasource.id, depName: 'foo:baz', currentValue: '4.5.6', }, diff --git a/lib/manager/leiningen/extract.ts b/lib/manager/leiningen/extract.ts index 360c4ce3d9ab47a..ab9ef4e12d1d43a 100644 --- a/lib/manager/leiningen/extract.ts +++ b/lib/manager/leiningen/extract.ts @@ -1,4 +1,4 @@ -import * as datasourceClojure from '../../datasource/clojure'; +import { ClojureDatasource } from '../../datasource/clojure'; import type { PackageDependency, PackageFile } from '../types'; export function trimAtKey(str: string, kwName: string): string | null { @@ -49,7 +49,7 @@ export function extractFromVectors( if (artifactId && version && fileReplacePosition) { result.push({ ...ctx, - datasource: datasourceClojure.id, + datasource: ClojureDatasource.id, depName: expandDepName(cleanStrLiteral(artifactId)), currentValue: cleanStrLiteral(version), });