From a7b8c4e69ac26d53380e834dd663526c3ba7cd8e Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Sat, 12 Feb 2022 18:52:12 +0300 Subject: [PATCH 1/2] refactor(datasource/docker): Convert to class --- lib/datasource/api.ts | 4 +- lib/datasource/docker/index.spec.ts | 60 +- lib/datasource/docker/index.ts | 1019 +++++++++-------- lib/manager/ansible/index.ts | 4 +- lib/manager/bazel/extract.ts | 4 +- lib/manager/bazel/index.ts | 4 +- lib/manager/bitbucket-pipelines/index.ts | 4 +- lib/manager/circleci/index.ts | 4 +- lib/manager/cloudbuild/index.ts | 4 +- lib/manager/docker-compose/index.ts | 4 +- lib/manager/dockerfile/extract.ts | 4 +- lib/manager/dockerfile/index.ts | 4 +- lib/manager/droneci/index.ts | 4 +- lib/manager/gitlabci/index.ts | 4 +- lib/manager/helm-values/index.ts | 4 +- lib/manager/helmv3/artifacts.ts | 4 +- lib/manager/helmv3/extract.spec.ts | 4 +- lib/manager/helmv3/index.ts | 4 +- lib/manager/helmv3/utils.ts | 4 +- lib/manager/kubernetes/index.ts | 4 +- lib/manager/kustomize/extract.spec.ts | 12 +- lib/manager/kustomize/extract.ts | 8 +- lib/manager/kustomize/index.ts | 4 +- lib/manager/pyenv/extract.ts | 4 +- lib/manager/pyenv/index.ts | 4 +- lib/util/package-rules.spec.ts | 4 +- lib/workers/global/config/parse/cli.spec.ts | 6 +- lib/workers/global/index.spec.ts | 6 +- .../repository/process/lookup/index.spec.ts | 28 +- 29 files changed, 629 insertions(+), 598 deletions(-) diff --git a/lib/datasource/api.ts b/lib/datasource/api.ts index 11d8aaae45e798..7407377e6e49e1 100644 --- a/lib/datasource/api.ts +++ b/lib/datasource/api.ts @@ -7,7 +7,7 @@ import { ClojureDatasource } from './clojure'; import { ConanDatasource } from './conan'; import { CrateDatasource } from './crate'; import { DartDatasource } from './dart'; -import * as docker from './docker'; +import { DockerDatasource } from './docker'; import { GalaxyDatasource } from './galaxy'; import { GalaxyCollectionDatasource } from './galaxy-collection'; import { GitRefsDatasource } from './git-refs'; @@ -52,7 +52,7 @@ api.set(ClojureDatasource.id, new ClojureDatasource()); api.set(ConanDatasource.id, new ConanDatasource()); api.set(CrateDatasource.id, new CrateDatasource()); api.set(DartDatasource.id, new DartDatasource()); -api.set('docker', docker); +api.set(DockerDatasource.id, new DockerDatasource()); api.set(GalaxyDatasource.id, new GalaxyDatasource()); api.set(GalaxyCollectionDatasource.id, new GalaxyCollectionDatasource()); api.set(GitRefsDatasource.id, new GitRefsDatasource()); diff --git a/lib/datasource/docker/index.spec.ts b/lib/datasource/docker/index.spec.ts index c5825477cef933..2ffc289e4449ca 100644 --- a/lib/datasource/docker/index.spec.ts +++ b/lib/datasource/docker/index.spec.ts @@ -4,11 +4,14 @@ import * as httpMock from '../../../test/http-mock'; import { mocked, partial } from '../../../test/util'; import { EXTERNAL_HOST_ERROR } from '../../constants/error-messages'; import * as _hostRules from '../../util/host-rules'; +import { Http } from '../../util/http'; import { MediaType } from './types'; -import { getAuthHeaders, getRegistryRepository, id } from '.'; +import { DockerDatasource, getAuthHeaders, getRegistryRepository } from '.'; const hostRules = mocked(_hostRules); +const http = new Http(DockerDatasource.id); + jest.mock('@aws-sdk/client-ecr'); jest.mock('../../util/host-rules'); @@ -121,6 +124,7 @@ describe('datasource/docker/index', () => { }); const headers = await getAuthHeaders( + http, 'https://my.local.registry', 'https://my.local.registry/prefix' ); @@ -138,6 +142,7 @@ Object { }); const headers = await getAuthHeaders( + http, 'https://my.local.registry', 'https://my.local.registry/prefix' ); @@ -158,6 +163,7 @@ Object { .reply(401, '', {}); const headers = await getAuthHeaders( + http, 'https://my.local.registry', 'https://my.local.registry/prefix' ); @@ -534,7 +540,7 @@ Object { .get('/library/node/tags/list?n=10000') .reply(403); const res = await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'node', registryUrls: ['https://docker.io'], }); @@ -564,7 +570,7 @@ Object { .get('/user/9287/repos?page=3&per_page=100') .reply(200, { tags: ['latest'] }, {}); const config = { - datasource: id, + datasource: DockerDatasource.id, depName: 'node', registryUrls: ['https://registry.company.com'], }; @@ -585,7 +591,7 @@ Object { .get('/node/manifests/1.0.0') .reply(200, '', {}); const res = await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'registry.company.com/node', }); expect(res.releases).toHaveLength(1); @@ -608,7 +614,7 @@ Object { .get('/v2/bitnami/redis/manifests/5.0.12') .reply(200, '', {}); const config = { - datasource: id, + datasource: DockerDatasource.id, depName: 'bitnami/redis', registryUrls: ['https://quay.io'], }; @@ -624,7 +630,7 @@ Object { ) .reply(500); const config = { - datasource: id, + datasource: DockerDatasource.id, depName: 'bitnami/redis', registryUrls: ['https://quay.io'], }; @@ -646,7 +652,7 @@ Object { .reply(200); expect( await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: '123456789.dkr.ecr.us-east-1.amazonaws.com/node', }) ).toEqual({ @@ -700,7 +706,7 @@ Object { }); expect( await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'ecr-proxy.company.com/node', }) ).toEqual({ @@ -735,7 +741,7 @@ Object { }); expect( await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'ecr-proxy.company.com/node', }) ).toBeNull(); @@ -766,7 +772,7 @@ Object { ); expect( await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'ecr-proxy.company.com/node', }) ).toBeNull(); @@ -789,7 +795,7 @@ Object { }); expect( await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'ecr-proxy.company.com/node', }) ).toBeNull(); @@ -818,7 +824,7 @@ Object { ); expect( await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'ecr-proxy.company.com/node', }) ).toBeNull(); @@ -839,7 +845,7 @@ Object { ); expect( await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'ecr-proxy.company.com/node', }) ).toBeNull(); @@ -862,7 +868,7 @@ Object { ); expect( await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'ecr-proxy.company.com/node', }) ).toBeNull(); @@ -889,7 +895,7 @@ Object { ); expect( await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'ecr-proxy.company.com/node', }) ).toBeNull(); @@ -917,7 +923,7 @@ Object { ); expect( await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'ecr-proxy.company.com/node', }) ).toBeNull(); @@ -946,7 +952,7 @@ Object { ) .reply(200, { token: 'test' }); const res = await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'node', }); expect(res.releases).toHaveLength(1); @@ -974,7 +980,7 @@ Object { ) .reply(200, { token: 'test' }); const res = await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'docker.io/node', }); expect(res.releases).toHaveLength(1); @@ -1000,7 +1006,7 @@ Object { .get('/kubernetes-dashboard-amd64/manifests/1.0.0') .reply(200); const res = await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'k8s.gcr.io/kubernetes-dashboard-amd64', }); expect(res.releases).toHaveLength(1); @@ -1014,7 +1020,7 @@ Object { .get('/my/node/tags/list?n=10000') .replyWithError('error'); const res = await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'my/node', }); expect(res).toBeNull(); @@ -1039,7 +1045,7 @@ Object { .get('/token?service=registry.docker.io&scope=repository:my/node:pull') .reply(200, { token: 'some-token ' }); const res = await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'my/node', registryUrls: ['https://index.docker.io/'], }); @@ -1052,7 +1058,7 @@ Object { 'www-authenticate': 'Basic realm="My Private Docker Registry Server"', }); const res = await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'node', }); expect(res).toBeNull(); @@ -1092,7 +1098,7 @@ Object { }, }); const res = await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'registry.company.com/node', }); expect(res).toMatchSnapshot(); @@ -1128,7 +1134,7 @@ Object { }, }); const res = await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'registry.company.com/node', }); expect(res).toMatchSnapshot(); @@ -1148,7 +1154,7 @@ Object { mediaType: MediaType.manifestV1, }); const res = await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'registry.company.com/node', }); expect(res).toMatchSnapshot(); @@ -1165,7 +1171,7 @@ Object { .get('/node/manifests/latest') .reply(200, {}); const res = await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'registry.company.com/node', }); expect(res).toMatchSnapshot(); @@ -1208,7 +1214,7 @@ Object { config: {}, }); const res = await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'registry.company.com/node', }); expect(res).toMatchSnapshot(); diff --git a/lib/datasource/docker/index.ts b/lib/datasource/docker/index.ts index cce8d732ab3843..2c2ea7961938b2 100644 --- a/lib/datasource/docker/index.ts +++ b/lib/datasource/docker/index.ts @@ -10,7 +10,7 @@ import type { HostRule } from '../../types'; import { ExternalHostError } from '../../types/errors/external-host-error'; import * as packageCache from '../../util/cache/package'; import * as hostRules from '../../util/host-rules'; -import { Http, HttpOptions, HttpResponse } from '../../util/http'; +import type { Http, HttpOptions, HttpResponse } from '../../util/http'; import { HttpError } from '../../util/http/types'; import type { OutgoingHttpHeaders } from '../../util/http/types'; import { hasKey } from '../../util/object'; @@ -26,60 +26,22 @@ import { api as dockerVersioning, id as dockerVersioningId, } from '../../versioning/docker'; +import { Datasource } from '../datasource'; import type { GetReleasesConfig, ReleaseResult } from '../types'; import { sourceLabels } from './common'; import { Image, ImageList, MediaType, RegistryRepository } from './types'; -export const ecrRegex = regEx(/\d+\.dkr\.ecr\.([-a-z0-9]+)\.amazonaws\.com/); - -export const id = 'docker'; -export const http = new Http(id); - -const DOCKER_HUB = 'https://index.docker.io'; -export const defaultRegistryUrls = [DOCKER_HUB]; +export const DOCKER_HUB = 'https://index.docker.io'; -// TODO: add got typings when available (#9646) - -export const customRegistrySupport = true; -export const defaultVersioning = dockerVersioningId; -export const registryStrategy = 'first'; +export const ecrRegex = regEx(/\d+\.dkr\.ecr\.([-a-z0-9]+)\.amazonaws\.com/); function isDockerHost(host: string): boolean { const regex = regEx(/(?:^|\.)docker\.io$/); return regex.test(host); } -async function getECRAuthToken( - region: string, - opts: HostRule -): Promise { - const config: ECRClientConfig = { region }; - if (opts.username && opts.password) { - config.credentials = { - accessKeyId: opts.username, - secretAccessKey: opts.password, - ...(opts.token && { sessionToken: opts.token }), - }; - } - - const ecr = new ECR(config); - try { - const data = await ecr.getAuthorizationToken({}); - const authorizationToken = data?.authorizationData?.[0]?.authorizationToken; - if (authorizationToken) { - return authorizationToken; - } - logger.warn( - 'Could not extract authorizationToken from ECR getAuthorizationToken response' - ); - } catch (err) { - logger.trace({ err }, 'err'); - logger.debug('ECR getAuthorizationToken error'); - } - return null; -} - export async function getAuthHeaders( + http: Http, registryHost: string, dockerRepository: string ): Promise { @@ -110,7 +72,7 @@ export async function getAuthHeaders( ); const opts: HostRule & HttpOptions = hostRules.find({ - hostType: id, + hostType: DockerDatasource.id, url: apiCheckUrl, }); if (ecrRegex.test(registryHost)) { @@ -227,6 +189,36 @@ export async function getAuthHeaders( } } +async function getECRAuthToken( + region: string, + opts: HostRule +): Promise { + const config: ECRClientConfig = { region }; + if (opts.username && opts.password) { + config.credentials = { + accessKeyId: opts.username, + secretAccessKey: opts.password, + ...(opts.token && { sessionToken: opts.token }), + }; + } + + const ecr = new ECR(config); + try { + const data = await ecr.getAuthorizationToken({}); + const authorizationToken = data?.authorizationData?.[0]?.authorizationToken; + if (authorizationToken) { + return authorizationToken; + } + logger.warn( + 'Could not extract authorizationToken from ECR getAuthorizationToken response' + ); + } catch (err) { + logger.trace({ err }, 'err'); + logger.debug('ECR getAuthorizationToken error'); + } + return null; +} + export function getRegistryRepository( lookupName: string, registryUrl: string @@ -270,7 +262,10 @@ export function getRegistryRepository( if (!regEx(/^https?:\/\//).exec(registryHost)) { registryHost = `https://${registryHost}`; } - const opts = hostRules.find({ hostType: id, url: registryHost }); + const opts = hostRules.find({ + hostType: DockerDatasource.id, + url: registryHost, + }); if (opts?.insecureRegistry) { registryHost = registryHost.replace('https', 'http'); } @@ -293,245 +288,6 @@ export function extractDigestFromResponseBody( return digestFromManifestStr(manifestResponse.body); } -// TODO: debug why quay throws errors (#9612) -export async function getManifestResponse( - registryHost: string, - dockerRepository: string, - tag: string, - mode: 'head' | 'get' = 'get' -): Promise { - logger.debug( - `getManifestResponse(${registryHost}, ${dockerRepository}, ${tag})` - ); - try { - const headers = await getAuthHeaders(registryHost, dockerRepository); - if (!headers) { - logger.debug('No docker auth found - returning'); - return null; - } - headers.accept = - 'application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json'; - const url = `${registryHost}/v2/${dockerRepository}/manifests/${tag}`; - const manifestResponse = await http[mode](url, { - headers, - noAuth: true, - }); - return manifestResponse; - } catch (err) /* istanbul ignore next */ { - if (err instanceof ExternalHostError) { - throw err; - } - if (err.statusCode === 401) { - logger.debug( - { registryHost, dockerRepository }, - 'Unauthorized docker lookup' - ); - logger.debug({ err }); - return null; - } - if (err.statusCode === 404) { - logger.debug( - { - err, - registryHost, - dockerRepository, - tag, - }, - 'Docker Manifest is unknown' - ); - return null; - } - if (err.statusCode === 429 && isDockerHost(registryHost)) { - throw new ExternalHostError(err); - } - if (err.statusCode >= 500 && err.statusCode < 600) { - throw new ExternalHostError(err); - } - if (err.code === 'ETIMEDOUT') { - logger.debug( - { registryHost }, - 'Timeout when attempting to connect to docker registry' - ); - logger.debug({ err }); - return null; - } - logger.debug( - { - err, - registryHost, - dockerRepository, - tag, - }, - 'Unknown Error looking up docker manifest' - ); - return null; - } -} - -async function getConfigDigest( - registry: string, - dockerRepository: string, - tag: string -): Promise { - const manifestResponse = await getManifestResponse( - registry, - dockerRepository, - tag - ); - // If getting the manifest fails here, then abort - // This means that the latest tag doesn't have a manifest, which shouldn't - // be possible - // istanbul ignore if - if (!manifestResponse) { - return null; - } - const manifest = JSON.parse(manifestResponse.body) as ImageList | Image; - if (manifest.schemaVersion !== 2) { - logger.debug( - { registry, dockerRepository, tag }, - 'Manifest schema version is not 2' - ); - return null; - } - - if ( - manifest.mediaType === MediaType.manifestListV2 && - manifest.manifests.length - ) { - logger.trace( - { registry, dockerRepository, tag }, - 'Found manifest list, using first image' - ); - return getConfigDigest( - registry, - dockerRepository, - manifest.manifests[0].digest - ); - } - - if ( - manifest.mediaType === MediaType.manifestV2 && - is.string(manifest.config?.digest) - ) { - return manifest.config?.digest; - } - - logger.debug({ manifest }, 'Invalid manifest - returning'); - return null; -} - -/* - * docker.getLabels - * - * This function will: - * - Return the labels for the requested image - */ - -export async function getLabels( - registryHost: string, - dockerRepository: string, - tag: string -): Promise> { - logger.debug(`getLabels(${registryHost}, ${dockerRepository}, ${tag})`); - const cacheNamespace = 'datasource-docker-labels'; - const cacheKey = `${registryHost}:${dockerRepository}:${tag}`; - const cachedResult = await packageCache.get>( - cacheNamespace, - cacheKey - ); - // istanbul ignore if - if (cachedResult !== undefined) { - return cachedResult; - } - try { - let labels: Record = {}; - const configDigest = await getConfigDigest( - registryHost, - dockerRepository, - tag - ); - if (!configDigest) { - return {}; - } - - const headers = await getAuthHeaders(registryHost, dockerRepository); - // istanbul ignore if: Should never be happen - if (!headers) { - logger.debug('No docker auth found - returning'); - return {}; - } - const url = `${registryHost}/v2/${dockerRepository}/blobs/${configDigest}`; - const configResponse = await http.get(url, { - headers, - noAuth: true, - }); - labels = JSON.parse(configResponse.body).config.Labels; - - if (labels) { - logger.debug( - { - labels, - }, - 'found labels in manifest' - ); - } - const cacheMinutes = 60; - await packageCache.set(cacheNamespace, cacheKey, labels, cacheMinutes); - return labels; - } catch (err) /* istanbul ignore next: should be tested in future */ { - if (err instanceof ExternalHostError) { - throw err; - } - if (err.statusCode === 400 || err.statusCode === 401) { - logger.debug( - { registryHost, dockerRepository, err }, - 'Unauthorized docker lookup' - ); - } else if (err.statusCode === 404) { - logger.warn( - { - err, - registryHost, - dockerRepository, - tag, - }, - 'Config Manifest is unknown' - ); - } else if (err.statusCode === 429 && isDockerHost(registryHost)) { - logger.warn({ err }, 'docker registry failure: too many requests'); - } else if (err.statusCode >= 500 && err.statusCode < 600) { - logger.debug( - { - err, - registryHost, - dockerRepository, - tag, - }, - 'docker registry failure: internal error' - ); - } else if ( - err.code === 'ERR_TLS_CERT_ALTNAME_INVALID' || - err.code === 'ETIMEDOUT' - ) { - logger.debug( - { registryHost, err }, - 'Error connecting to docker registry' - ); - } else if (registryHost === 'https://quay.io') { - // istanbul ignore next - logger.debug( - 'Ignoring quay.io errors until they fully support v2 schema' - ); - } else { - logger.info( - { registryHost, dockerRepository, tag, err }, - 'Unknown error getting Docker labels' - ); - } - return {}; - } -} - export function isECRMaxResultsError(err: HttpError): boolean { return !!( err.response?.statusCode === 405 && @@ -543,31 +299,6 @@ export function isECRMaxResultsError(err: HttpError): boolean { ); } -export async function getTagsQuayRegistry( - registry: string, - repository: string -): Promise { - let tags: string[] = []; - const limit = 100; - - const pageUrl = (page: number): string => - `${registry}/api/v1/repository/${repository}/tag/?limit=${limit}&page=${page}&onlyActiveTags=true`; - - let page = 1; - let url = pageUrl(page); - do { - const res = await http.getJson<{ - tags: { name: string }[]; - has_additional: boolean; - }>(url, {}); - const pageTags = res.body.tags.map((tag) => tag.name); - tags = tags.concat(pageTags); - page += 1; - url = res.body.has_additional ? pageUrl(page) : null; - } while (url && page < 20); - return tags; -} - export const defaultConfig = { commitMessageTopic: '{{{depName}}} Docker tag', commitMessageExtra: @@ -595,149 +326,171 @@ export const defaultConfig = { }, }; -async function getDockerApiTags( - registryHost: string, - dockerRepository: string -): Promise { - let tags: string[] = []; - // AWS ECR limits the maximum number of results to 1000 - // See https://docs.aws.amazon.com/AmazonECR/latest/APIReference/API_DescribeRepositories.html#ECR-DescribeRepositories-request-maxResults - const limit = ecrRegex.test(registryHost) ? 1000 : 10000; - let url = `${registryHost}/${dockerRepository}/tags/list?n=${limit}`; - url = ensurePathPrefix(url, '/v2'); - const headers = await getAuthHeaders(registryHost, dockerRepository); - if (!headers) { - logger.debug('Failed to get authHeaders for getTags lookup'); - return null; +function findLatestStable(tags: string[]): string { + const versions = tags + .filter((v) => dockerVersioning.isValid(v) && dockerVersioning.isStable(v)) + .sort((a, b) => dockerVersioning.sortVersions(a, b)); + + return versions.pop() ?? tags.slice(-1).pop(); +} + +export class DockerDatasource extends Datasource { + static readonly id = 'docker'; + + override readonly defaultVersioning = dockerVersioningId; + + override readonly defaultRegistryUrls = [DOCKER_HUB]; + + constructor() { + super(DockerDatasource.id); } - let page = 1; - let foundMaxResultsError = false; - do { - let res: HttpResponse<{ tags: string[] }>; + + // TODO: debug why quay throws errors (#9612) + private async getManifestResponse( + registryHost: string, + dockerRepository: string, + tag: string, + mode: 'head' | 'get' = 'get' + ): Promise { + logger.debug( + `getManifestResponse(${registryHost}, ${dockerRepository}, ${tag})` + ); try { - res = await http.getJson<{ tags: string[] }>(url, { + const headers = await getAuthHeaders( + this.http, + registryHost, + dockerRepository + ); + if (!headers) { + logger.debug('No docker auth found - returning'); + return null; + } + headers.accept = + 'application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json'; + const url = `${registryHost}/v2/${dockerRepository}/manifests/${tag}`; + const manifestResponse = await this.http[mode](url, { headers, noAuth: true, }); - } catch (err) { - if ( - !foundMaxResultsError && - err instanceof HttpError && - isECRMaxResultsError(err) - ) { - const maxResults = 1000; - url = `${registryHost}/${dockerRepository}/tags/list?n=${maxResults}`; - url = ensurePathPrefix(url, '/v2'); - foundMaxResultsError = true; - continue; + return manifestResponse; + } catch (err) /* istanbul ignore next */ { + if (err instanceof ExternalHostError) { + throw err; } - throw err; + if (err.statusCode === 401) { + logger.debug( + { registryHost, dockerRepository }, + 'Unauthorized docker lookup' + ); + logger.debug({ err }); + return null; + } + if (err.statusCode === 404) { + logger.debug( + { + err, + registryHost, + dockerRepository, + tag, + }, + 'Docker Manifest is unknown' + ); + return null; + } + if (err.statusCode === 429 && isDockerHost(registryHost)) { + throw new ExternalHostError(err); + } + if (err.statusCode >= 500 && err.statusCode < 600) { + throw new ExternalHostError(err); + } + if (err.code === 'ETIMEDOUT') { + logger.debug( + { registryHost }, + 'Timeout when attempting to connect to docker registry' + ); + logger.debug({ err }); + return null; + } + logger.debug( + { + err, + registryHost, + dockerRepository, + tag, + }, + 'Unknown Error looking up docker manifest' + ); + return null; } - tags = tags.concat(res.body.tags); - const linkHeader = parseLinkHeader(res.headers.link); - url = linkHeader?.next ? URL.resolve(url, linkHeader.next.url) : null; - page += 1; - } while (url && page < 20); - return tags; -} + } -async function getTags( - registryHost: string, - dockerRepository: string -): Promise { - try { - const cacheNamespace = 'datasource-docker-tags'; - const cacheKey = `${registryHost}:${dockerRepository}`; - const cachedResult = await packageCache.get( - cacheNamespace, - cacheKey + private async getConfigDigest( + registry: string, + dockerRepository: string, + tag: string + ): Promise { + const manifestResponse = await this.getManifestResponse( + registry, + dockerRepository, + tag ); + // If getting the manifest fails here, then abort + // This means that the latest tag doesn't have a manifest, which shouldn't + // be possible // istanbul ignore if - if (cachedResult !== undefined) { - return cachedResult; - } - - const isQuay = regEx(/^https:\/\/quay\.io(?::[1-9][0-9]{0,4})?$/i).test( - registryHost - ); - let tags: string[] | null; - if (isQuay) { - tags = await getTagsQuayRegistry(registryHost, dockerRepository); - } else { - tags = await getDockerApiTags(registryHost, dockerRepository); - } - const cacheMinutes = 30; - await packageCache.set(cacheNamespace, cacheKey, tags, cacheMinutes); - return tags; - } catch (err) /* istanbul ignore next */ { - if (err instanceof ExternalHostError) { - throw err; + if (!manifestResponse) { + return null; } - if (err.statusCode === 404 && !dockerRepository.includes('/')) { + const manifest = JSON.parse(manifestResponse.body) as ImageList | Image; + if (manifest.schemaVersion !== 2) { logger.debug( - `Retrying Tags for ${registryHost}/${dockerRepository} using library/ prefix` - ); - return getTags(registryHost, 'library/' + dockerRepository); - } - // prettier-ignore - if (err.statusCode === 429 && isDockerHost(registryHost)) { - logger.warn( - { registryHost, dockerRepository, err }, - 'docker registry failure: too many requests' + { registry, dockerRepository, tag }, + 'Manifest schema version is not 2' ); - throw new ExternalHostError(err); + return null; } - // prettier-ignore - if (err.statusCode === 401 && isDockerHost(registryHost)) { - logger.warn( - { registryHost, dockerRepository, err }, - 'docker registry failure: unauthorized' + + if ( + manifest.mediaType === MediaType.manifestListV2 && + manifest.manifests.length + ) { + logger.trace( + { registry, dockerRepository, tag }, + 'Found manifest list, using first image' ); - throw new ExternalHostError(err); - } - if (err.statusCode >= 500 && err.statusCode < 600) { - logger.warn( - { registryHost, dockerRepository, err }, - 'docker registry failure: internal error' + return this.getConfigDigest( + registry, + dockerRepository, + manifest.manifests[0].digest ); - throw new ExternalHostError(err); } - throw err; - } -} -function findLatestStable(tags: string[]): string { - const versions = tags - .filter((v) => dockerVersioning.isValid(v) && dockerVersioning.isStable(v)) - .sort((a, b) => dockerVersioning.sortVersions(a, b)); + if ( + manifest.mediaType === MediaType.manifestV2 && + is.string(manifest.config?.digest) + ) { + return manifest.config?.digest; + } - return versions.pop() ?? tags.slice(-1).pop(); -} + logger.debug({ manifest }, 'Invalid manifest - returning'); + return null; + } -/** - * docker.getDigest - * - * The `newValue` supplied here should be a valid tag for the docker image. - * - * This function will: - * - Look up a sha256 digest for a tag on its registry - * - Return the digest as a string - */ -export async function getDigest( - { registryUrl, lookupName }: GetReleasesConfig, - newValue?: string -): Promise { - const { registryHost, dockerRepository } = getRegistryRepository( - lookupName, - registryUrl - ); - logger.debug(`getDigest(${registryHost}, ${dockerRepository}, ${newValue})`); - const newTag = newValue || 'latest'; - const cacheNamespace = 'datasource-docker-digest'; - const cacheKey = `${registryHost}:${dockerRepository}:${newTag}`; - let digest: string = null; - try { - const cachedResult = await packageCache.get( + /* + * docker.getLabels + * + * This function will: + * - Return the labels for the requested image + */ + private async getLabels( + registryHost: string, + dockerRepository: string, + tag: string + ): Promise> { + logger.debug(`getLabels(${registryHost}, ${dockerRepository}, ${tag})`); + const cacheNamespace = 'datasource-docker-labels'; + const cacheKey = `${registryHost}:${dockerRepository}:${tag}`; + const cachedResult = await packageCache.get>( cacheNamespace, cacheKey ); @@ -745,86 +498,360 @@ export async function getDigest( if (cachedResult !== undefined) { return cachedResult; } - let manifestResponse = await getManifestResponse( + try { + let labels: Record = {}; + const configDigest = await this.getConfigDigest( + registryHost, + dockerRepository, + tag + ); + if (!configDigest) { + return {}; + } + + const headers = await getAuthHeaders( + this.http, + registryHost, + dockerRepository + ); + // istanbul ignore if: Should never be happen + if (!headers) { + logger.debug('No docker auth found - returning'); + return {}; + } + const url = `${registryHost}/v2/${dockerRepository}/blobs/${configDigest}`; + const configResponse = await this.http.get(url, { + headers, + noAuth: true, + }); + labels = JSON.parse(configResponse.body).config.Labels; + + if (labels) { + logger.debug( + { + labels, + }, + 'found labels in manifest' + ); + } + const cacheMinutes = 60; + await packageCache.set(cacheNamespace, cacheKey, labels, cacheMinutes); + return labels; + } catch (err) /* istanbul ignore next: should be tested in future */ { + if (err instanceof ExternalHostError) { + throw err; + } + if (err.statusCode === 400 || err.statusCode === 401) { + logger.debug( + { registryHost, dockerRepository, err }, + 'Unauthorized docker lookup' + ); + } else if (err.statusCode === 404) { + logger.warn( + { + err, + registryHost, + dockerRepository, + tag, + }, + 'Config Manifest is unknown' + ); + } else if (err.statusCode === 429 && isDockerHost(registryHost)) { + logger.warn({ err }, 'docker registry failure: too many requests'); + } else if (err.statusCode >= 500 && err.statusCode < 600) { + logger.debug( + { + err, + registryHost, + dockerRepository, + tag, + }, + 'docker registry failure: internal error' + ); + } else if ( + err.code === 'ERR_TLS_CERT_ALTNAME_INVALID' || + err.code === 'ETIMEDOUT' + ) { + logger.debug( + { registryHost, err }, + 'Error connecting to docker registry' + ); + } else if (registryHost === 'https://quay.io') { + // istanbul ignore next + logger.debug( + 'Ignoring quay.io errors until they fully support v2 schema' + ); + } else { + logger.info( + { registryHost, dockerRepository, tag, err }, + 'Unknown error getting Docker labels' + ); + } + return {}; + } + } + + private async getTagsQuayRegistry( + registry: string, + repository: string + ): Promise { + let tags: string[] = []; + const limit = 100; + + const pageUrl = (page: number): string => + `${registry}/api/v1/repository/${repository}/tag/?limit=${limit}&page=${page}&onlyActiveTags=true`; + + let page = 1; + let url = pageUrl(page); + do { + const res = await this.http.getJson<{ + tags: { name: string }[]; + has_additional: boolean; + }>(url, {}); + const pageTags = res.body.tags.map((tag) => tag.name); + tags = tags.concat(pageTags); + page += 1; + url = res.body.has_additional ? pageUrl(page) : null; + } while (url && page < 20); + return tags; + } + + private async getDockerApiTags( + registryHost: string, + dockerRepository: string + ): Promise { + let tags: string[] = []; + // AWS ECR limits the maximum number of results to 1000 + // See https://docs.aws.amazon.com/AmazonECR/latest/APIReference/API_DescribeRepositories.html#ECR-DescribeRepositories-request-maxResults + const limit = ecrRegex.test(registryHost) ? 1000 : 10000; + let url = `${registryHost}/${dockerRepository}/tags/list?n=${limit}`; + url = ensurePathPrefix(url, '/v2'); + const headers = await getAuthHeaders( + this.http, registryHost, - dockerRepository, - newTag, - 'head' + dockerRepository ); - if (manifestResponse) { - if (hasKey('docker-content-digest', manifestResponse.headers)) { - digest = - (manifestResponse.headers['docker-content-digest'] as string) || null; + if (!headers) { + logger.debug('Failed to get authHeaders for getTags lookup'); + return null; + } + let page = 1; + let foundMaxResultsError = false; + do { + let res: HttpResponse<{ tags: string[] }>; + try { + res = await this.http.getJson<{ tags: string[] }>(url, { + headers, + noAuth: true, + }); + } catch (err) { + if ( + !foundMaxResultsError && + err instanceof HttpError && + isECRMaxResultsError(err) + ) { + const maxResults = 1000; + url = `${registryHost}/${dockerRepository}/tags/list?n=${maxResults}`; + url = ensurePathPrefix(url, '/v2'); + foundMaxResultsError = true; + continue; + } + throw err; + } + tags = tags.concat(res.body.tags); + const linkHeader = parseLinkHeader(res.headers.link); + url = linkHeader?.next ? URL.resolve(url, linkHeader.next.url) : null; + page += 1; + } while (url && page < 20); + return tags; + } + + private async getTags( + registryHost: string, + dockerRepository: string + ): Promise { + try { + const cacheNamespace = 'datasource-docker-tags'; + const cacheKey = `${registryHost}:${dockerRepository}`; + const cachedResult = await packageCache.get( + cacheNamespace, + cacheKey + ); + // istanbul ignore if + if (cachedResult !== undefined) { + return cachedResult; + } + + const isQuay = regEx(/^https:\/\/quay\.io(?::[1-9][0-9]{0,4})?$/i).test( + registryHost + ); + let tags: string[] | null; + if (isQuay) { + tags = await this.getTagsQuayRegistry(registryHost, dockerRepository); } else { + tags = await this.getDockerApiTags(registryHost, dockerRepository); + } + const cacheMinutes = 30; + await packageCache.set(cacheNamespace, cacheKey, tags, cacheMinutes); + return tags; + } catch (err) /* istanbul ignore next */ { + if (err instanceof ExternalHostError) { + throw err; + } + if (err.statusCode === 404 && !dockerRepository.includes('/')) { logger.debug( - { registryHost }, - 'Missing docker content digest header, pulling full manifest' + `Retrying Tags for ${registryHost}/${dockerRepository} using library/ prefix` ); - manifestResponse = await getManifestResponse( - registryHost, - dockerRepository, - newTag + return this.getTags(registryHost, 'library/' + dockerRepository); + } + // prettier-ignore + if (err.statusCode === 429 && isDockerHost(registryHost)) { + logger.warn( + { registryHost, dockerRepository, err }, + 'docker registry failure: too many requests' ); - digest = extractDigestFromResponseBody(manifestResponse); + throw new ExternalHostError(err); + } + // prettier-ignore + if (err.statusCode === 401 && isDockerHost(registryHost)) { + logger.warn( + { registryHost, dockerRepository, err }, + 'docker registry failure: unauthorized' + ); + throw new ExternalHostError(err); + } + if (err.statusCode >= 500 && err.statusCode < 600) { + logger.warn( + { registryHost, dockerRepository, err }, + 'docker registry failure: internal error' + ); + throw new ExternalHostError(err); } - logger.debug({ digest }, 'Got docker digest'); - } - } catch (err) /* istanbul ignore next */ { - if (err instanceof ExternalHostError) { throw err; } + } + + /** + * docker.getDigest + * + * The `newValue` supplied here should be a valid tag for the docker image. + * + * This function will: + * - Look up a sha256 digest for a tag on its registry + * - Return the digest as a string + */ + override async getDigest( + { registryUrl, lookupName }: GetReleasesConfig, + newValue?: string + ): Promise { + const { registryHost, dockerRepository } = getRegistryRepository( + lookupName, + registryUrl + ); logger.debug( - { - err, - lookupName, - newTag, - }, - 'Unknown Error looking up docker image digest' + `getDigest(${registryHost}, ${dockerRepository}, ${newValue})` ); + const newTag = newValue || 'latest'; + const cacheNamespace = 'datasource-docker-digest'; + const cacheKey = `${registryHost}:${dockerRepository}:${newTag}`; + let digest: string = null; + try { + const cachedResult = await packageCache.get( + cacheNamespace, + cacheKey + ); + // istanbul ignore if + if (cachedResult !== undefined) { + return cachedResult; + } + let manifestResponse = await this.getManifestResponse( + registryHost, + dockerRepository, + newTag, + 'head' + ); + if (manifestResponse) { + if (hasKey('docker-content-digest', manifestResponse.headers)) { + digest = + (manifestResponse.headers['docker-content-digest'] as string) || + null; + } else { + logger.debug( + { registryHost }, + 'Missing docker content digest header, pulling full manifest' + ); + manifestResponse = await this.getManifestResponse( + registryHost, + dockerRepository, + newTag + ); + digest = extractDigestFromResponseBody(manifestResponse); + } + logger.debug({ digest }, 'Got docker digest'); + } + } catch (err) /* istanbul ignore next */ { + if (err instanceof ExternalHostError) { + throw err; + } + logger.debug( + { + err, + lookupName, + newTag, + }, + 'Unknown Error looking up docker image digest' + ); + } + const cacheMinutes = 30; + await packageCache.set(cacheNamespace, cacheKey, digest, cacheMinutes); + return digest; } - const cacheMinutes = 30; - await packageCache.set(cacheNamespace, cacheKey, digest, cacheMinutes); - return digest; -} -/** - * docker.getReleases - * - * A docker image usually looks something like this: somehost.io/owner/repo:8.1.0-alpine - * In the above: - * - 'somehost.io' is the registry - * - 'owner/repo' is the package name - * - '8.1.0-alpine' is the tag - * - * This function will filter only tags that contain a semver version - */ -export async function getReleases({ - lookupName, - registryUrl, -}: GetReleasesConfig): Promise { - const { registryHost, dockerRepository } = getRegistryRepository( + /** + * docker.getReleases + * + * A docker image usually looks something like this: somehost.io/owner/repo:8.1.0-alpine + * In the above: + * - 'somehost.io' is the registry + * - 'owner/repo' is the package name + * - '8.1.0-alpine' is the tag + * + * This function will filter only tags that contain a semver version + */ + async getReleases({ lookupName, - registryUrl - ); - const tags = await getTags(registryHost, dockerRepository); - if (!tags) { - return null; - } - const releases = tags.map((version) => ({ version })); - const ret: ReleaseResult = { - registryUrl: registryHost, - releases, - }; + registryUrl, + }: GetReleasesConfig): Promise { + const { registryHost, dockerRepository } = getRegistryRepository( + lookupName, + registryUrl + ); + const tags = await this.getTags(registryHost, dockerRepository); + if (!tags) { + return null; + } + const releases = tags.map((version) => ({ version })); + const ret: ReleaseResult = { + registryUrl: registryHost, + releases, + }; - const latestTag = tags.includes('latest') ? 'latest' : findLatestStable(tags); - const labels = await getLabels(registryHost, dockerRepository, latestTag); - if (labels) { - for (const label of sourceLabels) { - if (is.nonEmptyString(labels[label])) { - ret.sourceUrl = labels[label]; - break; + const latestTag = tags.includes('latest') + ? 'latest' + : findLatestStable(tags); + const labels = await this.getLabels( + registryHost, + dockerRepository, + latestTag + ); + if (labels) { + for (const label of sourceLabels) { + if (is.nonEmptyString(labels[label])) { + ret.sourceUrl = labels[label]; + break; + } } } + return ret; } - return ret; } diff --git a/lib/manager/ansible/index.ts b/lib/manager/ansible/index.ts index cbe22b450165cd..d1a13cdbea55fb 100644 --- a/lib/manager/ansible/index.ts +++ b/lib/manager/ansible/index.ts @@ -1,5 +1,5 @@ import { ProgrammingLanguage } from '../../constants'; -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; export { extractPackageFile } from './extract'; export const language = ProgrammingLanguage.Docker; @@ -8,4 +8,4 @@ export const defaultConfig = { fileMatch: ['(^|/)tasks/[^/]+\\.ya?ml$'], }; -export const supportedDatasources = [datasourceDocker.id]; +export const supportedDatasources = [DockerDatasource.id]; diff --git a/lib/manager/bazel/extract.ts b/lib/manager/bazel/extract.ts index a35ac770be7317..180042a2ef69c8 100644 --- a/lib/manager/bazel/extract.ts +++ b/lib/manager/bazel/extract.ts @@ -2,7 +2,7 @@ import { parse as _parse } from 'url'; import parse from 'github-url-from-git'; import moo from 'moo'; -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import * as datasourceGithubReleases from '../../datasource/github-releases'; import * as datasourceGithubTags from '../../datasource/github-tags'; import { GoDatasource } from '../../datasource/go'; @@ -300,7 +300,7 @@ export function extractPackageFile( dep.currentValue = currentValue; dep.depName = depName; dep.versioning = dockerVersioning.id; - dep.datasource = datasourceDocker.id; + dep.datasource = DockerDatasource.id; dep.lookupName = repository; dep.registryUrls = [registry]; deps.push(dep); diff --git a/lib/manager/bazel/index.ts b/lib/manager/bazel/index.ts index aa0568fcec5588..9de5569ecb5cd3 100644 --- a/lib/manager/bazel/index.ts +++ b/lib/manager/bazel/index.ts @@ -1,4 +1,4 @@ -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import * as datasourceGithubReleases from '../../datasource/github-releases'; import * as datasourceGithubTags from '../../datasource/github-tags'; import { GoDatasource } from '../../datasource/go'; @@ -12,7 +12,7 @@ export const defaultConfig = { }; export const supportedDatasources = [ - datasourceDocker.id, + DockerDatasource.id, datasourceGithubReleases.id, datasourceGithubTags.id, GoDatasource.id, diff --git a/lib/manager/bitbucket-pipelines/index.ts b/lib/manager/bitbucket-pipelines/index.ts index a51d837bc58277..930c4a868f90d6 100644 --- a/lib/manager/bitbucket-pipelines/index.ts +++ b/lib/manager/bitbucket-pipelines/index.ts @@ -1,4 +1,4 @@ -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import { extractPackageFile } from './extract'; export { extractPackageFile }; @@ -7,4 +7,4 @@ export const defaultConfig = { fileMatch: ['(^|/)\\.?bitbucket-pipelines\\.ya?ml$'], }; -export const supportedDatasources = [datasourceDocker.id]; +export const supportedDatasources = [DockerDatasource.id]; diff --git a/lib/manager/circleci/index.ts b/lib/manager/circleci/index.ts index 705099e16def2c..6bb051449a4f78 100644 --- a/lib/manager/circleci/index.ts +++ b/lib/manager/circleci/index.ts @@ -1,4 +1,4 @@ -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import { OrbDatasource } from '../../datasource/orb'; import { extractPackageFile } from './extract'; @@ -8,4 +8,4 @@ export const defaultConfig = { fileMatch: ['(^|/).circleci/config.yml$'], }; -export const supportedDatasources = [datasourceDocker.id, OrbDatasource.id]; +export const supportedDatasources = [DockerDatasource.id, OrbDatasource.id]; diff --git a/lib/manager/cloudbuild/index.ts b/lib/manager/cloudbuild/index.ts index cb4dc10014fd2f..b1c2d7f1c43142 100644 --- a/lib/manager/cloudbuild/index.ts +++ b/lib/manager/cloudbuild/index.ts @@ -1,4 +1,4 @@ -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import { extractPackageFile } from './extract'; export { extractPackageFile }; @@ -7,4 +7,4 @@ export const defaultConfig = { fileMatch: ['(^|/)cloudbuild.ya?ml'], }; -export const supportedDatasources = [datasourceDocker.id]; +export const supportedDatasources = [DockerDatasource.id]; diff --git a/lib/manager/docker-compose/index.ts b/lib/manager/docker-compose/index.ts index d2314c5b646bf7..697140b7d883ee 100644 --- a/lib/manager/docker-compose/index.ts +++ b/lib/manager/docker-compose/index.ts @@ -1,5 +1,5 @@ import { ProgrammingLanguage } from '../../constants'; -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import { extractPackageFile } from './extract'; const language = ProgrammingLanguage.Docker; @@ -10,4 +10,4 @@ export const defaultConfig = { fileMatch: ['(^|/)docker-compose[^/]*\\.ya?ml$'], }; -export const supportedDatasources = [datasourceDocker.id]; +export const supportedDatasources = [DockerDatasource.id]; diff --git a/lib/manager/dockerfile/extract.ts b/lib/manager/dockerfile/extract.ts index 1a7e973460850a..78bd5a2f124bbc 100644 --- a/lib/manager/dockerfile/extract.ts +++ b/lib/manager/dockerfile/extract.ts @@ -1,5 +1,5 @@ import is from '@sindresorhus/is'; -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import { logger } from '../../logger'; import { regEx } from '../../util/regex'; import * as ubuntuVersioning from '../../versioning/ubuntu'; @@ -119,7 +119,7 @@ export function getDep( dep.autoReplaceStringTemplate = '{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}'; } - dep.datasource = datasourceDocker.id; + dep.datasource = DockerDatasource.id; // Pretty up special prefixes if (dep.depName) { diff --git a/lib/manager/dockerfile/index.ts b/lib/manager/dockerfile/index.ts index 7ebec5c8773dc1..59f6c25ef446a1 100644 --- a/lib/manager/dockerfile/index.ts +++ b/lib/manager/dockerfile/index.ts @@ -1,5 +1,5 @@ import { ProgrammingLanguage } from '../../constants'; -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import { extractPackageFile } from './extract'; const language = ProgrammingLanguage.Docker; @@ -10,4 +10,4 @@ export const defaultConfig = { fileMatch: ['(^|/|\\.)Dockerfile$', '(^|/)Dockerfile\\.[^/]*$'], }; -export const supportedDatasources = [datasourceDocker.id]; +export const supportedDatasources = [DockerDatasource.id]; diff --git a/lib/manager/droneci/index.ts b/lib/manager/droneci/index.ts index a46b91d3f3752c..4eb3e0545acec6 100644 --- a/lib/manager/droneci/index.ts +++ b/lib/manager/droneci/index.ts @@ -1,5 +1,5 @@ import { ProgrammingLanguage } from '../../constants'; -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import { extractPackageFile } from './extract'; const language = ProgrammingLanguage.Docker; @@ -10,4 +10,4 @@ export const defaultConfig = { fileMatch: ['(^|/).drone.yml$'], }; -export const supportedDatasources = [datasourceDocker.id]; +export const supportedDatasources = [DockerDatasource.id]; diff --git a/lib/manager/gitlabci/index.ts b/lib/manager/gitlabci/index.ts index 802c5072bb62f4..10d679fdb39f9e 100644 --- a/lib/manager/gitlabci/index.ts +++ b/lib/manager/gitlabci/index.ts @@ -1,5 +1,5 @@ import { ProgrammingLanguage } from '../../constants'; -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import { extractAllPackageFiles, extractPackageFile } from './extract'; const language = ProgrammingLanguage.Docker; @@ -10,4 +10,4 @@ export const defaultConfig = { fileMatch: ['\\.gitlab-ci\\.yml$'], }; -export const supportedDatasources = [datasourceDocker.id]; +export const supportedDatasources = [DockerDatasource.id]; diff --git a/lib/manager/helm-values/index.ts b/lib/manager/helm-values/index.ts index 2e267255000042..f941ce869432d7 100644 --- a/lib/manager/helm-values/index.ts +++ b/lib/manager/helm-values/index.ts @@ -1,4 +1,4 @@ -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; export { extractPackageFile } from './extract'; export const defaultConfig = { @@ -7,4 +7,4 @@ export const defaultConfig = { pinDigests: false, }; -export const supportedDatasources = [datasourceDocker.id]; +export const supportedDatasources = [DockerDatasource.id]; diff --git a/lib/manager/helmv3/artifacts.ts b/lib/manager/helmv3/artifacts.ts index fe52ce7ff06263..f2aa750463897d 100644 --- a/lib/manager/helmv3/artifacts.ts +++ b/lib/manager/helmv3/artifacts.ts @@ -2,7 +2,7 @@ import yaml from 'js-yaml'; import { quote } from 'shlex'; import upath from 'upath'; import { TEMPORARY_ERROR } from '../../constants/error-messages'; -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import { logger } from '../../logger'; import { exec } from '../../util/exec'; import type { ExecOptions, ToolConstraint } from '../../util/exec/types'; @@ -44,7 +44,7 @@ async function helmCommands( repository: value.repository.replace('oci://', ''), hostRule: hostRules.find({ url: value.repository.replace('oci://', 'https://'), //TODO we need to replace this, as oci:// will not be accepted as protocol - hostType: datasourceDocker.id, + hostType: DockerDatasource.id, }), }; }); diff --git a/lib/manager/helmv3/extract.spec.ts b/lib/manager/helmv3/extract.spec.ts index 1261309886300f..0c385afa438afc 100644 --- a/lib/manager/helmv3/extract.spec.ts +++ b/lib/manager/helmv3/extract.spec.ts @@ -1,5 +1,5 @@ import { fs } from '../../../test/util'; -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import { extractPackageFile } from './extract'; jest.mock('../../util/fs'); @@ -97,7 +97,7 @@ describe('manager/helmv3/extract', () => { deps: [ { depName: 'library', - datasource: datasourceDocker.id, + datasource: DockerDatasource.id, currentValue: '0.1.0', }, { depName: 'postgresql', currentValue: '0.8.1' }, diff --git a/lib/manager/helmv3/index.ts b/lib/manager/helmv3/index.ts index de5feaa3731a6e..d0ef5e718dfe53 100644 --- a/lib/manager/helmv3/index.ts +++ b/lib/manager/helmv3/index.ts @@ -1,4 +1,4 @@ -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import { HelmDatasource } from '../../datasource/helm'; export { updateArtifacts } from './artifacts'; export { extractPackageFile } from './extract'; @@ -14,4 +14,4 @@ export const defaultConfig = { fileMatch: ['(^|/)Chart.yaml$'], }; -export const supportedDatasources = [datasourceDocker.id, HelmDatasource.id]; +export const supportedDatasources = [DockerDatasource.id, HelmDatasource.id]; diff --git a/lib/manager/helmv3/utils.ts b/lib/manager/helmv3/utils.ts index 0e520c44808f71..03dfe95d1233ce 100644 --- a/lib/manager/helmv3/utils.ts +++ b/lib/manager/helmv3/utils.ts @@ -1,4 +1,4 @@ -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import { logger } from '../../logger'; import type { PackageDependency } from '../types'; import type { ChartDefinition, Repository } from './types'; @@ -13,7 +13,7 @@ export function parseRepository( const url = new URL(repositoryURL); switch (url.protocol) { case 'oci:': - res.datasource = datasourceDocker.id; + res.datasource = DockerDatasource.id; res.lookupName = `${repositoryURL.replace('oci://', '')}/${depName}`; break; case 'file:': diff --git a/lib/manager/kubernetes/index.ts b/lib/manager/kubernetes/index.ts index 053562ef183ae2..df3a060c61c98f 100644 --- a/lib/manager/kubernetes/index.ts +++ b/lib/manager/kubernetes/index.ts @@ -1,5 +1,5 @@ import { ProgrammingLanguage } from '../../constants'; -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; export { extractPackageFile } from './extract'; @@ -9,4 +9,4 @@ export const defaultConfig = { fileMatch: [], }; -export const supportedDatasources = [datasourceDocker.id]; +export const supportedDatasources = [DockerDatasource.id]; diff --git a/lib/manager/kustomize/extract.spec.ts b/lib/manager/kustomize/extract.spec.ts index a0120ad4f7aa83..9ea52829f241ba 100644 --- a/lib/manager/kustomize/extract.spec.ts +++ b/lib/manager/kustomize/extract.spec.ts @@ -1,5 +1,5 @@ import { loadFixture } from '../../../test/util'; -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import { GitTagsDatasource } from '../../datasource/git-tags'; import * as datasourceGitHubTags from '../../datasource/github-tags'; import { HelmDatasource } from '../../datasource/helm'; @@ -158,7 +158,7 @@ describe('manager/kustomize/extract', () => { const sample = { currentDigest: undefined, currentValue: 'v1.0.0', - datasource: datasourceDocker.id, + datasource: DockerDatasource.id, replaceString: 'v1.0.0', depName: 'node', }; @@ -172,7 +172,7 @@ describe('manager/kustomize/extract', () => { const sample = { currentDigest: undefined, currentValue: 'v1.0.0', - datasource: datasourceDocker.id, + datasource: DockerDatasource.id, replaceString: 'v1.0.0', depName: 'test/node', }; @@ -186,7 +186,7 @@ describe('manager/kustomize/extract', () => { const sample = { currentDigest: undefined, currentValue: 'v1.0.0', - datasource: datasourceDocker.id, + datasource: DockerDatasource.id, replaceString: 'v1.0.0', depName: 'quay.io/repo/image', }; @@ -200,7 +200,7 @@ describe('manager/kustomize/extract', () => { const sample = { currentDigest: undefined, currentValue: 'v1.0.0', - datasource: datasourceDocker.id, + datasource: DockerDatasource.id, replaceString: 'v1.0.0', depName: 'localhost:5000/repo/image', }; @@ -215,7 +215,7 @@ describe('manager/kustomize/extract', () => { currentDigest: undefined, currentValue: 'v1.0.0', replaceString: 'v1.0.0', - datasource: datasourceDocker.id, + datasource: DockerDatasource.id, depName: 'localhost:5000/repo/image/service', }; const pkg = extractImage({ diff --git a/lib/manager/kustomize/extract.ts b/lib/manager/kustomize/extract.ts index 3b58a1bf6b4107..cd0389b5155102 100644 --- a/lib/manager/kustomize/extract.ts +++ b/lib/manager/kustomize/extract.ts @@ -1,6 +1,6 @@ import is from '@sindresorhus/is'; import { load } from 'js-yaml'; -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import { GitTagsDatasource } from '../../datasource/git-tags'; import * as datasourceGitHubTags from '../../datasource/github-tags'; import { HelmDatasource } from '../../datasource/helm'; @@ -70,7 +70,7 @@ export function extractImage(image: Image): PackageDependency | null { } return { - datasource: datasourceDocker.id, + datasource: DockerDatasource.id, depName, currentValue: nameDep.currentValue, currentDigest: digest, @@ -90,7 +90,7 @@ export function extractImage(image: Image): PackageDependency | null { const dep = splitImageParts(`${depName}:${newTag}`); return { ...dep, - datasource: datasourceDocker.id, + datasource: DockerDatasource.id, replaceString: newTag, }; } @@ -98,7 +98,7 @@ export function extractImage(image: Image): PackageDependency | null { if (image.newName) { return { ...nameDep, - datasource: datasourceDocker.id, + datasource: DockerDatasource.id, replaceString: image.newName, }; } diff --git a/lib/manager/kustomize/index.ts b/lib/manager/kustomize/index.ts index c3ac88cd6e180c..3d7b6a048399cd 100644 --- a/lib/manager/kustomize/index.ts +++ b/lib/manager/kustomize/index.ts @@ -1,4 +1,4 @@ -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import { GitTagsDatasource } from '../../datasource/git-tags'; import * as datasourceGitHubTags from '../../datasource/github-tags'; import { HelmDatasource } from '../../datasource/helm'; @@ -10,7 +10,7 @@ export const defaultConfig = { }; export const supportedDatasources = [ - datasourceDocker.id, + DockerDatasource.id, GitTagsDatasource.id, datasourceGitHubTags.id, HelmDatasource.id, diff --git a/lib/manager/pyenv/extract.ts b/lib/manager/pyenv/extract.ts index 8fee5607c1ff01..a402da093160f4 100644 --- a/lib/manager/pyenv/extract.ts +++ b/lib/manager/pyenv/extract.ts @@ -1,11 +1,11 @@ -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import type { PackageDependency, PackageFile } from '../types'; export function extractPackageFile(content: string): PackageFile { const dep: PackageDependency = { depName: 'python', currentValue: content.trim(), - datasource: datasourceDocker.id, + datasource: DockerDatasource.id, }; return { deps: [dep] }; } diff --git a/lib/manager/pyenv/index.ts b/lib/manager/pyenv/index.ts index 3be9c1f5836ee6..f4ddc5b275c442 100644 --- a/lib/manager/pyenv/index.ts +++ b/lib/manager/pyenv/index.ts @@ -1,12 +1,12 @@ import { ProgrammingLanguage } from '../../constants'; -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import * as dockerVersioning from '../../versioning/docker'; export { extractPackageFile } from './extract'; export const language = ProgrammingLanguage.Python; -export const supportedDatasources = [datasourceDocker.id]; +export const supportedDatasources = [DockerDatasource.id]; export const defaultConfig = { fileMatch: ['(^|/).python-version$'], diff --git a/lib/util/package-rules.spec.ts b/lib/util/package-rules.spec.ts index c39a6e7cae1810..518627bd27302a 100644 --- a/lib/util/package-rules.spec.ts +++ b/lib/util/package-rules.spec.ts @@ -1,7 +1,7 @@ import type { PackageRuleInputConfig, UpdateType } from '../config/types'; import { ProgrammingLanguage } from '../constants'; -import * as datasourceDocker from '../datasource/docker'; +import { DockerDatasource } from '../datasource/docker'; import { OrbDatasource } from '../datasource/orb'; import { applyPackageRules } from './package-rules'; @@ -319,7 +319,7 @@ describe('util/package-rules', () => { const config: TestConfig = { packageRules: [ { - matchDatasources: [OrbDatasource.id, datasourceDocker.id], + matchDatasources: [OrbDatasource.id, DockerDatasource.id], x: 1, }, ], diff --git a/lib/workers/global/config/parse/cli.spec.ts b/lib/workers/global/config/parse/cli.spec.ts index 41114a2980defa..806f547ed95265 100644 --- a/lib/workers/global/config/parse/cli.spec.ts +++ b/lib/workers/global/config/parse/cli.spec.ts @@ -1,4 +1,4 @@ -import * as datasourceDocker from '../../../../datasource/docker'; +import { DockerDatasource } from '../../../../datasource/docker'; import getArgv from './__fixtures__/argv'; import * as cli from './cli'; import type { ParseConfigOptions } from './types'; @@ -78,13 +78,13 @@ describe('workers/global/config/parse/cli', () => { }); it('parses json lists correctly', () => { argv.push( - `--host-rules=[{"matchHost":"docker.io","hostType":"${datasourceDocker.id}","username":"user","password":"password"}]` + `--host-rules=[{"matchHost":"docker.io","hostType":"${DockerDatasource.id}","username":"user","password":"password"}]` ); expect(cli.getConfig(argv)).toEqual({ hostRules: [ { matchHost: 'docker.io', - hostType: datasourceDocker.id, + hostType: DockerDatasource.id, username: 'user', password: 'password', }, diff --git a/lib/workers/global/index.spec.ts b/lib/workers/global/index.spec.ts index b1d7df99b4bd5a..0bc8db30fe9f61 100644 --- a/lib/workers/global/index.spec.ts +++ b/lib/workers/global/index.spec.ts @@ -4,7 +4,7 @@ import { fs, logger, mocked } from '../../../test/util'; import * as _presets from '../../config/presets'; import { PlatformId } from '../../constants'; import { CONFIG_PRESETS_INVALID } from '../../constants/error-messages'; -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import * as _platform from '../../platform'; import * as _repositoryWorker from '../repository'; import * as _configParser from './config/parse'; @@ -81,7 +81,7 @@ describe('workers/global/index', () => { repositories: ['a', 'b'], hostRules: [ { - hostType: datasourceDocker.id, + hostType: DockerDatasource.id, username: 'some-user', password: 'some-password', }, @@ -100,7 +100,7 @@ describe('workers/global/index', () => { repositories: ['a', 'b'], hostRules: [ { - hostType: datasourceDocker.id, + hostType: DockerDatasource.id, username: 'some-user', password: 'some-password', }, diff --git a/lib/workers/repository/process/lookup/index.spec.ts b/lib/workers/repository/process/lookup/index.spec.ts index 1afcb616b05ee3..c2c894f8cfacef 100644 --- a/lib/workers/repository/process/lookup/index.spec.ts +++ b/lib/workers/repository/process/lookup/index.spec.ts @@ -6,8 +6,7 @@ import { partial, } from '../../../../../test/util'; import { CONFIG_VALIDATION } from '../../../../constants/error-messages'; -import * as datasourceDocker from '../../../../datasource/docker'; -import { id as datasourceDockerId } from '../../../../datasource/docker'; +import { DockerDatasource } from '../../../../datasource/docker'; import { GitRefsDatasource } from '../../../../datasource/git-refs'; import { GitDatasource } from '../../../../datasource/git-refs/base'; import * as datasourceGithubReleases from '../../../../datasource/github-releases'; @@ -39,8 +38,7 @@ const typescriptJson = loadJsonFixture('typescript.json', fixtureRoot); const vueJson = loadJsonFixture('vue.json', fixtureRoot); const webpackJson = loadJsonFixture('webpack.json', fixtureRoot); -const docker = mocked(datasourceDocker) as any; -docker.defaultRegistryUrls = ['https://index.docker.io']; +const docker = mocked(DockerDatasource.prototype); const githubReleases = mocked(datasourceGithubReleases); Object.assign(githubReleases, { defaultRegistryUrls: ['https://github.com'] }); @@ -1302,20 +1300,20 @@ describe('workers/repository/process/lookup/index', () => { it('skips unsupported values', async () => { config.currentValue = 'alpine'; config.depName = 'node'; - config.datasource = datasourceDockerId; + config.datasource = DockerDatasource.id; const res = await lookup.lookupUpdates(config); expect(res).toMatchSnapshot({ skipReason: 'invalid-value' }); }); it('skips undefined values', async () => { config.depName = 'node'; - config.datasource = datasourceDockerId; + config.datasource = DockerDatasource.id; const res = await lookup.lookupUpdates(config); expect(res).toMatchSnapshot({ skipReason: 'invalid-value' }); }); it('handles digest pin', async () => { config.currentValue = '8.0.0'; config.depName = 'node'; - config.datasource = datasourceDockerId; + config.datasource = DockerDatasource.id; config.pinDigests = true; docker.getReleases.mockResolvedValueOnce({ releases: [ @@ -1351,7 +1349,7 @@ describe('workers/repository/process/lookup/index', () => { config.currentValue = '8.1.0'; config.depName = 'node'; config.versioning = dockerVersioningId; - config.datasource = datasourceDockerId; + config.datasource = DockerDatasource.id; docker.getReleases.mockResolvedValueOnce({ releases: [ { version: '8.1.0' }, @@ -1374,7 +1372,7 @@ describe('workers/repository/process/lookup/index', () => { config.currentValue = '8.1'; config.depName = 'node'; config.versioning = dockerVersioningId; - config.datasource = datasourceDockerId; + config.datasource = DockerDatasource.id; docker.getReleases.mockResolvedValueOnce({ releases: [ { version: '8.1.0' }, @@ -1400,7 +1398,7 @@ describe('workers/repository/process/lookup/index', () => { config.currentValue = '8'; config.depName = 'node'; config.versioning = dockerVersioningId; - config.datasource = datasourceDockerId; + config.datasource = DockerDatasource.id; docker.getReleases.mockResolvedValueOnce({ releases: [ { version: '8.1.0' }, @@ -1422,7 +1420,7 @@ describe('workers/repository/process/lookup/index', () => { it('handles digest pin for up to date version', async () => { config.currentValue = '8.1.0'; config.depName = 'node'; - config.datasource = datasourceDockerId; + config.datasource = DockerDatasource.id; config.pinDigests = true; docker.getReleases.mockResolvedValueOnce({ releases: [ @@ -1449,7 +1447,7 @@ describe('workers/repository/process/lookup/index', () => { it('handles digest pin for non-version', async () => { config.currentValue = 'alpine'; config.depName = 'node'; - config.datasource = datasourceDockerId; + config.datasource = DockerDatasource.id; config.pinDigests = true; docker.getReleases.mockResolvedValueOnce({ releases: [ @@ -1479,7 +1477,7 @@ describe('workers/repository/process/lookup/index', () => { it('handles digest lookup failure', async () => { config.currentValue = 'alpine'; config.depName = 'node'; - config.datasource = datasourceDockerId; + config.datasource = DockerDatasource.id; config.pinDigests = true; docker.getReleases.mockResolvedValueOnce({ releases: [ @@ -1501,7 +1499,7 @@ describe('workers/repository/process/lookup/index', () => { it('handles digest update', async () => { config.currentValue = '8.0.0'; config.depName = 'node'; - config.datasource = datasourceDockerId; + config.datasource = DockerDatasource.id; config.currentDigest = 'sha256:zzzzzzzzzzzzzzz'; config.pinDigests = true; docker.getReleases.mockResolvedValueOnce({ @@ -1535,7 +1533,7 @@ describe('workers/repository/process/lookup/index', () => { it('handles digest update for non-version', async () => { config.currentValue = 'alpine'; config.depName = 'node'; - config.datasource = datasourceDockerId; + config.datasource = DockerDatasource.id; config.currentDigest = 'sha256:zzzzzzzzzzzzzzz'; config.pinDigests = true; docker.getReleases.mockResolvedValueOnce({ From dc1da473102f4134cf94f302512747ecdcba950f Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Sat, 12 Feb 2022 18:52:12 +0300 Subject: [PATCH 2/2] refactor(datasource/docker): Convert to class --- lib/datasource/api.ts | 4 +- lib/datasource/docker/index.spec.ts | 60 +- lib/datasource/docker/index.ts | 1019 +++++++++-------- lib/manager/ansible/index.ts | 4 +- lib/manager/bazel/extract.ts | 4 +- lib/manager/bazel/index.ts | 4 +- lib/manager/bitbucket-pipelines/index.ts | 4 +- lib/manager/circleci/index.ts | 4 +- lib/manager/cloudbuild/index.ts | 4 +- lib/manager/docker-compose/index.ts | 4 +- lib/manager/dockerfile/extract.ts | 4 +- lib/manager/dockerfile/index.ts | 4 +- lib/manager/droneci/index.ts | 4 +- lib/manager/gitlabci/index.ts | 4 +- lib/manager/helm-values/index.ts | 4 +- lib/manager/helmv3/artifacts.ts | 4 +- lib/manager/helmv3/extract.spec.ts | 4 +- lib/manager/helmv3/index.ts | 4 +- lib/manager/helmv3/utils.ts | 4 +- lib/manager/kubernetes/index.ts | 4 +- lib/manager/kustomize/extract.spec.ts | 12 +- lib/manager/kustomize/extract.ts | 8 +- lib/manager/kustomize/index.ts | 4 +- lib/manager/pyenv/extract.ts | 4 +- lib/manager/pyenv/index.ts | 4 +- lib/util/package-rules.spec.ts | 4 +- lib/workers/global/config/parse/cli.spec.ts | 6 +- lib/workers/global/index.spec.ts | 6 +- .../repository/process/lookup/index.spec.ts | 28 +- 29 files changed, 629 insertions(+), 598 deletions(-) diff --git a/lib/datasource/api.ts b/lib/datasource/api.ts index 11d8aaae45e798..7407377e6e49e1 100644 --- a/lib/datasource/api.ts +++ b/lib/datasource/api.ts @@ -7,7 +7,7 @@ import { ClojureDatasource } from './clojure'; import { ConanDatasource } from './conan'; import { CrateDatasource } from './crate'; import { DartDatasource } from './dart'; -import * as docker from './docker'; +import { DockerDatasource } from './docker'; import { GalaxyDatasource } from './galaxy'; import { GalaxyCollectionDatasource } from './galaxy-collection'; import { GitRefsDatasource } from './git-refs'; @@ -52,7 +52,7 @@ api.set(ClojureDatasource.id, new ClojureDatasource()); api.set(ConanDatasource.id, new ConanDatasource()); api.set(CrateDatasource.id, new CrateDatasource()); api.set(DartDatasource.id, new DartDatasource()); -api.set('docker', docker); +api.set(DockerDatasource.id, new DockerDatasource()); api.set(GalaxyDatasource.id, new GalaxyDatasource()); api.set(GalaxyCollectionDatasource.id, new GalaxyCollectionDatasource()); api.set(GitRefsDatasource.id, new GitRefsDatasource()); diff --git a/lib/datasource/docker/index.spec.ts b/lib/datasource/docker/index.spec.ts index c5825477cef933..2ffc289e4449ca 100644 --- a/lib/datasource/docker/index.spec.ts +++ b/lib/datasource/docker/index.spec.ts @@ -4,11 +4,14 @@ import * as httpMock from '../../../test/http-mock'; import { mocked, partial } from '../../../test/util'; import { EXTERNAL_HOST_ERROR } from '../../constants/error-messages'; import * as _hostRules from '../../util/host-rules'; +import { Http } from '../../util/http'; import { MediaType } from './types'; -import { getAuthHeaders, getRegistryRepository, id } from '.'; +import { DockerDatasource, getAuthHeaders, getRegistryRepository } from '.'; const hostRules = mocked(_hostRules); +const http = new Http(DockerDatasource.id); + jest.mock('@aws-sdk/client-ecr'); jest.mock('../../util/host-rules'); @@ -121,6 +124,7 @@ describe('datasource/docker/index', () => { }); const headers = await getAuthHeaders( + http, 'https://my.local.registry', 'https://my.local.registry/prefix' ); @@ -138,6 +142,7 @@ Object { }); const headers = await getAuthHeaders( + http, 'https://my.local.registry', 'https://my.local.registry/prefix' ); @@ -158,6 +163,7 @@ Object { .reply(401, '', {}); const headers = await getAuthHeaders( + http, 'https://my.local.registry', 'https://my.local.registry/prefix' ); @@ -534,7 +540,7 @@ Object { .get('/library/node/tags/list?n=10000') .reply(403); const res = await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'node', registryUrls: ['https://docker.io'], }); @@ -564,7 +570,7 @@ Object { .get('/user/9287/repos?page=3&per_page=100') .reply(200, { tags: ['latest'] }, {}); const config = { - datasource: id, + datasource: DockerDatasource.id, depName: 'node', registryUrls: ['https://registry.company.com'], }; @@ -585,7 +591,7 @@ Object { .get('/node/manifests/1.0.0') .reply(200, '', {}); const res = await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'registry.company.com/node', }); expect(res.releases).toHaveLength(1); @@ -608,7 +614,7 @@ Object { .get('/v2/bitnami/redis/manifests/5.0.12') .reply(200, '', {}); const config = { - datasource: id, + datasource: DockerDatasource.id, depName: 'bitnami/redis', registryUrls: ['https://quay.io'], }; @@ -624,7 +630,7 @@ Object { ) .reply(500); const config = { - datasource: id, + datasource: DockerDatasource.id, depName: 'bitnami/redis', registryUrls: ['https://quay.io'], }; @@ -646,7 +652,7 @@ Object { .reply(200); expect( await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: '123456789.dkr.ecr.us-east-1.amazonaws.com/node', }) ).toEqual({ @@ -700,7 +706,7 @@ Object { }); expect( await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'ecr-proxy.company.com/node', }) ).toEqual({ @@ -735,7 +741,7 @@ Object { }); expect( await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'ecr-proxy.company.com/node', }) ).toBeNull(); @@ -766,7 +772,7 @@ Object { ); expect( await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'ecr-proxy.company.com/node', }) ).toBeNull(); @@ -789,7 +795,7 @@ Object { }); expect( await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'ecr-proxy.company.com/node', }) ).toBeNull(); @@ -818,7 +824,7 @@ Object { ); expect( await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'ecr-proxy.company.com/node', }) ).toBeNull(); @@ -839,7 +845,7 @@ Object { ); expect( await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'ecr-proxy.company.com/node', }) ).toBeNull(); @@ -862,7 +868,7 @@ Object { ); expect( await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'ecr-proxy.company.com/node', }) ).toBeNull(); @@ -889,7 +895,7 @@ Object { ); expect( await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'ecr-proxy.company.com/node', }) ).toBeNull(); @@ -917,7 +923,7 @@ Object { ); expect( await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'ecr-proxy.company.com/node', }) ).toBeNull(); @@ -946,7 +952,7 @@ Object { ) .reply(200, { token: 'test' }); const res = await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'node', }); expect(res.releases).toHaveLength(1); @@ -974,7 +980,7 @@ Object { ) .reply(200, { token: 'test' }); const res = await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'docker.io/node', }); expect(res.releases).toHaveLength(1); @@ -1000,7 +1006,7 @@ Object { .get('/kubernetes-dashboard-amd64/manifests/1.0.0') .reply(200); const res = await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'k8s.gcr.io/kubernetes-dashboard-amd64', }); expect(res.releases).toHaveLength(1); @@ -1014,7 +1020,7 @@ Object { .get('/my/node/tags/list?n=10000') .replyWithError('error'); const res = await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'my/node', }); expect(res).toBeNull(); @@ -1039,7 +1045,7 @@ Object { .get('/token?service=registry.docker.io&scope=repository:my/node:pull') .reply(200, { token: 'some-token ' }); const res = await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'my/node', registryUrls: ['https://index.docker.io/'], }); @@ -1052,7 +1058,7 @@ Object { 'www-authenticate': 'Basic realm="My Private Docker Registry Server"', }); const res = await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'node', }); expect(res).toBeNull(); @@ -1092,7 +1098,7 @@ Object { }, }); const res = await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'registry.company.com/node', }); expect(res).toMatchSnapshot(); @@ -1128,7 +1134,7 @@ Object { }, }); const res = await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'registry.company.com/node', }); expect(res).toMatchSnapshot(); @@ -1148,7 +1154,7 @@ Object { mediaType: MediaType.manifestV1, }); const res = await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'registry.company.com/node', }); expect(res).toMatchSnapshot(); @@ -1165,7 +1171,7 @@ Object { .get('/node/manifests/latest') .reply(200, {}); const res = await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'registry.company.com/node', }); expect(res).toMatchSnapshot(); @@ -1208,7 +1214,7 @@ Object { config: {}, }); const res = await getPkgReleases({ - datasource: id, + datasource: DockerDatasource.id, depName: 'registry.company.com/node', }); expect(res).toMatchSnapshot(); diff --git a/lib/datasource/docker/index.ts b/lib/datasource/docker/index.ts index cce8d732ab3843..2c2ea7961938b2 100644 --- a/lib/datasource/docker/index.ts +++ b/lib/datasource/docker/index.ts @@ -10,7 +10,7 @@ import type { HostRule } from '../../types'; import { ExternalHostError } from '../../types/errors/external-host-error'; import * as packageCache from '../../util/cache/package'; import * as hostRules from '../../util/host-rules'; -import { Http, HttpOptions, HttpResponse } from '../../util/http'; +import type { Http, HttpOptions, HttpResponse } from '../../util/http'; import { HttpError } from '../../util/http/types'; import type { OutgoingHttpHeaders } from '../../util/http/types'; import { hasKey } from '../../util/object'; @@ -26,60 +26,22 @@ import { api as dockerVersioning, id as dockerVersioningId, } from '../../versioning/docker'; +import { Datasource } from '../datasource'; import type { GetReleasesConfig, ReleaseResult } from '../types'; import { sourceLabels } from './common'; import { Image, ImageList, MediaType, RegistryRepository } from './types'; -export const ecrRegex = regEx(/\d+\.dkr\.ecr\.([-a-z0-9]+)\.amazonaws\.com/); - -export const id = 'docker'; -export const http = new Http(id); - -const DOCKER_HUB = 'https://index.docker.io'; -export const defaultRegistryUrls = [DOCKER_HUB]; +export const DOCKER_HUB = 'https://index.docker.io'; -// TODO: add got typings when available (#9646) - -export const customRegistrySupport = true; -export const defaultVersioning = dockerVersioningId; -export const registryStrategy = 'first'; +export const ecrRegex = regEx(/\d+\.dkr\.ecr\.([-a-z0-9]+)\.amazonaws\.com/); function isDockerHost(host: string): boolean { const regex = regEx(/(?:^|\.)docker\.io$/); return regex.test(host); } -async function getECRAuthToken( - region: string, - opts: HostRule -): Promise { - const config: ECRClientConfig = { region }; - if (opts.username && opts.password) { - config.credentials = { - accessKeyId: opts.username, - secretAccessKey: opts.password, - ...(opts.token && { sessionToken: opts.token }), - }; - } - - const ecr = new ECR(config); - try { - const data = await ecr.getAuthorizationToken({}); - const authorizationToken = data?.authorizationData?.[0]?.authorizationToken; - if (authorizationToken) { - return authorizationToken; - } - logger.warn( - 'Could not extract authorizationToken from ECR getAuthorizationToken response' - ); - } catch (err) { - logger.trace({ err }, 'err'); - logger.debug('ECR getAuthorizationToken error'); - } - return null; -} - export async function getAuthHeaders( + http: Http, registryHost: string, dockerRepository: string ): Promise { @@ -110,7 +72,7 @@ export async function getAuthHeaders( ); const opts: HostRule & HttpOptions = hostRules.find({ - hostType: id, + hostType: DockerDatasource.id, url: apiCheckUrl, }); if (ecrRegex.test(registryHost)) { @@ -227,6 +189,36 @@ export async function getAuthHeaders( } } +async function getECRAuthToken( + region: string, + opts: HostRule +): Promise { + const config: ECRClientConfig = { region }; + if (opts.username && opts.password) { + config.credentials = { + accessKeyId: opts.username, + secretAccessKey: opts.password, + ...(opts.token && { sessionToken: opts.token }), + }; + } + + const ecr = new ECR(config); + try { + const data = await ecr.getAuthorizationToken({}); + const authorizationToken = data?.authorizationData?.[0]?.authorizationToken; + if (authorizationToken) { + return authorizationToken; + } + logger.warn( + 'Could not extract authorizationToken from ECR getAuthorizationToken response' + ); + } catch (err) { + logger.trace({ err }, 'err'); + logger.debug('ECR getAuthorizationToken error'); + } + return null; +} + export function getRegistryRepository( lookupName: string, registryUrl: string @@ -270,7 +262,10 @@ export function getRegistryRepository( if (!regEx(/^https?:\/\//).exec(registryHost)) { registryHost = `https://${registryHost}`; } - const opts = hostRules.find({ hostType: id, url: registryHost }); + const opts = hostRules.find({ + hostType: DockerDatasource.id, + url: registryHost, + }); if (opts?.insecureRegistry) { registryHost = registryHost.replace('https', 'http'); } @@ -293,245 +288,6 @@ export function extractDigestFromResponseBody( return digestFromManifestStr(manifestResponse.body); } -// TODO: debug why quay throws errors (#9612) -export async function getManifestResponse( - registryHost: string, - dockerRepository: string, - tag: string, - mode: 'head' | 'get' = 'get' -): Promise { - logger.debug( - `getManifestResponse(${registryHost}, ${dockerRepository}, ${tag})` - ); - try { - const headers = await getAuthHeaders(registryHost, dockerRepository); - if (!headers) { - logger.debug('No docker auth found - returning'); - return null; - } - headers.accept = - 'application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json'; - const url = `${registryHost}/v2/${dockerRepository}/manifests/${tag}`; - const manifestResponse = await http[mode](url, { - headers, - noAuth: true, - }); - return manifestResponse; - } catch (err) /* istanbul ignore next */ { - if (err instanceof ExternalHostError) { - throw err; - } - if (err.statusCode === 401) { - logger.debug( - { registryHost, dockerRepository }, - 'Unauthorized docker lookup' - ); - logger.debug({ err }); - return null; - } - if (err.statusCode === 404) { - logger.debug( - { - err, - registryHost, - dockerRepository, - tag, - }, - 'Docker Manifest is unknown' - ); - return null; - } - if (err.statusCode === 429 && isDockerHost(registryHost)) { - throw new ExternalHostError(err); - } - if (err.statusCode >= 500 && err.statusCode < 600) { - throw new ExternalHostError(err); - } - if (err.code === 'ETIMEDOUT') { - logger.debug( - { registryHost }, - 'Timeout when attempting to connect to docker registry' - ); - logger.debug({ err }); - return null; - } - logger.debug( - { - err, - registryHost, - dockerRepository, - tag, - }, - 'Unknown Error looking up docker manifest' - ); - return null; - } -} - -async function getConfigDigest( - registry: string, - dockerRepository: string, - tag: string -): Promise { - const manifestResponse = await getManifestResponse( - registry, - dockerRepository, - tag - ); - // If getting the manifest fails here, then abort - // This means that the latest tag doesn't have a manifest, which shouldn't - // be possible - // istanbul ignore if - if (!manifestResponse) { - return null; - } - const manifest = JSON.parse(manifestResponse.body) as ImageList | Image; - if (manifest.schemaVersion !== 2) { - logger.debug( - { registry, dockerRepository, tag }, - 'Manifest schema version is not 2' - ); - return null; - } - - if ( - manifest.mediaType === MediaType.manifestListV2 && - manifest.manifests.length - ) { - logger.trace( - { registry, dockerRepository, tag }, - 'Found manifest list, using first image' - ); - return getConfigDigest( - registry, - dockerRepository, - manifest.manifests[0].digest - ); - } - - if ( - manifest.mediaType === MediaType.manifestV2 && - is.string(manifest.config?.digest) - ) { - return manifest.config?.digest; - } - - logger.debug({ manifest }, 'Invalid manifest - returning'); - return null; -} - -/* - * docker.getLabels - * - * This function will: - * - Return the labels for the requested image - */ - -export async function getLabels( - registryHost: string, - dockerRepository: string, - tag: string -): Promise> { - logger.debug(`getLabels(${registryHost}, ${dockerRepository}, ${tag})`); - const cacheNamespace = 'datasource-docker-labels'; - const cacheKey = `${registryHost}:${dockerRepository}:${tag}`; - const cachedResult = await packageCache.get>( - cacheNamespace, - cacheKey - ); - // istanbul ignore if - if (cachedResult !== undefined) { - return cachedResult; - } - try { - let labels: Record = {}; - const configDigest = await getConfigDigest( - registryHost, - dockerRepository, - tag - ); - if (!configDigest) { - return {}; - } - - const headers = await getAuthHeaders(registryHost, dockerRepository); - // istanbul ignore if: Should never be happen - if (!headers) { - logger.debug('No docker auth found - returning'); - return {}; - } - const url = `${registryHost}/v2/${dockerRepository}/blobs/${configDigest}`; - const configResponse = await http.get(url, { - headers, - noAuth: true, - }); - labels = JSON.parse(configResponse.body).config.Labels; - - if (labels) { - logger.debug( - { - labels, - }, - 'found labels in manifest' - ); - } - const cacheMinutes = 60; - await packageCache.set(cacheNamespace, cacheKey, labels, cacheMinutes); - return labels; - } catch (err) /* istanbul ignore next: should be tested in future */ { - if (err instanceof ExternalHostError) { - throw err; - } - if (err.statusCode === 400 || err.statusCode === 401) { - logger.debug( - { registryHost, dockerRepository, err }, - 'Unauthorized docker lookup' - ); - } else if (err.statusCode === 404) { - logger.warn( - { - err, - registryHost, - dockerRepository, - tag, - }, - 'Config Manifest is unknown' - ); - } else if (err.statusCode === 429 && isDockerHost(registryHost)) { - logger.warn({ err }, 'docker registry failure: too many requests'); - } else if (err.statusCode >= 500 && err.statusCode < 600) { - logger.debug( - { - err, - registryHost, - dockerRepository, - tag, - }, - 'docker registry failure: internal error' - ); - } else if ( - err.code === 'ERR_TLS_CERT_ALTNAME_INVALID' || - err.code === 'ETIMEDOUT' - ) { - logger.debug( - { registryHost, err }, - 'Error connecting to docker registry' - ); - } else if (registryHost === 'https://quay.io') { - // istanbul ignore next - logger.debug( - 'Ignoring quay.io errors until they fully support v2 schema' - ); - } else { - logger.info( - { registryHost, dockerRepository, tag, err }, - 'Unknown error getting Docker labels' - ); - } - return {}; - } -} - export function isECRMaxResultsError(err: HttpError): boolean { return !!( err.response?.statusCode === 405 && @@ -543,31 +299,6 @@ export function isECRMaxResultsError(err: HttpError): boolean { ); } -export async function getTagsQuayRegistry( - registry: string, - repository: string -): Promise { - let tags: string[] = []; - const limit = 100; - - const pageUrl = (page: number): string => - `${registry}/api/v1/repository/${repository}/tag/?limit=${limit}&page=${page}&onlyActiveTags=true`; - - let page = 1; - let url = pageUrl(page); - do { - const res = await http.getJson<{ - tags: { name: string }[]; - has_additional: boolean; - }>(url, {}); - const pageTags = res.body.tags.map((tag) => tag.name); - tags = tags.concat(pageTags); - page += 1; - url = res.body.has_additional ? pageUrl(page) : null; - } while (url && page < 20); - return tags; -} - export const defaultConfig = { commitMessageTopic: '{{{depName}}} Docker tag', commitMessageExtra: @@ -595,149 +326,171 @@ export const defaultConfig = { }, }; -async function getDockerApiTags( - registryHost: string, - dockerRepository: string -): Promise { - let tags: string[] = []; - // AWS ECR limits the maximum number of results to 1000 - // See https://docs.aws.amazon.com/AmazonECR/latest/APIReference/API_DescribeRepositories.html#ECR-DescribeRepositories-request-maxResults - const limit = ecrRegex.test(registryHost) ? 1000 : 10000; - let url = `${registryHost}/${dockerRepository}/tags/list?n=${limit}`; - url = ensurePathPrefix(url, '/v2'); - const headers = await getAuthHeaders(registryHost, dockerRepository); - if (!headers) { - logger.debug('Failed to get authHeaders for getTags lookup'); - return null; +function findLatestStable(tags: string[]): string { + const versions = tags + .filter((v) => dockerVersioning.isValid(v) && dockerVersioning.isStable(v)) + .sort((a, b) => dockerVersioning.sortVersions(a, b)); + + return versions.pop() ?? tags.slice(-1).pop(); +} + +export class DockerDatasource extends Datasource { + static readonly id = 'docker'; + + override readonly defaultVersioning = dockerVersioningId; + + override readonly defaultRegistryUrls = [DOCKER_HUB]; + + constructor() { + super(DockerDatasource.id); } - let page = 1; - let foundMaxResultsError = false; - do { - let res: HttpResponse<{ tags: string[] }>; + + // TODO: debug why quay throws errors (#9612) + private async getManifestResponse( + registryHost: string, + dockerRepository: string, + tag: string, + mode: 'head' | 'get' = 'get' + ): Promise { + logger.debug( + `getManifestResponse(${registryHost}, ${dockerRepository}, ${tag})` + ); try { - res = await http.getJson<{ tags: string[] }>(url, { + const headers = await getAuthHeaders( + this.http, + registryHost, + dockerRepository + ); + if (!headers) { + logger.debug('No docker auth found - returning'); + return null; + } + headers.accept = + 'application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json'; + const url = `${registryHost}/v2/${dockerRepository}/manifests/${tag}`; + const manifestResponse = await this.http[mode](url, { headers, noAuth: true, }); - } catch (err) { - if ( - !foundMaxResultsError && - err instanceof HttpError && - isECRMaxResultsError(err) - ) { - const maxResults = 1000; - url = `${registryHost}/${dockerRepository}/tags/list?n=${maxResults}`; - url = ensurePathPrefix(url, '/v2'); - foundMaxResultsError = true; - continue; + return manifestResponse; + } catch (err) /* istanbul ignore next */ { + if (err instanceof ExternalHostError) { + throw err; } - throw err; + if (err.statusCode === 401) { + logger.debug( + { registryHost, dockerRepository }, + 'Unauthorized docker lookup' + ); + logger.debug({ err }); + return null; + } + if (err.statusCode === 404) { + logger.debug( + { + err, + registryHost, + dockerRepository, + tag, + }, + 'Docker Manifest is unknown' + ); + return null; + } + if (err.statusCode === 429 && isDockerHost(registryHost)) { + throw new ExternalHostError(err); + } + if (err.statusCode >= 500 && err.statusCode < 600) { + throw new ExternalHostError(err); + } + if (err.code === 'ETIMEDOUT') { + logger.debug( + { registryHost }, + 'Timeout when attempting to connect to docker registry' + ); + logger.debug({ err }); + return null; + } + logger.debug( + { + err, + registryHost, + dockerRepository, + tag, + }, + 'Unknown Error looking up docker manifest' + ); + return null; } - tags = tags.concat(res.body.tags); - const linkHeader = parseLinkHeader(res.headers.link); - url = linkHeader?.next ? URL.resolve(url, linkHeader.next.url) : null; - page += 1; - } while (url && page < 20); - return tags; -} + } -async function getTags( - registryHost: string, - dockerRepository: string -): Promise { - try { - const cacheNamespace = 'datasource-docker-tags'; - const cacheKey = `${registryHost}:${dockerRepository}`; - const cachedResult = await packageCache.get( - cacheNamespace, - cacheKey + private async getConfigDigest( + registry: string, + dockerRepository: string, + tag: string + ): Promise { + const manifestResponse = await this.getManifestResponse( + registry, + dockerRepository, + tag ); + // If getting the manifest fails here, then abort + // This means that the latest tag doesn't have a manifest, which shouldn't + // be possible // istanbul ignore if - if (cachedResult !== undefined) { - return cachedResult; - } - - const isQuay = regEx(/^https:\/\/quay\.io(?::[1-9][0-9]{0,4})?$/i).test( - registryHost - ); - let tags: string[] | null; - if (isQuay) { - tags = await getTagsQuayRegistry(registryHost, dockerRepository); - } else { - tags = await getDockerApiTags(registryHost, dockerRepository); - } - const cacheMinutes = 30; - await packageCache.set(cacheNamespace, cacheKey, tags, cacheMinutes); - return tags; - } catch (err) /* istanbul ignore next */ { - if (err instanceof ExternalHostError) { - throw err; + if (!manifestResponse) { + return null; } - if (err.statusCode === 404 && !dockerRepository.includes('/')) { + const manifest = JSON.parse(manifestResponse.body) as ImageList | Image; + if (manifest.schemaVersion !== 2) { logger.debug( - `Retrying Tags for ${registryHost}/${dockerRepository} using library/ prefix` - ); - return getTags(registryHost, 'library/' + dockerRepository); - } - // prettier-ignore - if (err.statusCode === 429 && isDockerHost(registryHost)) { - logger.warn( - { registryHost, dockerRepository, err }, - 'docker registry failure: too many requests' + { registry, dockerRepository, tag }, + 'Manifest schema version is not 2' ); - throw new ExternalHostError(err); + return null; } - // prettier-ignore - if (err.statusCode === 401 && isDockerHost(registryHost)) { - logger.warn( - { registryHost, dockerRepository, err }, - 'docker registry failure: unauthorized' + + if ( + manifest.mediaType === MediaType.manifestListV2 && + manifest.manifests.length + ) { + logger.trace( + { registry, dockerRepository, tag }, + 'Found manifest list, using first image' ); - throw new ExternalHostError(err); - } - if (err.statusCode >= 500 && err.statusCode < 600) { - logger.warn( - { registryHost, dockerRepository, err }, - 'docker registry failure: internal error' + return this.getConfigDigest( + registry, + dockerRepository, + manifest.manifests[0].digest ); - throw new ExternalHostError(err); } - throw err; - } -} -function findLatestStable(tags: string[]): string { - const versions = tags - .filter((v) => dockerVersioning.isValid(v) && dockerVersioning.isStable(v)) - .sort((a, b) => dockerVersioning.sortVersions(a, b)); + if ( + manifest.mediaType === MediaType.manifestV2 && + is.string(manifest.config?.digest) + ) { + return manifest.config?.digest; + } - return versions.pop() ?? tags.slice(-1).pop(); -} + logger.debug({ manifest }, 'Invalid manifest - returning'); + return null; + } -/** - * docker.getDigest - * - * The `newValue` supplied here should be a valid tag for the docker image. - * - * This function will: - * - Look up a sha256 digest for a tag on its registry - * - Return the digest as a string - */ -export async function getDigest( - { registryUrl, lookupName }: GetReleasesConfig, - newValue?: string -): Promise { - const { registryHost, dockerRepository } = getRegistryRepository( - lookupName, - registryUrl - ); - logger.debug(`getDigest(${registryHost}, ${dockerRepository}, ${newValue})`); - const newTag = newValue || 'latest'; - const cacheNamespace = 'datasource-docker-digest'; - const cacheKey = `${registryHost}:${dockerRepository}:${newTag}`; - let digest: string = null; - try { - const cachedResult = await packageCache.get( + /* + * docker.getLabels + * + * This function will: + * - Return the labels for the requested image + */ + private async getLabels( + registryHost: string, + dockerRepository: string, + tag: string + ): Promise> { + logger.debug(`getLabels(${registryHost}, ${dockerRepository}, ${tag})`); + const cacheNamespace = 'datasource-docker-labels'; + const cacheKey = `${registryHost}:${dockerRepository}:${tag}`; + const cachedResult = await packageCache.get>( cacheNamespace, cacheKey ); @@ -745,86 +498,360 @@ export async function getDigest( if (cachedResult !== undefined) { return cachedResult; } - let manifestResponse = await getManifestResponse( + try { + let labels: Record = {}; + const configDigest = await this.getConfigDigest( + registryHost, + dockerRepository, + tag + ); + if (!configDigest) { + return {}; + } + + const headers = await getAuthHeaders( + this.http, + registryHost, + dockerRepository + ); + // istanbul ignore if: Should never be happen + if (!headers) { + logger.debug('No docker auth found - returning'); + return {}; + } + const url = `${registryHost}/v2/${dockerRepository}/blobs/${configDigest}`; + const configResponse = await this.http.get(url, { + headers, + noAuth: true, + }); + labels = JSON.parse(configResponse.body).config.Labels; + + if (labels) { + logger.debug( + { + labels, + }, + 'found labels in manifest' + ); + } + const cacheMinutes = 60; + await packageCache.set(cacheNamespace, cacheKey, labels, cacheMinutes); + return labels; + } catch (err) /* istanbul ignore next: should be tested in future */ { + if (err instanceof ExternalHostError) { + throw err; + } + if (err.statusCode === 400 || err.statusCode === 401) { + logger.debug( + { registryHost, dockerRepository, err }, + 'Unauthorized docker lookup' + ); + } else if (err.statusCode === 404) { + logger.warn( + { + err, + registryHost, + dockerRepository, + tag, + }, + 'Config Manifest is unknown' + ); + } else if (err.statusCode === 429 && isDockerHost(registryHost)) { + logger.warn({ err }, 'docker registry failure: too many requests'); + } else if (err.statusCode >= 500 && err.statusCode < 600) { + logger.debug( + { + err, + registryHost, + dockerRepository, + tag, + }, + 'docker registry failure: internal error' + ); + } else if ( + err.code === 'ERR_TLS_CERT_ALTNAME_INVALID' || + err.code === 'ETIMEDOUT' + ) { + logger.debug( + { registryHost, err }, + 'Error connecting to docker registry' + ); + } else if (registryHost === 'https://quay.io') { + // istanbul ignore next + logger.debug( + 'Ignoring quay.io errors until they fully support v2 schema' + ); + } else { + logger.info( + { registryHost, dockerRepository, tag, err }, + 'Unknown error getting Docker labels' + ); + } + return {}; + } + } + + private async getTagsQuayRegistry( + registry: string, + repository: string + ): Promise { + let tags: string[] = []; + const limit = 100; + + const pageUrl = (page: number): string => + `${registry}/api/v1/repository/${repository}/tag/?limit=${limit}&page=${page}&onlyActiveTags=true`; + + let page = 1; + let url = pageUrl(page); + do { + const res = await this.http.getJson<{ + tags: { name: string }[]; + has_additional: boolean; + }>(url, {}); + const pageTags = res.body.tags.map((tag) => tag.name); + tags = tags.concat(pageTags); + page += 1; + url = res.body.has_additional ? pageUrl(page) : null; + } while (url && page < 20); + return tags; + } + + private async getDockerApiTags( + registryHost: string, + dockerRepository: string + ): Promise { + let tags: string[] = []; + // AWS ECR limits the maximum number of results to 1000 + // See https://docs.aws.amazon.com/AmazonECR/latest/APIReference/API_DescribeRepositories.html#ECR-DescribeRepositories-request-maxResults + const limit = ecrRegex.test(registryHost) ? 1000 : 10000; + let url = `${registryHost}/${dockerRepository}/tags/list?n=${limit}`; + url = ensurePathPrefix(url, '/v2'); + const headers = await getAuthHeaders( + this.http, registryHost, - dockerRepository, - newTag, - 'head' + dockerRepository ); - if (manifestResponse) { - if (hasKey('docker-content-digest', manifestResponse.headers)) { - digest = - (manifestResponse.headers['docker-content-digest'] as string) || null; + if (!headers) { + logger.debug('Failed to get authHeaders for getTags lookup'); + return null; + } + let page = 1; + let foundMaxResultsError = false; + do { + let res: HttpResponse<{ tags: string[] }>; + try { + res = await this.http.getJson<{ tags: string[] }>(url, { + headers, + noAuth: true, + }); + } catch (err) { + if ( + !foundMaxResultsError && + err instanceof HttpError && + isECRMaxResultsError(err) + ) { + const maxResults = 1000; + url = `${registryHost}/${dockerRepository}/tags/list?n=${maxResults}`; + url = ensurePathPrefix(url, '/v2'); + foundMaxResultsError = true; + continue; + } + throw err; + } + tags = tags.concat(res.body.tags); + const linkHeader = parseLinkHeader(res.headers.link); + url = linkHeader?.next ? URL.resolve(url, linkHeader.next.url) : null; + page += 1; + } while (url && page < 20); + return tags; + } + + private async getTags( + registryHost: string, + dockerRepository: string + ): Promise { + try { + const cacheNamespace = 'datasource-docker-tags'; + const cacheKey = `${registryHost}:${dockerRepository}`; + const cachedResult = await packageCache.get( + cacheNamespace, + cacheKey + ); + // istanbul ignore if + if (cachedResult !== undefined) { + return cachedResult; + } + + const isQuay = regEx(/^https:\/\/quay\.io(?::[1-9][0-9]{0,4})?$/i).test( + registryHost + ); + let tags: string[] | null; + if (isQuay) { + tags = await this.getTagsQuayRegistry(registryHost, dockerRepository); } else { + tags = await this.getDockerApiTags(registryHost, dockerRepository); + } + const cacheMinutes = 30; + await packageCache.set(cacheNamespace, cacheKey, tags, cacheMinutes); + return tags; + } catch (err) /* istanbul ignore next */ { + if (err instanceof ExternalHostError) { + throw err; + } + if (err.statusCode === 404 && !dockerRepository.includes('/')) { logger.debug( - { registryHost }, - 'Missing docker content digest header, pulling full manifest' + `Retrying Tags for ${registryHost}/${dockerRepository} using library/ prefix` ); - manifestResponse = await getManifestResponse( - registryHost, - dockerRepository, - newTag + return this.getTags(registryHost, 'library/' + dockerRepository); + } + // prettier-ignore + if (err.statusCode === 429 && isDockerHost(registryHost)) { + logger.warn( + { registryHost, dockerRepository, err }, + 'docker registry failure: too many requests' ); - digest = extractDigestFromResponseBody(manifestResponse); + throw new ExternalHostError(err); + } + // prettier-ignore + if (err.statusCode === 401 && isDockerHost(registryHost)) { + logger.warn( + { registryHost, dockerRepository, err }, + 'docker registry failure: unauthorized' + ); + throw new ExternalHostError(err); + } + if (err.statusCode >= 500 && err.statusCode < 600) { + logger.warn( + { registryHost, dockerRepository, err }, + 'docker registry failure: internal error' + ); + throw new ExternalHostError(err); } - logger.debug({ digest }, 'Got docker digest'); - } - } catch (err) /* istanbul ignore next */ { - if (err instanceof ExternalHostError) { throw err; } + } + + /** + * docker.getDigest + * + * The `newValue` supplied here should be a valid tag for the docker image. + * + * This function will: + * - Look up a sha256 digest for a tag on its registry + * - Return the digest as a string + */ + override async getDigest( + { registryUrl, lookupName }: GetReleasesConfig, + newValue?: string + ): Promise { + const { registryHost, dockerRepository } = getRegistryRepository( + lookupName, + registryUrl + ); logger.debug( - { - err, - lookupName, - newTag, - }, - 'Unknown Error looking up docker image digest' + `getDigest(${registryHost}, ${dockerRepository}, ${newValue})` ); + const newTag = newValue || 'latest'; + const cacheNamespace = 'datasource-docker-digest'; + const cacheKey = `${registryHost}:${dockerRepository}:${newTag}`; + let digest: string = null; + try { + const cachedResult = await packageCache.get( + cacheNamespace, + cacheKey + ); + // istanbul ignore if + if (cachedResult !== undefined) { + return cachedResult; + } + let manifestResponse = await this.getManifestResponse( + registryHost, + dockerRepository, + newTag, + 'head' + ); + if (manifestResponse) { + if (hasKey('docker-content-digest', manifestResponse.headers)) { + digest = + (manifestResponse.headers['docker-content-digest'] as string) || + null; + } else { + logger.debug( + { registryHost }, + 'Missing docker content digest header, pulling full manifest' + ); + manifestResponse = await this.getManifestResponse( + registryHost, + dockerRepository, + newTag + ); + digest = extractDigestFromResponseBody(manifestResponse); + } + logger.debug({ digest }, 'Got docker digest'); + } + } catch (err) /* istanbul ignore next */ { + if (err instanceof ExternalHostError) { + throw err; + } + logger.debug( + { + err, + lookupName, + newTag, + }, + 'Unknown Error looking up docker image digest' + ); + } + const cacheMinutes = 30; + await packageCache.set(cacheNamespace, cacheKey, digest, cacheMinutes); + return digest; } - const cacheMinutes = 30; - await packageCache.set(cacheNamespace, cacheKey, digest, cacheMinutes); - return digest; -} -/** - * docker.getReleases - * - * A docker image usually looks something like this: somehost.io/owner/repo:8.1.0-alpine - * In the above: - * - 'somehost.io' is the registry - * - 'owner/repo' is the package name - * - '8.1.0-alpine' is the tag - * - * This function will filter only tags that contain a semver version - */ -export async function getReleases({ - lookupName, - registryUrl, -}: GetReleasesConfig): Promise { - const { registryHost, dockerRepository } = getRegistryRepository( + /** + * docker.getReleases + * + * A docker image usually looks something like this: somehost.io/owner/repo:8.1.0-alpine + * In the above: + * - 'somehost.io' is the registry + * - 'owner/repo' is the package name + * - '8.1.0-alpine' is the tag + * + * This function will filter only tags that contain a semver version + */ + async getReleases({ lookupName, - registryUrl - ); - const tags = await getTags(registryHost, dockerRepository); - if (!tags) { - return null; - } - const releases = tags.map((version) => ({ version })); - const ret: ReleaseResult = { - registryUrl: registryHost, - releases, - }; + registryUrl, + }: GetReleasesConfig): Promise { + const { registryHost, dockerRepository } = getRegistryRepository( + lookupName, + registryUrl + ); + const tags = await this.getTags(registryHost, dockerRepository); + if (!tags) { + return null; + } + const releases = tags.map((version) => ({ version })); + const ret: ReleaseResult = { + registryUrl: registryHost, + releases, + }; - const latestTag = tags.includes('latest') ? 'latest' : findLatestStable(tags); - const labels = await getLabels(registryHost, dockerRepository, latestTag); - if (labels) { - for (const label of sourceLabels) { - if (is.nonEmptyString(labels[label])) { - ret.sourceUrl = labels[label]; - break; + const latestTag = tags.includes('latest') + ? 'latest' + : findLatestStable(tags); + const labels = await this.getLabels( + registryHost, + dockerRepository, + latestTag + ); + if (labels) { + for (const label of sourceLabels) { + if (is.nonEmptyString(labels[label])) { + ret.sourceUrl = labels[label]; + break; + } } } + return ret; } - return ret; } diff --git a/lib/manager/ansible/index.ts b/lib/manager/ansible/index.ts index cbe22b450165cd..d1a13cdbea55fb 100644 --- a/lib/manager/ansible/index.ts +++ b/lib/manager/ansible/index.ts @@ -1,5 +1,5 @@ import { ProgrammingLanguage } from '../../constants'; -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; export { extractPackageFile } from './extract'; export const language = ProgrammingLanguage.Docker; @@ -8,4 +8,4 @@ export const defaultConfig = { fileMatch: ['(^|/)tasks/[^/]+\\.ya?ml$'], }; -export const supportedDatasources = [datasourceDocker.id]; +export const supportedDatasources = [DockerDatasource.id]; diff --git a/lib/manager/bazel/extract.ts b/lib/manager/bazel/extract.ts index a35ac770be7317..180042a2ef69c8 100644 --- a/lib/manager/bazel/extract.ts +++ b/lib/manager/bazel/extract.ts @@ -2,7 +2,7 @@ import { parse as _parse } from 'url'; import parse from 'github-url-from-git'; import moo from 'moo'; -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import * as datasourceGithubReleases from '../../datasource/github-releases'; import * as datasourceGithubTags from '../../datasource/github-tags'; import { GoDatasource } from '../../datasource/go'; @@ -300,7 +300,7 @@ export function extractPackageFile( dep.currentValue = currentValue; dep.depName = depName; dep.versioning = dockerVersioning.id; - dep.datasource = datasourceDocker.id; + dep.datasource = DockerDatasource.id; dep.lookupName = repository; dep.registryUrls = [registry]; deps.push(dep); diff --git a/lib/manager/bazel/index.ts b/lib/manager/bazel/index.ts index aa0568fcec5588..9de5569ecb5cd3 100644 --- a/lib/manager/bazel/index.ts +++ b/lib/manager/bazel/index.ts @@ -1,4 +1,4 @@ -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import * as datasourceGithubReleases from '../../datasource/github-releases'; import * as datasourceGithubTags from '../../datasource/github-tags'; import { GoDatasource } from '../../datasource/go'; @@ -12,7 +12,7 @@ export const defaultConfig = { }; export const supportedDatasources = [ - datasourceDocker.id, + DockerDatasource.id, datasourceGithubReleases.id, datasourceGithubTags.id, GoDatasource.id, diff --git a/lib/manager/bitbucket-pipelines/index.ts b/lib/manager/bitbucket-pipelines/index.ts index a51d837bc58277..930c4a868f90d6 100644 --- a/lib/manager/bitbucket-pipelines/index.ts +++ b/lib/manager/bitbucket-pipelines/index.ts @@ -1,4 +1,4 @@ -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import { extractPackageFile } from './extract'; export { extractPackageFile }; @@ -7,4 +7,4 @@ export const defaultConfig = { fileMatch: ['(^|/)\\.?bitbucket-pipelines\\.ya?ml$'], }; -export const supportedDatasources = [datasourceDocker.id]; +export const supportedDatasources = [DockerDatasource.id]; diff --git a/lib/manager/circleci/index.ts b/lib/manager/circleci/index.ts index 705099e16def2c..6bb051449a4f78 100644 --- a/lib/manager/circleci/index.ts +++ b/lib/manager/circleci/index.ts @@ -1,4 +1,4 @@ -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import { OrbDatasource } from '../../datasource/orb'; import { extractPackageFile } from './extract'; @@ -8,4 +8,4 @@ export const defaultConfig = { fileMatch: ['(^|/).circleci/config.yml$'], }; -export const supportedDatasources = [datasourceDocker.id, OrbDatasource.id]; +export const supportedDatasources = [DockerDatasource.id, OrbDatasource.id]; diff --git a/lib/manager/cloudbuild/index.ts b/lib/manager/cloudbuild/index.ts index cb4dc10014fd2f..b1c2d7f1c43142 100644 --- a/lib/manager/cloudbuild/index.ts +++ b/lib/manager/cloudbuild/index.ts @@ -1,4 +1,4 @@ -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import { extractPackageFile } from './extract'; export { extractPackageFile }; @@ -7,4 +7,4 @@ export const defaultConfig = { fileMatch: ['(^|/)cloudbuild.ya?ml'], }; -export const supportedDatasources = [datasourceDocker.id]; +export const supportedDatasources = [DockerDatasource.id]; diff --git a/lib/manager/docker-compose/index.ts b/lib/manager/docker-compose/index.ts index d2314c5b646bf7..697140b7d883ee 100644 --- a/lib/manager/docker-compose/index.ts +++ b/lib/manager/docker-compose/index.ts @@ -1,5 +1,5 @@ import { ProgrammingLanguage } from '../../constants'; -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import { extractPackageFile } from './extract'; const language = ProgrammingLanguage.Docker; @@ -10,4 +10,4 @@ export const defaultConfig = { fileMatch: ['(^|/)docker-compose[^/]*\\.ya?ml$'], }; -export const supportedDatasources = [datasourceDocker.id]; +export const supportedDatasources = [DockerDatasource.id]; diff --git a/lib/manager/dockerfile/extract.ts b/lib/manager/dockerfile/extract.ts index 1a7e973460850a..78bd5a2f124bbc 100644 --- a/lib/manager/dockerfile/extract.ts +++ b/lib/manager/dockerfile/extract.ts @@ -1,5 +1,5 @@ import is from '@sindresorhus/is'; -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import { logger } from '../../logger'; import { regEx } from '../../util/regex'; import * as ubuntuVersioning from '../../versioning/ubuntu'; @@ -119,7 +119,7 @@ export function getDep( dep.autoReplaceStringTemplate = '{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}'; } - dep.datasource = datasourceDocker.id; + dep.datasource = DockerDatasource.id; // Pretty up special prefixes if (dep.depName) { diff --git a/lib/manager/dockerfile/index.ts b/lib/manager/dockerfile/index.ts index 7ebec5c8773dc1..59f6c25ef446a1 100644 --- a/lib/manager/dockerfile/index.ts +++ b/lib/manager/dockerfile/index.ts @@ -1,5 +1,5 @@ import { ProgrammingLanguage } from '../../constants'; -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import { extractPackageFile } from './extract'; const language = ProgrammingLanguage.Docker; @@ -10,4 +10,4 @@ export const defaultConfig = { fileMatch: ['(^|/|\\.)Dockerfile$', '(^|/)Dockerfile\\.[^/]*$'], }; -export const supportedDatasources = [datasourceDocker.id]; +export const supportedDatasources = [DockerDatasource.id]; diff --git a/lib/manager/droneci/index.ts b/lib/manager/droneci/index.ts index a46b91d3f3752c..4eb3e0545acec6 100644 --- a/lib/manager/droneci/index.ts +++ b/lib/manager/droneci/index.ts @@ -1,5 +1,5 @@ import { ProgrammingLanguage } from '../../constants'; -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import { extractPackageFile } from './extract'; const language = ProgrammingLanguage.Docker; @@ -10,4 +10,4 @@ export const defaultConfig = { fileMatch: ['(^|/).drone.yml$'], }; -export const supportedDatasources = [datasourceDocker.id]; +export const supportedDatasources = [DockerDatasource.id]; diff --git a/lib/manager/gitlabci/index.ts b/lib/manager/gitlabci/index.ts index 802c5072bb62f4..10d679fdb39f9e 100644 --- a/lib/manager/gitlabci/index.ts +++ b/lib/manager/gitlabci/index.ts @@ -1,5 +1,5 @@ import { ProgrammingLanguage } from '../../constants'; -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import { extractAllPackageFiles, extractPackageFile } from './extract'; const language = ProgrammingLanguage.Docker; @@ -10,4 +10,4 @@ export const defaultConfig = { fileMatch: ['\\.gitlab-ci\\.yml$'], }; -export const supportedDatasources = [datasourceDocker.id]; +export const supportedDatasources = [DockerDatasource.id]; diff --git a/lib/manager/helm-values/index.ts b/lib/manager/helm-values/index.ts index 2e267255000042..f941ce869432d7 100644 --- a/lib/manager/helm-values/index.ts +++ b/lib/manager/helm-values/index.ts @@ -1,4 +1,4 @@ -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; export { extractPackageFile } from './extract'; export const defaultConfig = { @@ -7,4 +7,4 @@ export const defaultConfig = { pinDigests: false, }; -export const supportedDatasources = [datasourceDocker.id]; +export const supportedDatasources = [DockerDatasource.id]; diff --git a/lib/manager/helmv3/artifacts.ts b/lib/manager/helmv3/artifacts.ts index fe52ce7ff06263..f2aa750463897d 100644 --- a/lib/manager/helmv3/artifacts.ts +++ b/lib/manager/helmv3/artifacts.ts @@ -2,7 +2,7 @@ import yaml from 'js-yaml'; import { quote } from 'shlex'; import upath from 'upath'; import { TEMPORARY_ERROR } from '../../constants/error-messages'; -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import { logger } from '../../logger'; import { exec } from '../../util/exec'; import type { ExecOptions, ToolConstraint } from '../../util/exec/types'; @@ -44,7 +44,7 @@ async function helmCommands( repository: value.repository.replace('oci://', ''), hostRule: hostRules.find({ url: value.repository.replace('oci://', 'https://'), //TODO we need to replace this, as oci:// will not be accepted as protocol - hostType: datasourceDocker.id, + hostType: DockerDatasource.id, }), }; }); diff --git a/lib/manager/helmv3/extract.spec.ts b/lib/manager/helmv3/extract.spec.ts index 1261309886300f..0c385afa438afc 100644 --- a/lib/manager/helmv3/extract.spec.ts +++ b/lib/manager/helmv3/extract.spec.ts @@ -1,5 +1,5 @@ import { fs } from '../../../test/util'; -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import { extractPackageFile } from './extract'; jest.mock('../../util/fs'); @@ -97,7 +97,7 @@ describe('manager/helmv3/extract', () => { deps: [ { depName: 'library', - datasource: datasourceDocker.id, + datasource: DockerDatasource.id, currentValue: '0.1.0', }, { depName: 'postgresql', currentValue: '0.8.1' }, diff --git a/lib/manager/helmv3/index.ts b/lib/manager/helmv3/index.ts index de5feaa3731a6e..d0ef5e718dfe53 100644 --- a/lib/manager/helmv3/index.ts +++ b/lib/manager/helmv3/index.ts @@ -1,4 +1,4 @@ -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import { HelmDatasource } from '../../datasource/helm'; export { updateArtifacts } from './artifacts'; export { extractPackageFile } from './extract'; @@ -14,4 +14,4 @@ export const defaultConfig = { fileMatch: ['(^|/)Chart.yaml$'], }; -export const supportedDatasources = [datasourceDocker.id, HelmDatasource.id]; +export const supportedDatasources = [DockerDatasource.id, HelmDatasource.id]; diff --git a/lib/manager/helmv3/utils.ts b/lib/manager/helmv3/utils.ts index 0e520c44808f71..03dfe95d1233ce 100644 --- a/lib/manager/helmv3/utils.ts +++ b/lib/manager/helmv3/utils.ts @@ -1,4 +1,4 @@ -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import { logger } from '../../logger'; import type { PackageDependency } from '../types'; import type { ChartDefinition, Repository } from './types'; @@ -13,7 +13,7 @@ export function parseRepository( const url = new URL(repositoryURL); switch (url.protocol) { case 'oci:': - res.datasource = datasourceDocker.id; + res.datasource = DockerDatasource.id; res.lookupName = `${repositoryURL.replace('oci://', '')}/${depName}`; break; case 'file:': diff --git a/lib/manager/kubernetes/index.ts b/lib/manager/kubernetes/index.ts index 053562ef183ae2..df3a060c61c98f 100644 --- a/lib/manager/kubernetes/index.ts +++ b/lib/manager/kubernetes/index.ts @@ -1,5 +1,5 @@ import { ProgrammingLanguage } from '../../constants'; -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; export { extractPackageFile } from './extract'; @@ -9,4 +9,4 @@ export const defaultConfig = { fileMatch: [], }; -export const supportedDatasources = [datasourceDocker.id]; +export const supportedDatasources = [DockerDatasource.id]; diff --git a/lib/manager/kustomize/extract.spec.ts b/lib/manager/kustomize/extract.spec.ts index a0120ad4f7aa83..9ea52829f241ba 100644 --- a/lib/manager/kustomize/extract.spec.ts +++ b/lib/manager/kustomize/extract.spec.ts @@ -1,5 +1,5 @@ import { loadFixture } from '../../../test/util'; -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import { GitTagsDatasource } from '../../datasource/git-tags'; import * as datasourceGitHubTags from '../../datasource/github-tags'; import { HelmDatasource } from '../../datasource/helm'; @@ -158,7 +158,7 @@ describe('manager/kustomize/extract', () => { const sample = { currentDigest: undefined, currentValue: 'v1.0.0', - datasource: datasourceDocker.id, + datasource: DockerDatasource.id, replaceString: 'v1.0.0', depName: 'node', }; @@ -172,7 +172,7 @@ describe('manager/kustomize/extract', () => { const sample = { currentDigest: undefined, currentValue: 'v1.0.0', - datasource: datasourceDocker.id, + datasource: DockerDatasource.id, replaceString: 'v1.0.0', depName: 'test/node', }; @@ -186,7 +186,7 @@ describe('manager/kustomize/extract', () => { const sample = { currentDigest: undefined, currentValue: 'v1.0.0', - datasource: datasourceDocker.id, + datasource: DockerDatasource.id, replaceString: 'v1.0.0', depName: 'quay.io/repo/image', }; @@ -200,7 +200,7 @@ describe('manager/kustomize/extract', () => { const sample = { currentDigest: undefined, currentValue: 'v1.0.0', - datasource: datasourceDocker.id, + datasource: DockerDatasource.id, replaceString: 'v1.0.0', depName: 'localhost:5000/repo/image', }; @@ -215,7 +215,7 @@ describe('manager/kustomize/extract', () => { currentDigest: undefined, currentValue: 'v1.0.0', replaceString: 'v1.0.0', - datasource: datasourceDocker.id, + datasource: DockerDatasource.id, depName: 'localhost:5000/repo/image/service', }; const pkg = extractImage({ diff --git a/lib/manager/kustomize/extract.ts b/lib/manager/kustomize/extract.ts index 3b58a1bf6b4107..cd0389b5155102 100644 --- a/lib/manager/kustomize/extract.ts +++ b/lib/manager/kustomize/extract.ts @@ -1,6 +1,6 @@ import is from '@sindresorhus/is'; import { load } from 'js-yaml'; -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import { GitTagsDatasource } from '../../datasource/git-tags'; import * as datasourceGitHubTags from '../../datasource/github-tags'; import { HelmDatasource } from '../../datasource/helm'; @@ -70,7 +70,7 @@ export function extractImage(image: Image): PackageDependency | null { } return { - datasource: datasourceDocker.id, + datasource: DockerDatasource.id, depName, currentValue: nameDep.currentValue, currentDigest: digest, @@ -90,7 +90,7 @@ export function extractImage(image: Image): PackageDependency | null { const dep = splitImageParts(`${depName}:${newTag}`); return { ...dep, - datasource: datasourceDocker.id, + datasource: DockerDatasource.id, replaceString: newTag, }; } @@ -98,7 +98,7 @@ export function extractImage(image: Image): PackageDependency | null { if (image.newName) { return { ...nameDep, - datasource: datasourceDocker.id, + datasource: DockerDatasource.id, replaceString: image.newName, }; } diff --git a/lib/manager/kustomize/index.ts b/lib/manager/kustomize/index.ts index c3ac88cd6e180c..3d7b6a048399cd 100644 --- a/lib/manager/kustomize/index.ts +++ b/lib/manager/kustomize/index.ts @@ -1,4 +1,4 @@ -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import { GitTagsDatasource } from '../../datasource/git-tags'; import * as datasourceGitHubTags from '../../datasource/github-tags'; import { HelmDatasource } from '../../datasource/helm'; @@ -10,7 +10,7 @@ export const defaultConfig = { }; export const supportedDatasources = [ - datasourceDocker.id, + DockerDatasource.id, GitTagsDatasource.id, datasourceGitHubTags.id, HelmDatasource.id, diff --git a/lib/manager/pyenv/extract.ts b/lib/manager/pyenv/extract.ts index 8fee5607c1ff01..a402da093160f4 100644 --- a/lib/manager/pyenv/extract.ts +++ b/lib/manager/pyenv/extract.ts @@ -1,11 +1,11 @@ -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import type { PackageDependency, PackageFile } from '../types'; export function extractPackageFile(content: string): PackageFile { const dep: PackageDependency = { depName: 'python', currentValue: content.trim(), - datasource: datasourceDocker.id, + datasource: DockerDatasource.id, }; return { deps: [dep] }; } diff --git a/lib/manager/pyenv/index.ts b/lib/manager/pyenv/index.ts index 3be9c1f5836ee6..f4ddc5b275c442 100644 --- a/lib/manager/pyenv/index.ts +++ b/lib/manager/pyenv/index.ts @@ -1,12 +1,12 @@ import { ProgrammingLanguage } from '../../constants'; -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import * as dockerVersioning from '../../versioning/docker'; export { extractPackageFile } from './extract'; export const language = ProgrammingLanguage.Python; -export const supportedDatasources = [datasourceDocker.id]; +export const supportedDatasources = [DockerDatasource.id]; export const defaultConfig = { fileMatch: ['(^|/).python-version$'], diff --git a/lib/util/package-rules.spec.ts b/lib/util/package-rules.spec.ts index c39a6e7cae1810..518627bd27302a 100644 --- a/lib/util/package-rules.spec.ts +++ b/lib/util/package-rules.spec.ts @@ -1,7 +1,7 @@ import type { PackageRuleInputConfig, UpdateType } from '../config/types'; import { ProgrammingLanguage } from '../constants'; -import * as datasourceDocker from '../datasource/docker'; +import { DockerDatasource } from '../datasource/docker'; import { OrbDatasource } from '../datasource/orb'; import { applyPackageRules } from './package-rules'; @@ -319,7 +319,7 @@ describe('util/package-rules', () => { const config: TestConfig = { packageRules: [ { - matchDatasources: [OrbDatasource.id, datasourceDocker.id], + matchDatasources: [OrbDatasource.id, DockerDatasource.id], x: 1, }, ], diff --git a/lib/workers/global/config/parse/cli.spec.ts b/lib/workers/global/config/parse/cli.spec.ts index 41114a2980defa..806f547ed95265 100644 --- a/lib/workers/global/config/parse/cli.spec.ts +++ b/lib/workers/global/config/parse/cli.spec.ts @@ -1,4 +1,4 @@ -import * as datasourceDocker from '../../../../datasource/docker'; +import { DockerDatasource } from '../../../../datasource/docker'; import getArgv from './__fixtures__/argv'; import * as cli from './cli'; import type { ParseConfigOptions } from './types'; @@ -78,13 +78,13 @@ describe('workers/global/config/parse/cli', () => { }); it('parses json lists correctly', () => { argv.push( - `--host-rules=[{"matchHost":"docker.io","hostType":"${datasourceDocker.id}","username":"user","password":"password"}]` + `--host-rules=[{"matchHost":"docker.io","hostType":"${DockerDatasource.id}","username":"user","password":"password"}]` ); expect(cli.getConfig(argv)).toEqual({ hostRules: [ { matchHost: 'docker.io', - hostType: datasourceDocker.id, + hostType: DockerDatasource.id, username: 'user', password: 'password', }, diff --git a/lib/workers/global/index.spec.ts b/lib/workers/global/index.spec.ts index b1d7df99b4bd5a..0bc8db30fe9f61 100644 --- a/lib/workers/global/index.spec.ts +++ b/lib/workers/global/index.spec.ts @@ -4,7 +4,7 @@ import { fs, logger, mocked } from '../../../test/util'; import * as _presets from '../../config/presets'; import { PlatformId } from '../../constants'; import { CONFIG_PRESETS_INVALID } from '../../constants/error-messages'; -import * as datasourceDocker from '../../datasource/docker'; +import { DockerDatasource } from '../../datasource/docker'; import * as _platform from '../../platform'; import * as _repositoryWorker from '../repository'; import * as _configParser from './config/parse'; @@ -81,7 +81,7 @@ describe('workers/global/index', () => { repositories: ['a', 'b'], hostRules: [ { - hostType: datasourceDocker.id, + hostType: DockerDatasource.id, username: 'some-user', password: 'some-password', }, @@ -100,7 +100,7 @@ describe('workers/global/index', () => { repositories: ['a', 'b'], hostRules: [ { - hostType: datasourceDocker.id, + hostType: DockerDatasource.id, username: 'some-user', password: 'some-password', }, diff --git a/lib/workers/repository/process/lookup/index.spec.ts b/lib/workers/repository/process/lookup/index.spec.ts index 1afcb616b05ee3..c2c894f8cfacef 100644 --- a/lib/workers/repository/process/lookup/index.spec.ts +++ b/lib/workers/repository/process/lookup/index.spec.ts @@ -6,8 +6,7 @@ import { partial, } from '../../../../../test/util'; import { CONFIG_VALIDATION } from '../../../../constants/error-messages'; -import * as datasourceDocker from '../../../../datasource/docker'; -import { id as datasourceDockerId } from '../../../../datasource/docker'; +import { DockerDatasource } from '../../../../datasource/docker'; import { GitRefsDatasource } from '../../../../datasource/git-refs'; import { GitDatasource } from '../../../../datasource/git-refs/base'; import * as datasourceGithubReleases from '../../../../datasource/github-releases'; @@ -39,8 +38,7 @@ const typescriptJson = loadJsonFixture('typescript.json', fixtureRoot); const vueJson = loadJsonFixture('vue.json', fixtureRoot); const webpackJson = loadJsonFixture('webpack.json', fixtureRoot); -const docker = mocked(datasourceDocker) as any; -docker.defaultRegistryUrls = ['https://index.docker.io']; +const docker = mocked(DockerDatasource.prototype); const githubReleases = mocked(datasourceGithubReleases); Object.assign(githubReleases, { defaultRegistryUrls: ['https://github.com'] }); @@ -1302,20 +1300,20 @@ describe('workers/repository/process/lookup/index', () => { it('skips unsupported values', async () => { config.currentValue = 'alpine'; config.depName = 'node'; - config.datasource = datasourceDockerId; + config.datasource = DockerDatasource.id; const res = await lookup.lookupUpdates(config); expect(res).toMatchSnapshot({ skipReason: 'invalid-value' }); }); it('skips undefined values', async () => { config.depName = 'node'; - config.datasource = datasourceDockerId; + config.datasource = DockerDatasource.id; const res = await lookup.lookupUpdates(config); expect(res).toMatchSnapshot({ skipReason: 'invalid-value' }); }); it('handles digest pin', async () => { config.currentValue = '8.0.0'; config.depName = 'node'; - config.datasource = datasourceDockerId; + config.datasource = DockerDatasource.id; config.pinDigests = true; docker.getReleases.mockResolvedValueOnce({ releases: [ @@ -1351,7 +1349,7 @@ describe('workers/repository/process/lookup/index', () => { config.currentValue = '8.1.0'; config.depName = 'node'; config.versioning = dockerVersioningId; - config.datasource = datasourceDockerId; + config.datasource = DockerDatasource.id; docker.getReleases.mockResolvedValueOnce({ releases: [ { version: '8.1.0' }, @@ -1374,7 +1372,7 @@ describe('workers/repository/process/lookup/index', () => { config.currentValue = '8.1'; config.depName = 'node'; config.versioning = dockerVersioningId; - config.datasource = datasourceDockerId; + config.datasource = DockerDatasource.id; docker.getReleases.mockResolvedValueOnce({ releases: [ { version: '8.1.0' }, @@ -1400,7 +1398,7 @@ describe('workers/repository/process/lookup/index', () => { config.currentValue = '8'; config.depName = 'node'; config.versioning = dockerVersioningId; - config.datasource = datasourceDockerId; + config.datasource = DockerDatasource.id; docker.getReleases.mockResolvedValueOnce({ releases: [ { version: '8.1.0' }, @@ -1422,7 +1420,7 @@ describe('workers/repository/process/lookup/index', () => { it('handles digest pin for up to date version', async () => { config.currentValue = '8.1.0'; config.depName = 'node'; - config.datasource = datasourceDockerId; + config.datasource = DockerDatasource.id; config.pinDigests = true; docker.getReleases.mockResolvedValueOnce({ releases: [ @@ -1449,7 +1447,7 @@ describe('workers/repository/process/lookup/index', () => { it('handles digest pin for non-version', async () => { config.currentValue = 'alpine'; config.depName = 'node'; - config.datasource = datasourceDockerId; + config.datasource = DockerDatasource.id; config.pinDigests = true; docker.getReleases.mockResolvedValueOnce({ releases: [ @@ -1479,7 +1477,7 @@ describe('workers/repository/process/lookup/index', () => { it('handles digest lookup failure', async () => { config.currentValue = 'alpine'; config.depName = 'node'; - config.datasource = datasourceDockerId; + config.datasource = DockerDatasource.id; config.pinDigests = true; docker.getReleases.mockResolvedValueOnce({ releases: [ @@ -1501,7 +1499,7 @@ describe('workers/repository/process/lookup/index', () => { it('handles digest update', async () => { config.currentValue = '8.0.0'; config.depName = 'node'; - config.datasource = datasourceDockerId; + config.datasource = DockerDatasource.id; config.currentDigest = 'sha256:zzzzzzzzzzzzzzz'; config.pinDigests = true; docker.getReleases.mockResolvedValueOnce({ @@ -1535,7 +1533,7 @@ describe('workers/repository/process/lookup/index', () => { it('handles digest update for non-version', async () => { config.currentValue = 'alpine'; config.depName = 'node'; - config.datasource = datasourceDockerId; + config.datasource = DockerDatasource.id; config.currentDigest = 'sha256:zzzzzzzzzzzzzzz'; config.pinDigests = true; docker.getReleases.mockResolvedValueOnce({