Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor(helm): convert to class-based datasource #10425

Merged
merged 2 commits into from
Jun 14, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions lib/datasource/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import * as githubTags from './github-tags';
import * as gitlabTags from './gitlab-tags';
import * as go from './go';
import * as gradleVersion from './gradle-version';
import * as helm from './helm';
import { HelmDatasource } from './helm';
import * as hex from './hex';
import * as jenkinsPlugins from './jenkins-plugins';
import * as maven from './maven';
Expand Down Expand Up @@ -50,7 +50,7 @@ api.set('github-tags', githubTags);
api.set('gitlab-tags', gitlabTags);
api.set('go', go);
api.set('gradle-version', gradleVersion);
api.set('helm', helm);
api.set('helm', new HelmDatasource());
api.set('hex', hex);
api.set('jenkins-plugins', jenkinsPlugins);
api.set('maven', maven);
Expand Down
26 changes: 13 additions & 13 deletions lib/datasource/helm/index.spec.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { getPkgReleases } from '..';
import * as httpMock from '../../../test/http-mock';
import { getName, loadFixture } from '../../../test/util';
import { id as datasource } from '.';
import { HelmDatasource } from '.';

// Truncated index.yaml file
const indexYaml = loadFixture('index.yaml');
Expand All @@ -15,7 +15,7 @@ describe(getName(), () => {
it('returns null if lookupName was not provided', async () => {
expect(
await getPkgReleases({
datasource,
datasource: HelmDatasource.id,
depName: undefined,
registryUrls: ['https://example-repository.com'],
})
Expand All @@ -24,7 +24,7 @@ describe(getName(), () => {
it('returns null if repository was not provided', async () => {
expect(
await getPkgReleases({
datasource,
datasource: HelmDatasource.id,
depName: 'some_chart',
registryUrls: [],
})
Expand All @@ -37,7 +37,7 @@ describe(getName(), () => {
.reply(200, null);
expect(
await getPkgReleases({
datasource,
datasource: HelmDatasource.id,
depName: 'non_existent_chart',
registryUrls: ['https://example-repository.com'],
})
Expand All @@ -51,7 +51,7 @@ describe(getName(), () => {
.reply(200, undefined);
expect(
await getPkgReleases({
datasource,
datasource: HelmDatasource.id,
depName: 'non_existent_chart',
registryUrls: ['https://example-repository.com'],
})
Expand All @@ -65,7 +65,7 @@ describe(getName(), () => {
.reply(404);
expect(
await getPkgReleases({
datasource,
datasource: HelmDatasource.id,
depName: 'some_chart',
registryUrls: ['https://example-repository.com'],
})
Expand All @@ -80,7 +80,7 @@ describe(getName(), () => {
let e;
try {
await getPkgReleases({
datasource,
datasource: HelmDatasource.id,
depName: 'some_chart',
registryUrls: ['https://example-repository.com'],
});
Expand All @@ -98,7 +98,7 @@ describe(getName(), () => {
.replyWithError('');
expect(
await getPkgReleases({
datasource,
datasource: HelmDatasource.id,
depName: 'some_chart',
registryUrls: ['https://example-repository.com'],
})
Expand All @@ -111,7 +111,7 @@ describe(getName(), () => {
.get('/index.yaml')
.reply(200, '# A comment');
const releases = await getPkgReleases({
datasource,
datasource: HelmDatasource.id,
depName: 'non_existent_chart',
registryUrls: ['https://example-repository.com'],
});
Expand All @@ -130,7 +130,7 @@ describe(getName(), () => {
.get('/index.yaml')
.reply(200, res);
const releases = await getPkgReleases({
datasource,
datasource: HelmDatasource.id,
depName: 'non_existent_chart',
registryUrls: ['https://example-repository.com'],
});
Expand All @@ -143,7 +143,7 @@ describe(getName(), () => {
.get('/index.yaml')
.reply(200, indexYaml);
const releases = await getPkgReleases({
datasource,
datasource: HelmDatasource.id,
depName: 'non_existent_chart',
registryUrls: ['https://example-repository.com'],
});
Expand All @@ -156,7 +156,7 @@ describe(getName(), () => {
.get('/index.yaml')
.reply(200, indexYaml);
const releases = await getPkgReleases({
datasource,
datasource: HelmDatasource.id,
depName: 'ambassador',
registryUrls: ['https://example-repository.com'],
});
Expand All @@ -170,7 +170,7 @@ describe(getName(), () => {
.get('/subdir/index.yaml')
.reply(200, indexYaml);
await getPkgReleases({
datasource,
datasource: HelmDatasource.id,
depName: 'ambassador',
registryUrls: ['https://example-repository.com/subdir'],
});
Expand Down
156 changes: 71 additions & 85 deletions lib/datasource/helm/index.ts
Original file line number Diff line number Diff line change
@@ -1,104 +1,90 @@
import is from '@sindresorhus/is';
import { load } from 'js-yaml';
import { logger } from '../../logger';
import { ExternalHostError } from '../../types/errors/external-host-error';
import * as packageCache from '../../util/cache/package';
import { Http } from '../../util/http';
import { cache } from '../../util/cache/package/decorator';
import { ensureTrailingSlash } from '../../util/url';
import { Datasource } from '../datasource';
import type { GetReleasesConfig, ReleaseResult } from '../types';
import type { HelmRepository, RepositoryData } from './types';

export const id = 'helm';
export class HelmDatasource extends Datasource {
static readonly id = 'helm';

const http = new Http(id);
constructor() {
super(HelmDatasource.id);
}

export const customRegistrySupport = true;
export const defaultRegistryUrls = ['https://charts.helm.sh/stable'];
export const registryStrategy = 'first';
readonly defaultRegistryUrls = ['https://charts.helm.sh/stable'];

export const defaultConfig = {
commitMessageTopic: 'Helm release {{depName}}',
group: {
commitMessageTopic: '{{{groupName}}} Helm releases',
},
};
readonly defaultConfig = {
commitMessageTopic: 'Helm release {{depName}}',
group: {
commitMessageTopic: '{{{groupName}}} Helm releases',
},
};

export async function getRepositoryData(
repository: string
): Promise<RepositoryData> {
const cacheNamespace = 'datasource-helm';
const cacheKey = repository;
const cachedIndex = await packageCache.get<RepositoryData>(
cacheNamespace,
cacheKey
);
// istanbul ignore if
if (cachedIndex) {
return cachedIndex;
}
let res: any;
try {
res = await http.get('index.yaml', {
baseUrl: ensureTrailingSlash(repository),
});
if (!res || !res.body) {
logger.warn(`Received invalid response from ${repository}`);
return null;
}
} catch (err) {
if (
err.statusCode === 429 ||
(err.statusCode >= 500 && err.statusCode < 600)
) {
throw new ExternalHostError(err);
@cache({
namespace: `datasource-${HelmDatasource.id}`,
key: (repository: string) => repository,
})
async getRepositoryData(repository: string): Promise<RepositoryData | null> {
let res: any;
try {
res = await this.http.get('index.yaml', {
baseUrl: ensureTrailingSlash(repository),
});
if (!res || !res.body) {
logger.warn(`Received invalid response from ${repository}`);
return null;
}
} catch (err) {
this.handleGenericErrors(err);
}
throw err;
}
try {
const doc = load(res.body, {
json: true,
}) as HelmRepository;
if (!is.plainObject<HelmRepository>(doc)) {
try {
const doc = load(res.body, {
json: true,
}) as HelmRepository;
if (!is.plainObject<HelmRepository>(doc)) {
logger.warn(`Failed to parse index.yaml from ${repository}`);
return null;
}
const result: RepositoryData = {};
for (const [name, releases] of Object.entries(doc.entries)) {
result[name] = {
homepage: releases[0].home,
sourceUrl: releases[0].sources ? releases[0].sources[0] : undefined,
releases: releases.map((release) => ({
version: release.version,
releaseTimestamp: release.created ? release.created : null,
})),
};
}

return result;
} catch (err) {
logger.warn(`Failed to parse index.yaml from ${repository}`);
logger.debug(err);
return null;
}
const result: RepositoryData = {};
for (const [name, releases] of Object.entries(doc.entries)) {
result[name] = {
homepage: releases[0].home,
sourceUrl: releases[0].sources ? releases[0].sources[0] : undefined,
releases: releases.map((release) => ({
version: release.version,
releaseTimestamp: release.created ? release.created : null,
})),
};
}
const cacheMinutes = 20;
await packageCache.set(cacheNamespace, cacheKey, result, cacheMinutes);
return result;
} catch (err) {
logger.warn(`Failed to parse index.yaml from ${repository}`);
logger.debug(err);
return null;
}
}

export async function getReleases({
lookupName,
registryUrl: helmRepository,
}: GetReleasesConfig): Promise<ReleaseResult | null> {
const repositoryData = await getRepositoryData(helmRepository);
if (!repositoryData) {
logger.debug(`Couldn't get index.yaml file from ${helmRepository}`);
return null;
}
const releases = repositoryData[lookupName];
if (!releases) {
logger.debug(
{ dependency: lookupName },
`Entry ${lookupName} doesn't exist in index.yaml from ${helmRepository}`
);
return null;
async getReleases({
lookupName,
registryUrl: helmRepository,
}: GetReleasesConfig): Promise<ReleaseResult | null> {
const repositoryData = await this.getRepositoryData(helmRepository);
if (!repositoryData) {
logger.debug(`Couldn't get index.yaml file from ${helmRepository}`);
return null;
}
const releases = repositoryData[lookupName];
if (!releases) {
logger.debug(
{ dependency: lookupName },
`Entry ${lookupName} doesn't exist in index.yaml from ${helmRepository}`
);
return null;
}
return releases;
}
return releases;
}
4 changes: 2 additions & 2 deletions lib/manager/argocd/extract.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { loadAll } from 'js-yaml';
import * as gitTags from '../../datasource/git-tags';
import * as helm from '../../datasource/helm';
import { HelmDatasource } from '../../datasource/helm';
import type { ExtractConfig, PackageDependency, PackageFile } from '../types';
import type { ApplicationDefinition } from './types';
import { fileTestRegex } from './util';
Expand All @@ -20,7 +20,7 @@ function createDependency(
depName: source.chart,
registryUrls: [source.repoURL],
currentValue: source.targetRevision,
datasource: helm.id,
datasource: HelmDatasource.id,
};
}
return {
Expand Down
4 changes: 2 additions & 2 deletions lib/manager/helm-requirements/extract.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import is from '@sindresorhus/is';
import { load } from 'js-yaml';
import * as datasourceHelm from '../../datasource/helm';
import { HelmDatasource } from '../../datasource/helm';
import { logger } from '../../logger';
import { SkipReason } from '../../types';
import type { ExtractConfig, PackageDependency, PackageFile } from '../types';
Expand Down Expand Up @@ -71,7 +71,7 @@ export function extractPackageFile(
});
const res = {
deps,
datasource: datasourceHelm.id,
datasource: HelmDatasource.id,
};
return res;
}
4 changes: 2 additions & 2 deletions lib/manager/helmfile/extract.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import is from '@sindresorhus/is';
import { loadAll } from 'js-yaml';
import * as datasourceHelm from '../../datasource/helm';
import { HelmDatasource } from '../../datasource/helm';
import { logger } from '../../logger';
import { SkipReason } from '../../types';
import type { ExtractConfig, PackageDependency, PackageFile } from '../types';
Expand Down Expand Up @@ -88,5 +88,5 @@ export function extractPackageFile(
return null;
}

return { deps, datasource: datasourceHelm.id } as PackageFile;
return { deps, datasource: HelmDatasource.id } as PackageFile;
}
4 changes: 2 additions & 2 deletions lib/manager/helmv3/extract.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import is from '@sindresorhus/is';
import { load } from 'js-yaml';
import * as datasourceHelm from '../../datasource/helm';
import { HelmDatasource } from '../../datasource/helm';
import { logger } from '../../logger';
import { SkipReason } from '../../types';
import { getSiblingFileName, localPathExists } from '../../util/fs';
Expand Down Expand Up @@ -90,7 +90,7 @@ export async function extractPackageFile(
});
const res: PackageFile = {
deps,
datasource: datasourceHelm.id,
datasource: HelmDatasource.id,
packageFileVersion,
};
const lockFileName = getSiblingFileName(fileName, 'Chart.lock');
Expand Down
4 changes: 2 additions & 2 deletions lib/manager/terraform/resources.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import * as datasourceHelm from '../../datasource/helm';
import { HelmDatasource } from '../../datasource/helm';
import { SkipReason } from '../../types';
import { getDep } from '../dockerfile/extract';
import type { PackageDependency } from '../types';
Expand Down Expand Up @@ -104,7 +104,7 @@ export function analyseTerraformResource(
dep.depType = 'helm_release';
dep.registryUrls = [dep.managerData.repository];
dep.depName = dep.managerData.chart;
dep.datasource = datasourceHelm.id;
dep.datasource = HelmDatasource.id;
break;

default:
Expand Down