-
Notifications
You must be signed in to change notification settings - Fork 2.4k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
refactor(datasource): migrate to class based datasource
A small experiment into what OOP/class based datasources might look like. Picked Cdnjs as it's the smallest & simplest.
With this approach we can share common logic, like error handling, rate limiting, etc. between different datasources, instead of having to reimplement it each time we write a new datasource. Currently there's nothing shared, as it's only 1 datasource, but the interesting stuff will come with the 2nd datasource- Loading branch information
1 parent
e33df09
commit ce19bb2
Showing
16 changed files
with
265 additions
and
96 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,61 +1,56 @@ | ||
| import { ExternalHostError } from '../../types/errors/external-host-error'; | ||
| import { Http } from '../../util/http'; | ||
| import { Datasource } from '../datasource'; | ||
| import type { GetReleasesConfig, ReleaseResult } from '../types'; | ||
| import type { CdnjsResponse } from './types'; | ||
|
|
||
| export const id = 'cdnjs'; | ||
| export const customRegistrySupport = false; | ||
| export const defaultRegistryUrls = ['https://api.cdnjs.com/']; | ||
| export const caching = true; | ||
| export class CdnJsDatasource extends Datasource { | ||
| static readonly id = 'cdnjs'; | ||
|
|
||
| const http = new Http(id); | ||
| constructor() { | ||
| super(CdnJsDatasource.id); | ||
| } | ||
|
|
||
| interface CdnjsAsset { | ||
| version: string; | ||
| files: string[]; | ||
| sri?: Record<string, string>; | ||
| } | ||
| customRegistrySupport = false; | ||
|
|
||
| interface CdnjsResponse { | ||
| homepage?: string; | ||
| repository?: { | ||
| type: 'git' | unknown; | ||
| url?: string; | ||
| }; | ||
| assets?: CdnjsAsset[]; | ||
| } | ||
| defaultRegistryUrls = ['https://api.cdnjs.com/']; | ||
|
|
||
| export async function getReleases({ | ||
| lookupName, | ||
| registryUrl, | ||
| }: GetReleasesConfig): Promise<ReleaseResult | null> { | ||
| // Each library contains multiple assets, so we cache at the library level instead of per-asset | ||
| const library = lookupName.split('/')[0]; | ||
| const url = `${registryUrl}libraries/${library}?fields=homepage,repository,assets`; | ||
| try { | ||
| const { assets, homepage, repository } = ( | ||
| await http.getJson<CdnjsResponse>(url) | ||
| ).body; | ||
| if (!assets) { | ||
| return null; | ||
| } | ||
| const assetName = lookupName.replace(`${library}/`, ''); | ||
| const releases = assets | ||
| .filter(({ files }) => files.includes(assetName)) | ||
| .map(({ version, sri }) => ({ version, newDigest: sri[assetName] })); | ||
| caching = true; | ||
|
|
||
| const result: ReleaseResult = { releases }; | ||
| // this.handleErrors will always throw | ||
| // eslint-disable-next-line consistent-return | ||
| async getReleases({ | ||
| lookupName, | ||
| registryUrl, | ||
| }: GetReleasesConfig): Promise<ReleaseResult | null> { | ||
| // Each library contains multiple assets, so we cache at the library level instead of per-asset | ||
| const library = lookupName.split('/')[0]; | ||
| const url = `${registryUrl}libraries/${library}?fields=homepage,repository,assets`; | ||
| try { | ||
| const { assets, homepage, repository } = ( | ||
| await this.http.getJson<CdnjsResponse>(url) | ||
| ).body; | ||
| if (!assets) { | ||
| return null; | ||
| } | ||
| const assetName = lookupName.replace(`${library}/`, ''); | ||
| const releases = assets | ||
| .filter(({ files }) => files.includes(assetName)) | ||
| .map(({ version, sri }) => ({ version, newDigest: sri[assetName] })); | ||
|
|
||
| if (homepage) { | ||
| result.homepage = homepage; | ||
| } | ||
| if (repository?.url) { | ||
| result.sourceUrl = repository.url; | ||
| } | ||
| return result; | ||
| } catch (err) { | ||
| if (err.statusCode !== 404) { | ||
| throw new ExternalHostError(err); | ||
| const result: ReleaseResult = { releases }; | ||
|
|
||
| if (homepage) { | ||
| result.homepage = homepage; | ||
| } | ||
| if (repository?.url) { | ||
| result.sourceUrl = repository.url; | ||
| } | ||
| return result; | ||
| } catch (err) { | ||
| if (err.statusCode !== 404) { | ||
| throw new ExternalHostError(err); | ||
| } | ||
| this.handleGenericErrors(err); | ||
| } | ||
| throw err; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| interface CdnjsAsset { | ||
| version: string; | ||
| files: string[]; | ||
| sri?: Record<string, string>; | ||
| } | ||
|
|
||
| export interface CdnjsResponse { | ||
| homepage?: string; | ||
| repository?: { | ||
| type: 'git' | unknown; | ||
| url?: string; | ||
| }; | ||
| assets?: CdnjsAsset[]; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| import { Release, getPkgReleases } from '..'; | ||
| import { getName } from '../../../test/util'; | ||
| import { id as mavenVersioning } from '../../versioning/maven'; | ||
| import { ClojureDatasource } from '.'; | ||
|
|
||
| const config = { | ||
| versioning: mavenVersioning, | ||
| datasource: ClojureDatasource.id, | ||
| }; | ||
|
|
||
| describe(getName(), () => { | ||
| describe('getReleases', () => { | ||
| function generateReleases(versions: string[]): Release[] { | ||
| return versions.map((v) => ({ version: v })); | ||
| } | ||
|
|
||
| it('should return empty if library is not found', async () => { | ||
| const releases = await getPkgReleases({ | ||
| ...config, | ||
| depName: 'unknown:unknown', | ||
| registryUrls: [ | ||
| 's3://somewhere.s3.aws.amazon.com', | ||
| 'file://lib/datasource/maven/__fixtures__/repo1.maven.org/maven2/', | ||
| ], | ||
| }); | ||
| expect(releases).toBeNull(); | ||
| }); | ||
|
|
||
| it('should simply return all versions of a specific library', async () => { | ||
| const releases = await getPkgReleases({ | ||
| ...config, | ||
| depName: 'org.hamcrest:hamcrest-core', | ||
| registryUrls: [ | ||
| 'file://lib/datasource/maven/__fixtures__/repo1.maven.org/maven2/', | ||
| 'file://lib/datasource/maven/__fixtures__/custom_maven_repo/maven2/', | ||
| 's3://somewhere.s3.aws.amazon.com', | ||
| ], | ||
| }); | ||
| expect(releases.releases).toEqual( | ||
| generateReleases([ | ||
| '1.1', | ||
| '1.2', | ||
| '1.2.1', | ||
| '1.3.RC2', | ||
| '1.3', | ||
| '2.1-rc2', | ||
| '2.1-rc3', | ||
| ]) | ||
| ); | ||
| }); | ||
| }); | ||
| }); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,8 +1,26 @@ | ||
| import { Datasource } from '../datasource'; | ||
| import { getReleases } from '../maven'; | ||
| import { MAVEN_REPO } from '../maven/common'; | ||
| import type { GetReleasesConfig, ReleaseResult } from '../types'; | ||
|
|
||
| export const id = 'clojure'; | ||
| export const customRegistrySupport = true; | ||
| export const defaultRegistryUrls = ['https://clojars.org/repo', MAVEN_REPO]; | ||
| export const registryStrategy = 'merge'; | ||
| export class ClojureDatasource extends Datasource { | ||
| static readonly id = 'clojure'; | ||
|
|
||
| export { getReleases } from '../maven'; | ||
| constructor() { | ||
| super(ClojureDatasource.id); | ||
| } | ||
|
|
||
| readonly registryStrategy = 'merge'; | ||
|
|
||
| readonly customRegistrySupport = true; | ||
|
|
||
| readonly defaultRegistryUrls = ['https://clojars.org/repo', MAVEN_REPO]; | ||
|
|
||
| // eslint-disable-next-line class-methods-use-this | ||
| getReleases({ | ||
| lookupName, | ||
| registryUrl, | ||
| }: GetReleasesConfig): Promise<ReleaseResult | null> { | ||
| return getReleases({ lookupName, registryUrl }); | ||
| } | ||
| } |
Oops, something went wrong.