refactor(datasource): migrate to class based datasource

    A small experiment into what OOP/class based datasources might look like. Picked Cdnjs as it's the smallest & simplest.

    With this approach we can share common logic, like error handling, rate limiting, etc. between different datasources, instead of having to reimplement it each time we write a new datasource. Currently there's nothing shared, as it's only 1 datasource, but the interesting stuff will come with the 2nd datasource

.eslintrc.js

@@ -42,6 +42,9 @@ module.exports = {
     'prefer-template': 0,
     'no-underscore-dangle': 0,
     'no-negated-condition': 'error',
+    // TODO: re-enable once datasource class migration
+    'class-methods-use-this': 0,
+
     'sort-imports': [
       'error',
       {

lib/datasource/api.ts

@@ -35,8 +35,8 @@ const api = new Map<string, DatasourceApi>();
 export default api;
 
 api.set('bitbucket-tags', bitbucketTags);
-api.set('cdnjs', cdnjs);
-api.set('clojure', clojure);
+api.set('cdnjs', new cdnjs.CdnJs());
+api.set('clojure', new clojure.Clojure());
 api.set('crate', crate);
 api.set('dart', dart);
 api.set('docker', docker);

lib/datasource/cdnjs/__snapshots__/index.spec.ts.snap

@@ -185,7 +185,7 @@ Array [
 ]
 `;
 
-exports[`datasource/cdnjs getReleases returns null for unknown error 1`] = `
+exports[`datasource/cdnjs getReleases throws for unknown error 1`] = `
 Array [
   Object {
     "headers": Object {

lib/datasource/cdnjs/index.spec.ts

@@ -87,7 +87,7 @@ describe('datasource/cdnjs', () => {
       ).rejects.toThrow(EXTERNAL_HOST_ERROR);
       expect(httpMock.getTrace()).toMatchSnapshot();
     });
-    it('returns null for unknown error', async () => {
+    it('throws for unknown error', async () => {
       httpMock.scope(baseUrl).get(pathFor('foo/bar')).replyWithError('error');
       await expect(
         getPkgReleases({ datasource, depName: 'foo/bar' })

lib/datasource/cdnjs/index.ts

@@ -1,14 +1,12 @@
 import { ExternalHostError } from '../../types/errors/external-host-error';
-import { Http } from '../../util/http';
-import type { GetReleasesConfig, ReleaseResult } from '../types';
+import { Datasource } from '../datasource';
+import { GetReleasesConfig, ReleaseResult } from '../types';
 
 export const id = 'cdnjs';
 export const customRegistrySupport = false;
 export const defaultRegistryUrls = ['https://api.cdnjs.com/'];
 export const caching = true;
 
-const http = new Http(id);
-
 interface CdnjsAsset {
   version: string;
   files: string[];
@@ -24,38 +22,43 @@ interface CdnjsResponse {
   assets?: CdnjsAsset[];
 }
 
-export async function getReleases({
-  lookupName,
-  registryUrl,
-}: GetReleasesConfig): Promise<ReleaseResult | null> {
-  // Each library contains multiple assets, so we cache at the library level instead of per-asset
-  const library = lookupName.split('/')[0];
-  const url = `${registryUrl}libraries/${library}?fields=homepage,repository,assets`;
-  try {
-    const { assets, homepage, repository } = (
-      await http.getJson<CdnjsResponse>(url)
-    ).body;
-    if (!assets) {
-      return null;
-    }
-    const assetName = lookupName.replace(`${library}/`, '');
-    const releases = assets
-      .filter(({ files }) => files.includes(assetName))
-      .map(({ version, sri }) => ({ version, newDigest: sri[assetName] }));
+export class CdnJs extends Datasource {
+  readonly id = 'cdnjs';
 
-    const result: ReleaseResult = { releases };
+  // this.handleErrors will always throw
+  // eslint-disable-next-line consistent-return
+  async getReleases({
+    lookupName,
+  }: GetReleasesConfig): Promise<ReleaseResult | null> {
+    // Each library contains multiple assets, so we cache at the library level instead of per-asset
+    const library = lookupName.split('/')[0];
+    const url = `https://api.cdnjs.com/libraries/${library}?fields=homepage,repository,assets`;
+    try {
+      const { assets, homepage, repository } = (
+        await this.http.getJson<CdnjsResponse>(url)
+      ).body;
+      if (!assets) {
+        return null;
+      }
+      const assetName = lookupName.replace(`${library}/`, '');
+      const releases = assets
+        .filter(({ files }) => files.includes(assetName))
+        .map(({ version, sri }) => ({ version, newDigest: sri[assetName] }));
 
-    if (homepage) {
-      result.homepage = homepage;
-    }
-    if (repository?.url) {
-      result.sourceUrl = repository.url;
-    }
-    return result;
-  } catch (err) {
-    if (err.statusCode !== 404) {
-      throw new ExternalHostError(err);
+      const result: ReleaseResult = { releases };
+
+      if (homepage) {
+        result.homepage = homepage;
+      }
+      if (repository?.url) {
+        result.sourceUrl = repository.url;
+      }
+      return result;
+    } catch (err) {
+      if (err.statusCode !== 404) {
+        throw new ExternalHostError(err);
+      }
+      this.handleGenericErrors(err);
     }
-    throw err;
   }
 }

lib/datasource/clojure/index.spec.ts

@@ -0,0 +1,51 @@
+import { Release, getPkgReleases } from '..';
+import * as mavenVersioning from '../../versioning/maven';
+import { id as datasource } from '.';
+
+const config = {
+  versioning: mavenVersioning.id,
+  datasource,
+};
+
+describe('datasource/clojure', () => {
+  describe('getReleases', () => {
+    function generateReleases(versions: string[]): Release[] {
+      return versions.map((v) => ({ version: v }));
+    }
+
+    it('should return empty if library is not found', async () => {
+      const releases = await getPkgReleases({
+        ...config,
+        depName: 'unknown:unknown',
+        registryUrls: [
+          's3://somewhere.s3.aws.amazon.com',
+          'file://lib/datasource/maven/__fixtures__/repo1.maven.org/maven2/',
+        ],
+      });
+      expect(releases).toBeNull();
+    });
+
+    it('should simply return all versions of a specific library', async () => {
+      const releases = await getPkgReleases({
+        ...config,
+        depName: 'org.hamcrest:hamcrest-core',
+        registryUrls: [
+          'file://lib/datasource/maven/__fixtures__/repo1.maven.org/maven2/',
+          'file://lib/datasource/maven/__fixtures__/custom_maven_repo/maven2/',
+          's3://somewhere.s3.aws.amazon.com',
+        ],
+      });
+      expect(releases.releases).toEqual(
+        generateReleases([
+          '1.1',
+          '1.2',
+          '1.2.1',
+          '1.3.RC2',
+          '1.3',
+          '2.1-rc2',
+          '2.1-rc3',
+        ])
+      );
+    });
+  });
+});

lib/datasource/clojure/index.ts

@@ -1,8 +1,26 @@
+import { Datasource } from '../datasource';
+import { getReleases } from '../maven';
 import { MAVEN_REPO } from '../maven/common';
+import { GetReleasesConfig, ReleaseResult } from '../types';
 
 export const id = 'clojure';
 export const customRegistrySupport = true;
 export const defaultRegistryUrls = ['https://clojars.org/repo', MAVEN_REPO];
 export const registryStrategy = 'merge';
 
-export { getReleases } from '../maven';
+export class Clojure extends Datasource {
+  readonly id = 'clojure';
+
+  readonly registryStrategy = 'merge';
+
+  readonly customRegistrySupport = true;
+
+  readonly defaultRegistryUrls = ['https://clojars.org/repo', MAVEN_REPO];
+
+  getReleases({
+    lookupName,
+    registryUrl,
+  }: GetReleasesConfig): Promise<ReleaseResult | null> {
+    return getReleases({ lookupName, registryUrl });
+  }
+}

lib/datasource/datasource.ts

@@ -0,0 +1,55 @@
+import { ExternalHostError } from '../types/errors/external-host-error';
+import { Http, HttpError } from '../util/http';
+import {
+  DatasourceApi,
+  DigestConfig,
+  GetReleasesConfig,
+  ReleaseResult,
+} from './types';
+
+export abstract class Datasource implements DatasourceApi {
+  constructor() {
+    this.http = new Http(this.getId());
+  }
+
+  abstract id: string;
+
+  caching: boolean;
+
+  defaultConfig: Record<string, unknown>;
+
+  customRegistrySupport: boolean;
+
+  defaultRegistryUrls: string[];
+
+  defaultVersioning: string;
+
+  registryStrategy: 'first' | 'hunt' | 'merge';
+
+  getId(): string {
+    return this.id;
+  }
+
+  protected http: Http;
+
+  abstract getReleases(
+    getReleasesConfig: GetReleasesConfig
+  ): Promise<ReleaseResult | null>;
+
+  getDigest?(config: DigestConfig, newValue?: string): Promise<string>;
+
+  handleSpecificErrors(err: HttpError): void {}
+
+  protected handleGenericErrors(err: HttpError): never {
+    this.handleSpecificErrors(err);
+    if (err.statusCode !== undefined) {
+      if (
+        err.statusCode === 429 ||
+        (err.statusCode >= 500 && err.statusCode < 600)
+      ) {
+        throw new ExternalHostError(err);
+      }
+    }
+    throw err;
+  }
+}

lib/datasource/index.spec.ts

@@ -32,9 +32,8 @@ describe(getName(__filename), () => {
   });
   it('returns datasources', () => {
     expect(datasource.getDatasources()).toBeDefined();
-    expect(datasource.getDatasourceList()).toBeDefined();
   });
-  it('validates dataource', () => {
+  it('validates datsource', () => {
     function validateDatasource(module: DatasourceApi, name: string): boolean {
       if (!module.getReleases) {
         return false;
@@ -44,9 +43,21 @@ describe(getName(__filename), () => {
       }
       return true;
     }
+    function filterClassBasedDatasources(name: string): boolean {
+      return !['cdnjs', 'clojure', 'crate'].includes(name);
+    }
     const dss = datasource.getDatasources();
 
-    const loadedDs = loadModules(__dirname, validateDatasource);
+    // class based datasources
+    dss.delete('cdnjs');
+    dss.delete('clojure');
+    dss.delete('crate');
+
+    const loadedDs = loadModules(
+      __dirname,
+      validateDatasource,
+      filterClassBasedDatasources
+    );
     expect(Array.from(dss.keys())).toEqual(Object.keys(loadedDs));
 
     for (const dsName of dss.keys()) {
@@ -83,6 +94,14 @@ describe(getName(__filename), () => {
       })
     ).toBeNull();
   });
+  it('returns class datasource', async () => {
+    expect(
+      await datasource.getPkgReleases({
+        datasource: 'cdnjs',
+        depName: null,
+      })
+    ).toBeNull();
+  });
   it('returns getDigest', async () => {
     expect(
       await datasource.getDigest({

lib/datasource/index.ts

@@ -9,6 +9,8 @@ import { clone } from '../util/clone';
 import { regEx } from '../util/regex';
 import * as allVersioning from '../versioning';
 import datasources from './api';
+import { CdnJs } from './cdnjs';
+import { Clojure } from './clojure';
 import { addMetaData } from './metadata';
 import type {
   DatasourceApi,
@@ -27,7 +29,10 @@ export const getDatasourceList = (): string[] => Array.from(datasources.keys());
 
 const cacheNamespace = 'datasource-releases';
 
-function load(datasource: string): DatasourceApi {
+datasources.set('cdnjs', new CdnJs());
+datasources.set('clojure', new Clojure());
+
+function getDatasourceFor(datasource: string): DatasourceApi {
   return datasources.get(datasource);
 }
 
@@ -192,19 +197,19 @@ function resolveRegistryUrls(
 }
 
 export function getDefaultVersioning(datasourceName: string): string {
-  const datasource = load(datasourceName);
+  const datasource = getDatasourceFor(datasourceName);
   return datasource.defaultVersioning || 'semver';
 }
 
 async function fetchReleases(
   config: GetReleasesInternalConfig
 ): Promise<ReleaseResult | null> {
   const { datasource: datasourceName } = config;
-  if (!datasourceName || !datasources.has(datasourceName)) {
+  if (!datasourceName || getDatasourceFor(datasourceName) === undefined) {
     logger.warn('Unknown datasource: ' + datasourceName);
     return null;
   }
-  const datasource = load(datasourceName);
+  const datasource = getDatasourceFor(datasourceName);
   const registryUrls = resolveRegistryUrls(datasource, config.registryUrls);
   let dep: ReleaseResult = null;
   const registryStrategy = datasource.registryStrategy || 'hunt';
@@ -346,14 +351,14 @@ export async function getPkgReleases(
 }
 
 export function supportsDigests(config: DigestConfig): boolean {
-  return 'getDigest' in load(config.datasource);
+  return 'getDigest' in getDatasourceFor(config.datasource);
 }
 
 export function getDigest(
   config: DigestConfig,
   value?: string
 ): Promise<string | null> {
-  const datasource = load(config.datasource);
+  const datasource = getDatasourceFor(config.datasource);
   const lookupName = config.lookupName || config.depName;
   const registryUrls = resolveRegistryUrls(datasource, config.registryUrls);
   return datasource.getDigest(
@@ -365,7 +370,7 @@ export function getDigest(
 export function getDefaultConfig(
   datasource: string
 ): Promise<Record<string, unknown>> {
-  const loadedDatasource = load(datasource);
+  const loadedDatasource = getDatasourceFor(datasource);
   return Promise.resolve<Record<string, unknown>>(
     loadedDatasource?.defaultConfig || Object.create({})
   );

lib/datasource/pod/index.ts

@@ -42,7 +42,7 @@ function releasesGithubUrl(
 function handleError(lookupName: string, err: HttpError): void {
   const errorData = { lookupName, err };
 
-  const statusCode = err.response?.statusCode;
+  const statusCode = err.statusCode;
   if (statusCode === 429 || (statusCode >= 500 && statusCode < 600)) {
     logger.warn({ lookupName, err }, `CocoaPods registry failure`);
     throw new ExternalHostError(err);