refactor(datasource): migrate to class based datasource

A small experiment into what OOP/class based datasources might look like. Picked Cdnjs as it's the smallest & simplest.

With this approach we can share common logic, like error handling, rate limiting, etc. between different datasources, instead of having to reimplement it each time we write a new datasource. Currently there's nothing shared, as it's only 1 datasource, but the interesting stuff will come with the 2nd datasource

lib/datasource/cdnjs/index.ts

@@ -2,6 +2,7 @@ import { ExternalHostError } from '../../types/errors/external-host-error';
 import { Http } from '../../util/http';
 import { CachePromise, cacheAble } from '../cache';
 import { GetReleasesConfig, ReleaseResult } from '../common';
+import { Datasource } from '../datasource';
 
 export const id = 'cdnjs';
 
@@ -27,38 +28,42 @@ async function downloadLibrary(library: string): CachePromise<CdnjsResponse> {
   return { data: (await http.getJson<CdnjsResponse>(url)).body };
 }
 
-export async function getReleases({
-  lookupName,
-}: GetReleasesConfig): Promise<ReleaseResult | null> {
-  // Each library contains multiple assets, so we cache at the library level instead of per-asset
-  const library = lookupName.split('/')[0];
-  try {
-    const { assets, homepage, repository } = await cacheAble({
-      id,
-      lookup: library,
-      cb: downloadLibrary,
-    });
-    if (!assets) {
-      return null;
-    }
-    const assetName = lookupName.replace(`${library}/`, '');
-    const releases = assets
-      .filter(({ files }) => files.includes(assetName))
-      .map(({ version, sri }) => ({ version, newDigest: sri[assetName] }));
+export class CdnJs extends Datasource {
+  readonly id = 'cdnjs';
 
-    const result: ReleaseResult = { releases };
+  // eslint-disable-next-line class-methods-use-this
+  async getReleases({
+    lookupName,
+  }: GetReleasesConfig): Promise<ReleaseResult | null> {
+    const library = lookupName.split('/')[0];
+    try {
+      const { assets, homepage, repository } = await cacheAble({
+        id,
+        lookup: library,
+        cb: downloadLibrary,
+      });
+      if (!assets) {
+        return null;
+      }
+      const assetName = lookupName.replace(`${library}/`, '');
+      const releases = assets
+        .filter(({ files }) => files.includes(assetName))
+        .map(({ version, sri }) => ({ version, newDigest: sri[assetName] }));
 
-    if (homepage) {
-      result.homepage = homepage;
-    }
-    if (repository?.url) {
-      result.sourceUrl = repository.url;
-    }
-    return result;
-  } catch (err) {
-    if (err.statusCode !== 404) {
-      throw new ExternalHostError(err);
+      const result: ReleaseResult = { releases };
+
+      if (homepage) {
+        result.homepage = homepage;
+      }
+      if (repository?.url) {
+        result.sourceUrl = repository.url;
+      }
+      return result;
+    } catch (err) {
+      if (err.statusCode !== 404) {
+        throw new ExternalHostError(err);
+      }
+      throw err;
     }
-    throw err;
   }
 }

lib/datasource/datasource.ts

@@ -0,0 +1,9 @@
+import { GetReleasesConfig, ReleaseResult } from './common';
+
+export abstract class Datasource {
+  public abstract id: string;
+
+  abstract getReleases(
+    getReleasesConfig: GetReleasesConfig
+  ): Promise<ReleaseResult | null>;
+}

lib/datasource/index.spec.ts

@@ -30,7 +30,7 @@ describe('datasource/index', () => {
     expect(datasource.getDatasources()).toBeDefined();
     expect(datasource.getDatasourceList()).toBeDefined();
   });
-  it('validates dataource', async () => {
+  it('validates datsource', async () => {
     function validateDatasource(
       module: datasource.Datasource,
       name: string
@@ -43,9 +43,16 @@ describe('datasource/index', () => {
       }
       return true;
     }
+    function filterClassBasedDatasources(name: string): boolean {
+      return !['cdnjs'].includes(name);
+    }
     const dss = datasource.getDatasources();
 
-    const loadedDs = loadModules(__dirname, validateDatasource);
+    const loadedDs = loadModules(
+      __dirname,
+      validateDatasource,
+      filterClassBasedDatasources
+    );
     expect(Array.from(dss.keys())).toEqual(Object.keys(loadedDs));
 
     for (const dsName of dss.keys()) {
@@ -82,6 +89,14 @@ describe('datasource/index', () => {
       })
     ).toBeNull();
   });
+  it('returns class datasource', async () => {
+    expect(
+      await datasource.getPkgReleases({
+        datasource: 'cdnjs',
+        depName: null,
+      })
+    ).toBeNull();
+  });
   it('returns getDigest', async () => {
     expect(
       await datasource.getDigest({

lib/datasource/index.ts

@@ -7,6 +7,7 @@ import * as memCache from '../util/cache/memory';
 import { clone } from '../util/clone';
 import * as allVersioning from '../versioning';
 import datasources from './api.generated';
+import { CdnJs } from './cdnjs';
 import {
   Datasource,
   DigestConfig,
@@ -25,7 +26,11 @@ export const getDatasourceList = (): string[] => Array.from(datasources.keys());
 
 const cacheNamespace = 'datasource-releases';
 
-function load(datasource: string): Promise<Datasource> {
+function getDatasourceFor(datasource: string): Promise<Datasource> {
+  datasources.set(
+    'cdnjs',
+    new Promise<Datasource>((resolve) => resolve(new CdnJs()))
+  );
   return datasources.get(datasource);
 }
 
@@ -166,11 +171,14 @@ async function fetchReleases(
   config: GetReleasesInternalConfig
 ): Promise<ReleaseResult | null> {
   const { datasource: datasourceName } = config;
-  if (!datasourceName || !datasources.has(datasourceName)) {
+  if (
+    !datasourceName ||
+    (await getDatasourceFor(datasourceName)) === undefined
+  ) {
     logger.warn('Unknown datasource: ' + datasourceName);
     return null;
   }
-  const datasource = await load(datasourceName);
+  const datasource = await getDatasourceFor(datasourceName);
   const registryUrls = resolveRegistryUrls(datasource, config.registryUrls);
   let dep: ReleaseResult = null;
   try {
@@ -276,14 +284,14 @@ export async function getPkgReleases(
 }
 
 export async function supportsDigests(config: DigestConfig): Promise<boolean> {
-  return 'getDigest' in (await load(config.datasource));
+  return 'getDigest' in (await getDatasourceFor(config.datasource));
 }
 
 export async function getDigest(
   config: DigestConfig,
   value?: string
 ): Promise<string | null> {
-  const datasource = await load(config.datasource);
+  const datasource = await getDatasourceFor(config.datasource);
   const lookupName = config.lookupName || config.depName;
   const registryUrls = resolveRegistryUrls(datasource, config.registryUrls);
   return datasource.getDigest(
@@ -293,6 +301,6 @@ export async function getDigest(
 }
 
 export async function getDefaultConfig(datasource: string): Promise<object> {
-  const loadedDatasource = await load(datasource);
+  const loadedDatasource = await getDatasourceFor(datasource);
   return loadedDatasource?.defaultConfig || {};
 }

tools/generate-imports.ts

@@ -9,12 +9,13 @@ if (!fs.existsSync('lib')) {
   shell.exit(0);
 }
 
-function findModules(dirname: string): string[] {
+function findModules(dirname: string, excludes: string[] = []): string[] {
   return fs
     .readdirSync(dirname, { withFileTypes: true })
     .filter((dirent) => dirent.isDirectory())
     .map((dirent) => dirent.name)
     .filter((name) => !name.startsWith('__'))
+    .filter((name) => !excludes.includes(name))
     .sort();
 }
 async function updateFile(file: string, code: string): Promise<void> {
@@ -74,7 +75,8 @@ import { Datasource } from './common';
 const api = new Map<string, Promise<Datasource>>();
 export default api;
 `;
-    for (const ds of findModules('lib/datasource')) {
+    const datasourceExcludes = ['cdnjs'];
+    for (const ds of findModules('lib/datasource', datasourceExcludes)) {
       code += `api.set('${ds}', import('./${ds}'));\n`;
     }
     await updateFile('lib/datasource/api.generated.ts', code);