From a290ed047264ac25870c7e77ff2cde5ee981c96b Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Mon, 20 Nov 2023 16:26:38 +0000 Subject: [PATCH] conda service implementation removed extra semicolon added architecture to conda revisions removed base version fix version fetching fixed lint errors update --- README.md | 4 ++ app.js | 1 + business/statsService.js | 2 + lib/licenseMatcher.js | 4 ++ providers/summary/clearlydefined.js | 40 ++++++++++++++++ routes/originConda.js | 72 +++++++++++++++++++++++++++++ schemas/curation-1.0.json | 5 ++ schemas/curations-1.0.json | 5 ++ schemas/definition-1.0.json | 5 ++ 9 files changed, 138 insertions(+) create mode 100644 routes/originConda.js diff --git a/README.md b/README.md index 0f2439663..3807d5ea4 100644 --- a/README.md +++ b/README.md @@ -243,6 +243,7 @@ The format of harvested data is tool-specific. Tool output is stored in the tool - composer - rubygem - deb +- conda ## Provider Registry @@ -258,6 +259,9 @@ The format of harvested data is tool-specific. Tool output is stored in the tool - packagist.org - proxy.golang.org - ftp.debian.org +- repo.anaconda.com/pkgs/main (anaconda-main) +- repo.anaconda.com/pkgs/r (anaconda-r) +- conda.anaconda.org/conda-forge (conda-forge) ## Tool Name Registry diff --git a/app.js b/app.js index 6859b5aa0..5547e90ce 100644 --- a/app.js +++ b/app.js @@ -184,6 +184,7 @@ function createApp(config) { app.use('/', require('./routes/index')) app.use('/origins/github', require('./routes/originGitHub')()) app.use('/origins/crate', require('./routes/originCrate')()) + app.use('/origins/conda', require('./routes/originConda')()) app.use('/origins/pod', require('./routes/originPod')()) app.use('/origins/npm', require('./routes/originNpm')()) app.use('/origins/maven', require('./routes/originMaven')()) diff --git a/business/statsService.js b/business/statsService.js index 8167713a1..507fc16c9 100644 --- a/business/statsService.js +++ b/business/statsService.js @@ -35,6 +35,8 @@ class StatsService { _getStatLookup() { return { total: () => this._getType('total'), + conda: () => this._getType('conda'), + condasource: () => this._getType('condasource'), crate: () => this._getType('crate'), gem: () => this._getType('gem'), git: () => this._getType('git'), diff --git a/lib/licenseMatcher.js b/lib/licenseMatcher.js index a4316068c..e43f542d3 100644 --- a/lib/licenseMatcher.js +++ b/lib/licenseMatcher.js @@ -111,6 +111,10 @@ class HarvestLicenseMatchPolicy { switch (type) { case 'maven': return new BaseHarvestLicenseMatchStrategy('maven', ['manifest.summary.licenses']) + case 'conda': + return new BaseHarvestLicenseMatchStrategy('conda', ['registryData.license']) + case 'condasource': + return new BaseHarvestLicenseMatchStrategy('condasource', ['registryData.license']) case 'crate': return new BaseHarvestLicenseMatchStrategy('crate', ['registryData.license']) case 'pod': diff --git a/providers/summary/clearlydefined.js b/providers/summary/clearlydefined.js index 13278b551..ba1fc6ce2 100644 --- a/providers/summary/clearlydefined.js +++ b/providers/summary/clearlydefined.js @@ -20,6 +20,12 @@ const mavenBasedUrls = { gradleplugin: 'https://plugins.gradle.org/m2' } +const condaChannels = { + 'anaconda-main': 'https://repo.anaconda.com/pkgs/main', + 'anaconda-r': 'https://repo.anaconda.com/pkgs/r', + 'conda-forge': 'https://conda.anaconda.org/conda-forge' +} + class ClearlyDescribedSummarizer { constructor(options) { this.options = options @@ -44,6 +50,12 @@ class ClearlyDescribedSummarizer { case 'npm': this.addNpmData(result, data, coordinates) break + case 'conda': + this.addCondaData(result, data, coordinates) + break + case 'condasource': + this.addCondaSrcData(result, data, coordinates) + break case 'crate': this.addCrateData(result, data, coordinates) break @@ -191,6 +203,34 @@ class ClearlyDescribedSummarizer { if (licenses.length) setIfValue(result, 'licensed.declared', SPDX.normalize(licenses.join(' OR '))) } + addCondaData(result, data, coordinates) { + setIfValue(result, 'described.releaseDate', extractDate(data.releaseDate)) + if (!data.registryData) return + const architecture = coordinates.revision.split('_')[1] + setIfValue(result, 'described.urls.registry', new URL(`${condaChannels[coordinates.provider]}/${architecture}/repodata.json`).href) + const downloadUrl = new URL(`${condaChannels[coordinates.provider]}/${architecture}/${data.registryData.packageFile}`).href + if (data.registryData.channelData.home) { + setIfValue(result, 'described.projectWebsite', data.registryData.channelData.home) + } + setIfValue(result, 'described.urls.download', downloadUrl) + setIfValue(result, 'licensed.declared', SPDX.normalize(data.declaredLicenses)) + } + + addCondaSrcData(result, data, coordinates) { + setIfValue(result, 'described.releaseDate', extractDate(data.releaseDate)) + if (!data.registryData) return + if (data.registryData.channelData.source_url) { + const downloadUrl = new URL(`${data.registryData.channelData.source_url}`).href + setIfValue(result, 'described.urls.download', downloadUrl) + } + setIfValue(result, 'described.urls.registry', new URL(`${condaChannels[coordinates.provider]}/channeldata.json`).href) + if (data.registryData.channelData.home) { + setIfValue(result, 'described.projectWebsite', data.registryData.channelData.home) + } + setIfValue(result, 'licensed.declared', SPDX.normalize(data.declaredLicenses)) + } + + addCrateData(result, data, coordinates) { setIfValue(result, 'described.releaseDate', extractDate(get(data, 'registryData.created_at'))) setIfValue(result, 'described.projectWebsite', get(data, 'manifest.homepage')) diff --git a/routes/originConda.js b/routes/originConda.js new file mode 100644 index 000000000..dfa0e7c90 --- /dev/null +++ b/routes/originConda.js @@ -0,0 +1,72 @@ +// Copyright (c) Microsoft Corporation and others. Licensed under the MIT license. +// SPDX-License-Identifier: MIT + +const asyncMiddleware = require('../middleware/asyncMiddleware') +const router = require('express').Router() +const requestPromise = require('request-promise-native') +const { uniq } = require('lodash') +const condaChannels = { + 'anaconda-main': 'https://repo.anaconda.com/pkgs/main', + 'anaconda-r': 'https://repo.anaconda.com/pkgs/r', + 'conda-forge': 'https://conda.anaconda.org/conda-forge' +} + +router.get( + '/:name/:channel/revisions', + asyncMiddleware(async (request, response) => { + const { name, channel } = request.params + if (!condaChannels[channel]) { + return response.status(404).send([]) + } + const url = `${condaChannels[channel]}/channeldata.json` + const channelData = await requestPromise({ url, method: 'GET', json: true }) + if (channelData.packages[name]) { + let revisions = [] + for (let subdir of channelData.packages[name].subdirs) { + const repoUrl = `${condaChannels[channel]}/${subdir}/repodata.json` + const repoData = await requestPromise({ url: repoUrl, method: 'GET', json: true }) + if (repoData['packages']) { + Object.entries(repoData['packages']) + .forEach(([, packageData]) => { + if (packageData.name == name) { + revisions.push(`${packageData.version}_${subdir}`) + } + }) + } + if (repoData['packages.conda']) { + Object.entries(repoData['packages.conda']) + .forEach(([, packageData]) => { + if (packageData.name == name) { + revisions.push(`${packageData.version}_${subdir}`) + } + }) + } + } + return response.status(200).send(uniq(revisions)) + } else { + return response.status(404).send([]) + } + }) +) + +router.get( + '/:name/:channel', + asyncMiddleware(async (request, response) => { + const { name, channel } = request.params + if (!condaChannels[channel]) { + return response.status(404).send([]) + } + const url = `${condaChannels[channel]}/channeldata.json` + const channelData = await requestPromise({ url, method: 'GET', json: true }) + let matches = Object.entries(channelData.packages).filter(([packageName,]) => packageName.includes(name)).map( + ([packageName,]) => { return { id: packageName } } + ) + return response.status(200).send(matches) + }) +) + +function setup() { + return router +} + +module.exports = setup \ No newline at end of file diff --git a/schemas/curation-1.0.json b/schemas/curation-1.0.json index 2bd32a9b1..4de658485 100644 --- a/schemas/curation-1.0.json +++ b/schemas/curation-1.0.json @@ -24,6 +24,8 @@ "type": "string", "enum": [ "npm", + "conda", + "condasource", "crate", "git", "maven", @@ -45,8 +47,11 @@ "provider": { "type": "string", "enum": [ + "anaconda-main", + "anaconda-r", "npmjs", "cocoapods", + "conda-forge", "cratesio", "github", "gitlab", diff --git a/schemas/curations-1.0.json b/schemas/curations-1.0.json index d520d53f4..df0788cd3 100644 --- a/schemas/curations-1.0.json +++ b/schemas/curations-1.0.json @@ -26,6 +26,8 @@ "type": "string", "enum": [ "npm", + "conda", + "condasource", "crate", "git", "go", @@ -47,8 +49,11 @@ "provider": { "type": "string", "enum": [ + "anaconda-main", + "anaconda-r", "npmjs", "cocoapods", + "conda-forge", "cratesio", "github", "gitlab", diff --git a/schemas/definition-1.0.json b/schemas/definition-1.0.json index f492f2cd6..e6b7b0814 100644 --- a/schemas/definition-1.0.json +++ b/schemas/definition-1.0.json @@ -32,6 +32,8 @@ "type": { "enum": [ "npm", + "conda", + "condasource", "crate", "git", "maven", @@ -48,8 +50,11 @@ }, "provider": { "enum": [ + "anaconda-main", + "anaconda-r", "npmjs", "cocoapods", + "conda-forge", "cratesio", "github", "gitlab",