diff --git a/lib/utils.js b/lib/utils.js index 30f9cd3c2..8b2709368 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -416,20 +416,18 @@ function joinExpressions(expressions) { return SPDX.normalize(joinedExpressionString) } -function normalizeLicenseExpression(licenseExpression, logger) { - if (!licenseExpression) return null - - const licenseVisitor = rawLicenseExpression => { - const mappedLicenseExpression = scancodeMap.get(rawLicenseExpression) - const licenseExpression = mappedLicenseExpression ? mappedLicenseExpression : rawLicenseExpression - - return SPDX.normalizeSingle(licenseExpression) - } - - const parsed = SPDX.parse(licenseExpression, licenseVisitor) +function normalizeLicenseExpression( + rawLicenseExpression, + logger, + licenseRefLookup = token => token && scancodeMap.get(token) +) { + if (!rawLicenseExpression) return null + + const licenseVisitor = licenseExpression => + scancodeMap.get(licenseExpression) || SPDX.normalizeSingle(licenseExpression) + const parsed = SPDX.parse(rawLicenseExpression, licenseVisitor, licenseRefLookup) const result = SPDX.stringify(parsed) - - if (result === 'NOASSERTION') logger.info(`ScanCode NOASSERTION from ${licenseExpression}`) + if (result === 'NOASSERTION') logger.info(`ScanCode NOASSERTION from ${rawLicenseExpression}`) return result } diff --git a/package-lock.json b/package-lock.json index eb078e18f..0a90fc9a1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -55,7 +55,7 @@ "semver": "7.6.0", "serialize-error": "^2.1.0", "spdx-expression-parse": "github:clearlydefined/spdx-expression-parse.js#fork", - "spdx-license-list": "^6.6.0", + "spdx-license-list": "^6.9.0", "swagger-ui-express": "^4.0.1", "throat": "^4.1.0", "tiny-attribution-generator": "0.1.2", @@ -1624,6 +1624,7 @@ "node_modules/@clearlydefined/spdx": { "version": "0.1.9", "resolved": "git+ssh://git@github.com/clearlydefined/spdx.git#93916093259bc8593400948b679b0dc32a5a12dd", + "license": "MIT", "dependencies": { "spdx-expression-parse": "github:clearlydefined/spdx-expression-parse.js#fork", "spdx-license-ids": "^3.0.20", @@ -10101,6 +10102,7 @@ "node_modules/spdx-expression-parse": { "version": "3.0.0", "resolved": "git+ssh://git@github.com/clearlydefined/spdx-expression-parse.js.git#86fda6a3f84bb9da35d3536822804ac3fd1e587d", + "license": "MIT", "dependencies": { "spdx-exceptions": "^2.1.0", "spdx-license-ids": "^3.0.13" diff --git a/package.json b/package.json index 27353c6e9..b2ea75b72 100644 --- a/package.json +++ b/package.json @@ -66,7 +66,7 @@ "semver": "7.6.0", "serialize-error": "^2.1.0", "spdx-expression-parse": "github:clearlydefined/spdx-expression-parse.js#fork", - "spdx-license-list": "^6.6.0", + "spdx-license-list": "^6.9.0", "swagger-ui-express": "^4.0.1", "throat": "^4.1.0", "tiny-attribution-generator": "0.1.2", diff --git a/providers/summary/scancode/legacy-summarizer.js b/providers/summary/scancode/legacy-summarizer.js index a0e74c481..82322c31f 100644 --- a/providers/summary/scancode/legacy-summarizer.js +++ b/providers/summary/scancode/legacy-summarizer.js @@ -83,7 +83,7 @@ class ScanCodeLegacySummarizer { _readLicenseExpressionFromSummary(harvested) { const licenseExpression = get(harvested, 'content.summary.packages[0].license_expression') - const result = licenseExpression && normalizeLicenseExpression(licenseExpression, this.logger) + const result = licenseExpression && normalizeLicenseExpression(licenseExpression, this.logger, null) return result?.includes('NOASSERTION') ? null : result } @@ -196,7 +196,7 @@ class ScanCodeLegacySummarizer { _createExpressionFromLicense(license) { const rule = license.matched_rule if (!rule || !rule.license_expression) return SPDX.normalize(license.spdx_license_key) - return normalizeLicenseExpression(rule.license_expression, this.logger) + return normalizeLicenseExpression(rule.license_expression, this.logger, null) } } diff --git a/test/lib/util.js b/test/lib/util.js index f4bb1d3b7..12f62b58e 100644 --- a/test/lib/util.js +++ b/test/lib/util.js @@ -880,3 +880,57 @@ describe('Utils buildSourceUrl', () => { expect(result).to.eq('https://pypi.org/project/zuul/3.3.0/') }) }) + +describe('normalizeLicenseExpression', () => { + it('should normalize license', () => { + const expression = 'MIT AND GPL-3.0' + const result = utils.normalizeLicenseExpression(expression) + expect(result).to.eq('MIT AND GPL-3.0') + }) + it('should normalize license to SPDX equivalent', () => { + /* + NOTE: If this fails in tests for generated scancode map workflow PR, it is incorrect if it is expecting a LicenseRef. + There is an SPDX valid license which does not require a LicenseRef meaning this test is correct as is. + */ + const expression = 'net-snmp' + const result = utils.normalizeLicenseExpression(expression) + expect(result).to.eq('Net-SNMP') + }) + it('should normalize single licenseRef', () => { + const expression = 'afpl-9.0' + const result = utils.normalizeLicenseExpression(expression) + expect(result).to.eq('LicenseRef-scancode-afpl-9.0') + }) + it('should normalize license and licenseRef', () => { + const expression = 'afl-1.1 AND afpl-9.0' + const result = utils.normalizeLicenseExpression(expression) + expect(result).to.eq('AFL-1.1 AND LicenseRef-scancode-afpl-9.0') + }) + it('should normalize licenseRef and license', () => { + const expression = 'afpl-9.0 AND MIT' + const result = utils.normalizeLicenseExpression(expression) + expect(result).to.eq('LicenseRef-scancode-afpl-9.0 AND MIT') + }) + it('should normalize licenseRef and licenseRef', () => { + const expression = 'afpl-9.0 AND activestate-community' + const result = utils.normalizeLicenseExpression(expression) + expect(result).to.eq('LicenseRef-scancode-afpl-9.0 AND LicenseRef-scancode-activestate-community') + }) + it('should normalize licenseRef and licenseRef or licenseRef', () => { + const expression = 'afpl-9.0 AND activestate-community OR ac3filter' + const result = utils.normalizeLicenseExpression(expression) + expect(result).to.eq( + 'LicenseRef-scancode-afpl-9.0 AND LicenseRef-scancode-activestate-community OR LicenseRef-scancode-ac3filter' + ) + }) + it('should normalize INVALID to NOASSERTION', () => { + const mockLogger = { + info: message => { + console.log(message) + } + } + const expression = 'INVALID' + const result = utils.normalizeLicenseExpression(expression, mockLogger) + expect(result).to.eq('NOASSERTION') + }) +}) diff --git a/test/providers/summary/scancode/new-summarizer.js b/test/providers/summary/scancode/new-summarizer.js index a2ac10b60..273039207 100644 --- a/test/providers/summary/scancode/new-summarizer.js +++ b/test/providers/summary/scancode/new-summarizer.js @@ -58,7 +58,7 @@ describe('ScanCodeNewSummarizer basic compatability', () => { const coordinates = { type: 'pypi', provider: 'pypi' } const harvestData = getHarvestData(scancodeVersion, 'pypi-complex-declared-license') const result = summarizer.summarize(coordinates, harvestData) - assert.equal(result.licensed.declared, 'HPND') + assert.equal(result.licensed.declared, 'LicenseRef-scancode-secret-labs-2011') } })