From b2c267c1ffbcd8e9e5a89e029ddffee1365d34bc Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Mon, 22 Apr 2024 12:47:40 -0700 Subject: [PATCH 1/4] Refactor definitionServiceTest Separate aggregator related tests to aggregatorTest.js --- test/business/aggregatorTest.js | 191 +++++++++++++++++++++++++ test/business/definitionServiceTest.js | 176 ----------------------- 2 files changed, 191 insertions(+), 176 deletions(-) create mode 100644 test/business/aggregatorTest.js diff --git a/test/business/aggregatorTest.js b/test/business/aggregatorTest.js new file mode 100644 index 000000000..5de3b0f09 --- /dev/null +++ b/test/business/aggregatorTest.js @@ -0,0 +1,191 @@ +// Copyright (c) Microsoft Corporation and others. Licensed under the MIT license. +// SPDX-License-Identifier: MIT + +const AggregatorService = require('../../business/aggregator') +const SummaryService = require('../../business/summarizer') +const EntityCoordinates = require('../../lib/entityCoordinates') +const { setIfValue } = require('../../lib/utils') +const deepEqualInAnyOrder = require('deep-equal-in-any-order') +const chai = require('chai') +chai.use(deepEqualInAnyOrder) +const expect = chai.expect + +describe('Aggregation service', () => { + it('handles no tool data', async () => { + const { service } = setupAggregator() + const aggregated = service.process({}) + expect(aggregated).to.be.null + }) + + it('handles one tool one version data', async () => { + const summaries = { tool2: { '1.0.0': { files: [buildFile('foo.txt', 'MIT')] } } } + const { service } = setupAggregator() + const aggregated = service.process(summaries) + expect(aggregated.files.length).to.eq(1) + }) + + it('handles one tool multiple version data', async () => { + const summaries = { + tool2: { + '1.0.0': { files: [buildFile('foo.txt', 'MIT'), buildFile('bar.txt', 'MIT')] }, + '2.0.0': { files: [buildFile('foo.txt', 'GPL-2.0')] } + } + } + const { service } = setupAggregator() + const aggregated = service.process(summaries) + expect(aggregated.files.length).to.eq(1) + expect(aggregated.files[0].path).to.eq('foo.txt') + expect(aggregated.files[0].license).to.eq('GPL-2.0') + }) + + it('handles multiple tools and one file data', async () => { + const summaries = { + tool2: { + '1.0.0': { files: [buildFile('foo.txt', 'MIT')] }, + '2.0.0': { files: [buildFile('foo.txt', 'GPL-2.0')] } + }, + tool1: { '3.0.0': { files: [buildFile('foo.txt', 'BSD-3-Clause')] } } + } + const { service } = setupAggregator() + const aggregated = service.process(summaries) + expect(aggregated.files.length).to.eq(1) + expect(aggregated.files[0].license).to.equal('BSD-3-Clause AND GPL-2.0') + }) + + it('handles multiple tools and multiple file data with extras ignored', async () => { + const summaries = { + tool2: { + '1.0.0': { files: [buildFile('foo.txt', 'MIT')] }, + '2.0.0': { files: [buildFile('foo.txt', 'GPL-2.0')] } + }, + tool1: { + '3.0.0': { files: [buildFile('foo.txt', 'BSD-3-Clause')] }, + '2.0.0': { files: [buildFile('bar.txt', 'GPL-2.0')] } + } + } + const { service } = setupAggregator() + const aggregated = service.process(summaries) + expect(aggregated.files.length).to.eq(1) + expect(aggregated.files[0].license).to.equal('BSD-3-Clause AND GPL-2.0') + }) + + it('handles multiple tools and multiple file data with extras included', async () => { + const summaries = { + tool2: { + '1.0.0': { files: [buildFile('foo.txt', 'MIT')] }, + '2.0.0': { files: [buildFile('foo.txt', 'GPL-2.0')] } + }, + tool1: { + '3.0.0': { files: [buildFile('foo.txt', 'BSD-3-Clause'), buildFile('bar.txt', 'GPL-2.0')] }, + '2.0.0': { files: [buildFile('bar.txt', 'GPL-2.0')] } + } + } + const { service } = setupAggregator() + const aggregated = service.process(summaries) + expect(aggregated.files.length).to.eq(2) + expect(aggregated.files[0].path).to.eq('foo.txt') + expect(aggregated.files[0].license).to.equal('BSD-3-Clause AND GPL-2.0') + expect(aggregated.files[1].path).to.eq('bar.txt') + expect(aggregated.files[1].license).to.eq('GPL-2.0') + }) + + it('handles Rust crates with license choices', async () => { + const testcases = [ + { + name: 'slog', + version: '2.7.0', + tools: [['clearlydefined', 'licensee', 'scancode']], + // Ideally this would be declared without any parentheses, but currently + // the SPDX normalization adds them. + expected: 'MPL-2.0 OR (MIT OR Apache-2.0)' + }, + { + name: 'quote', + version: '0.6.4', + tools: [['clearlydefined', 'fossology', 'licensee', 'scancode']], + expected: 'MIT OR Apache-2.0' + }, + { + name: 'quote', + version: '1.0.9', + tools: [['clearlydefined', 'licensee', 'scancode']], + expected: 'MIT OR Apache-2.0' + }, + { + name: 'rand', + version: '0.8.2', + tools: [['clearlydefined', 'licensee', 'scancode']], + expected: 'MIT OR Apache-2.0' + }, + { + name: 'regex', + version: '1.5.3', + tools: [['clearlydefined', 'licensee', 'scancode']], + expected: 'MIT OR Apache-2.0' + }, + { + name: 'serde', + version: '1.0.123', + tools: [['clearlydefined', 'licensee', 'scancode']], + expected: 'MIT OR Apache-2.0' + }, + { + name: 'mpmc', + version: '0.1.6', + tools: [['clearlydefined', 'licensee', 'scancode']], + expected: 'BSD-2-Clause-Views' + } + ] + + const summary_options = {} + const summaryService = SummaryService(summary_options) + + for (const testcase of testcases) { + const coordSpec = `crate/cratesio/-/${testcase.name}/${testcase.version}` + const coords = EntityCoordinates.fromString(coordSpec) + const raw = require(`./evidence/crate-${testcase.name}-${testcase.version}.json`) + const tools = testcase.tools + const summaries = summaryService.summarizeAll(coords, raw) + const { service } = setupAggregatorWithParams(coordSpec, tools) + const aggregated = service.process(summaries, coords) + expect(aggregated.licensed.declared, `${testcase.name}-${testcase.version}`).to.eq(testcase.expected) + } + }) + + it('should handle composer/packagist components', () => { + const tools = [['clearlydefined', 'licensee', 'scancode', 'reuse']] + const coordSpec = 'composer/packagist/mmucklo/krumo/0.7.0' + const coords = EntityCoordinates.fromString(coordSpec) + const raw = require(`./evidence/${coordSpec.replace(/\//g, '-')}.json`) + + const summary_options = {} + const summaryService = SummaryService(summary_options) + const summaries = summaryService.summarizeAll(coords, raw) + const { service } = setupAggregatorWithParams(coordSpec, tools) + const aggregated = service.process(summaries, coords) + expect(aggregated.licensed.declared).to.be.ok + // package manifest: LGPL-2.0-or-later, license: LGPL-2.1-only + expect(aggregated.licensed.declared).to.be.not.equal('NOASSERTION') + }) +}) + +function buildFile(path, license, holders) { + const result = { path } + setIfValue(result, 'license', license) + setIfValue(result, 'attributions', holders ? holders.map(entry => `Copyright ${entry}`) : null) + return result +} + +function setupAggregator() { + const coordinates = EntityCoordinates.fromString('npm/npmjs/-/test/1.0') + const config = { precedence: [['tool1', 'tool2', 'tool3']] } + const service = AggregatorService(config) + return { service, coordinates } +} + +function setupAggregatorWithParams(coordSpec, tool_precedence) { + const coordinates = EntityCoordinates.fromString(coordSpec) + const config = { precedence: tool_precedence } + const service = AggregatorService(config) + return { service, coordinates } +} diff --git a/test/business/definitionServiceTest.js b/test/business/definitionServiceTest.js index ab15c7371..d7d8c57b6 100644 --- a/test/business/definitionServiceTest.js +++ b/test/business/definitionServiceTest.js @@ -4,8 +4,6 @@ const sinon = require('sinon') const validator = require('../../schemas/validator') const DefinitionService = require('../../business/definitionService') -const AggregatorService = require('../../business/aggregator') -const SummaryService = require('../../business/summarizer') const EntityCoordinates = require('../../lib/entityCoordinates') const { setIfValue } = require('../../lib/utils') const Curation = require('../../lib/curation') @@ -245,166 +243,6 @@ describe('Definition Service Facet management', () => { }) }) -describe('Aggregation service', () => { - it('handles no tool data', async () => { - const { service } = setupAggregator() - const aggregated = service.process({}) - expect(aggregated).to.be.null - }) - - it('handles one tool one version data', async () => { - const summaries = { tool2: { '1.0.0': { files: [buildFile('foo.txt', 'MIT')] } } } - const { service } = setupAggregator() - const aggregated = service.process(summaries) - expect(aggregated.files.length).to.eq(1) - }) - - it('handles one tool multiple version data', async () => { - const summaries = { - tool2: { - '1.0.0': { files: [buildFile('foo.txt', 'MIT'), buildFile('bar.txt', 'MIT')] }, - '2.0.0': { files: [buildFile('foo.txt', 'GPL-2.0')] } - } - } - const { service } = setupAggregator() - const aggregated = service.process(summaries) - expect(aggregated.files.length).to.eq(1) - expect(aggregated.files[0].path).to.eq('foo.txt') - expect(aggregated.files[0].license).to.eq('GPL-2.0') - }) - - it('handles multiple tools and one file data', async () => { - const summaries = { - tool2: { - '1.0.0': { files: [buildFile('foo.txt', 'MIT')] }, - '2.0.0': { files: [buildFile('foo.txt', 'GPL-2.0')] } - }, - tool1: { '3.0.0': { files: [buildFile('foo.txt', 'BSD-3-Clause')] } } - } - const { service } = setupAggregator() - const aggregated = service.process(summaries) - expect(aggregated.files.length).to.eq(1) - expect(aggregated.files[0].license).to.equal('BSD-3-Clause AND GPL-2.0') - }) - - it('handles multiple tools and multiple file data with extras ignored', async () => { - const summaries = { - tool2: { - '1.0.0': { files: [buildFile('foo.txt', 'MIT')] }, - '2.0.0': { files: [buildFile('foo.txt', 'GPL-2.0')] } - }, - tool1: { - '3.0.0': { files: [buildFile('foo.txt', 'BSD-3-Clause')] }, - '2.0.0': { files: [buildFile('bar.txt', 'GPL-2.0')] } - } - } - const { service } = setupAggregator() - const aggregated = service.process(summaries) - expect(aggregated.files.length).to.eq(1) - expect(aggregated.files[0].license).to.equal('BSD-3-Clause AND GPL-2.0') - }) - - it('handles multiple tools and multiple file data with extras included', async () => { - const summaries = { - tool2: { - '1.0.0': { files: [buildFile('foo.txt', 'MIT')] }, - '2.0.0': { files: [buildFile('foo.txt', 'GPL-2.0')] } - }, - tool1: { - '3.0.0': { files: [buildFile('foo.txt', 'BSD-3-Clause'), buildFile('bar.txt', 'GPL-2.0')] }, - '2.0.0': { files: [buildFile('bar.txt', 'GPL-2.0')] } - } - } - const { service } = setupAggregator() - const aggregated = service.process(summaries) - expect(aggregated.files.length).to.eq(2) - expect(aggregated.files[0].path).to.eq('foo.txt') - expect(aggregated.files[0].license).to.equal('BSD-3-Clause AND GPL-2.0') - expect(aggregated.files[1].path).to.eq('bar.txt') - expect(aggregated.files[1].license).to.eq('GPL-2.0') - }) - - it('handles Rust crates with license choices', async () => { - const testcases = [ - { - name: 'slog', - version: '2.7.0', - tools: [['clearlydefined', 'licensee', 'scancode']], - // Ideally this would be declared without any parentheses, but currently - // the SPDX normalization adds them. - expected: 'MPL-2.0 OR (MIT OR Apache-2.0)' - }, - { - name: 'quote', - version: '0.6.4', - tools: [['clearlydefined', 'fossology', 'licensee', 'scancode']], - expected: 'MIT OR Apache-2.0' - }, - { - name: 'quote', - version: '1.0.9', - tools: [['clearlydefined', 'licensee', 'scancode']], - expected: 'MIT OR Apache-2.0' - }, - { - name: 'rand', - version: '0.8.2', - tools: [['clearlydefined', 'licensee', 'scancode']], - expected: 'MIT OR Apache-2.0' - }, - { - name: 'regex', - version: '1.5.3', - tools: [['clearlydefined', 'licensee', 'scancode']], - expected: 'MIT OR Apache-2.0' - }, - { - name: 'serde', - version: '1.0.123', - tools: [['clearlydefined', 'licensee', 'scancode']], - expected: 'MIT OR Apache-2.0' - }, - { - name: 'mpmc', - version: '0.1.6', - tools: [['clearlydefined', 'licensee', 'scancode']], - expected: 'BSD-2-Clause-Views' - } - ] - - const summary_options = {} - const summaryService = SummaryService(summary_options) - - for (let i = 0; i < testcases.length; i++) { - let testcase = testcases[i] - const coordSpec = `crate/cratesio/-/${testcase.name}/${testcase.version}` - const coords = EntityCoordinates.fromString(coordSpec) - const raw = require(`./evidence/crate-${testcase.name}-${testcase.version}.json`) - const tools = testcase.tools - const summaries = summaryService.summarizeAll(coords, raw) - const { service } = setupAggregatorWithParams(coordSpec, tools) - const aggregated = service.process(summaries, coords) - expect(aggregated.licensed.declared, `${testcase.name}-${testcase.version}`).to.eq(testcase.expected) - } - }) - - it('should handle composer/packagist components', () => { - const tools = [['clearlydefined', 'licensee', 'scancode', 'reuse']] - const coordSpec = 'composer/packagist/mmucklo/krumo/0.7.0' - const coords = EntityCoordinates.fromString(coordSpec) - const raw = require(`./evidence/${coordSpec.replace(/\//g, '-')}.json`) - - const summary_options = {} - const summaryService = SummaryService(summary_options) - const summaries = summaryService.summarizeAll(coords, raw) - const { service } = setupAggregatorWithParams(coordSpec, tools) - const aggregated = service.process(summaries, coords) - expect(aggregated.licensed.declared).to.be.ok - // package manifest: LGPL-2.0-or-later, license: LGPL-2.1-only - expect(aggregated.licensed.declared).to.be.not.equal('NOASSERTION') - }) -}) - function validate(definition) { // Tack on a dummy coordinates to keep the schema happy. Tool summarizations do not have to include coordinates definition.coordinates = { type: 'npm', provider: 'npmjs', namespace: null, name: 'foo', revision: '1.0' } @@ -447,17 +285,3 @@ function setup(definition, coordinateSpec, curation) { const coordinates = EntityCoordinates.fromString(coordinateSpec || 'npm/npmjs/-/test/1.0') return { coordinates, service } } - -function setupAggregator() { - const coordinates = EntityCoordinates.fromString('npm/npmjs/-/test/1.0') - const config = { precedence: [['tool1', 'tool2', 'tool3']] } - const service = AggregatorService(config) - return { service, coordinates } -} - -function setupAggregatorWithParams(coordSpec, tool_precedence) { - const coordinates = EntityCoordinates.fromString(coordSpec) - const config = { precedence: tool_precedence } - const service = AggregatorService(config) - return { service, coordinates } -} From 6a0967bf9b84fcf165c65a351e54903a618b386c Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Tue, 23 Apr 2024 14:13:26 -0700 Subject: [PATCH 2/4] Add unit tests to check for source location --- test/business/definitionServiceTest.js | 56 ++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/test/business/definitionServiceTest.js b/test/business/definitionServiceTest.js index d7d8c57b6..9a4331834 100644 --- a/test/business/definitionServiceTest.js +++ b/test/business/definitionServiceTest.js @@ -113,6 +113,62 @@ describe('Definition Service', () => { expect(result.length).to.eq(3) expect(result.map(x => x.name)).to.have.members(['test0', 'test1', 'testUpperCase']) }) + + describe('Build source location', () => { + const data = new Map([ + [ + 'pypi/pypi/-/platformdirs/4.2.0', + { + type: 'pypi', + provider: 'pypi', + name: 'platformdirs', + revision: '4.2.0', + url: 'https://pypi.org/project/platformdirs/4.2.0/' + } + ], + [ + 'go/golang/rsc.io/quote/v1.3.0', + { + type: 'go', + provider: 'golang', + namespace: 'rsc.io', + name: 'quote', + revision: 'v1.3.0', + url: 'https://pkg.go.dev/rsc.io/quote@v1.3.0' + } + ], + [ + 'git/github/ratatui-org/ratatui/bcf43688ec4a13825307aef88f3cdcd007b32641', + { + type: 'git', + provider: 'github', + namespace: 'ratatui-org', + name: 'ratatui', + revision: 'bcf43688ec4a13825307aef88f3cdcd007b32641', + url: 'https://github.com/ratatui-org/ratatui/tree/bcf43688ec4a13825307aef88f3cdcd007b32641' + } + ], + [ + 'git/gitlab/cznic/sqlite/282bdb12f8ce48a34b4b768863c4e44c310c4bd8', + { + type: 'git', + provider: 'gitlab', + namespace: 'cznic', + name: 'sqlite', + revision: '282bdb12f8ce48a34b4b768863c4e44c310c4bd8', + url: 'https://gitlab.com/cznic/sqlite/-/tree/282bdb12f8ce48a34b4b768863c4e44c310c4bd8' + } + ] + ]) + + data.forEach((expected, coordinatesString) => { + it(`should have source location for ${coordinatesString} package`, async () => { + const { service, coordinates } = setup(createDefinition(null, null, []), coordinatesString) + const definition = await service.compute(coordinates) + expect(definition.described.sourceLocation).to.be.deep.equal(expected) + }) + }) + }) }) describe('Definition Service Facet management', () => { From 918c67d1e27454365077b1e942008c4218d0e875 Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Tue, 23 Apr 2024 14:27:38 -0700 Subject: [PATCH 3/4] Add source location in definition for sourcearchive package Whether packages contain source files depends on the package type, not the package provider. Change the logic to build source location to reflect this. --- business/definitionService.js | 8 ++++---- test/business/definitionServiceTest.js | 11 +++++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/business/definitionService.js b/business/definitionService.js index 7991162f5..f114d2f97 100644 --- a/business/definitionService.js +++ b/business/definitionService.js @@ -561,10 +561,10 @@ class DefinitionService { if (get(definition, 'described.sourceLocation')) return updateSourceLocation(definition.described.sourceLocation) // For source components there may not be an explicit harvested source location (it is self-evident) // Make it explicit in the definition - switch (coordinates.provider) { - case 'golang': - case 'gitlab': - case 'github': + switch (coordinates.type) { + case 'go': + case 'git': + case 'sourcearchive': case 'pypi': { const url = buildSourceUrl(coordinates) if (!url) return diff --git a/test/business/definitionServiceTest.js b/test/business/definitionServiceTest.js index 9a4331834..7facac009 100644 --- a/test/business/definitionServiceTest.js +++ b/test/business/definitionServiceTest.js @@ -158,6 +158,17 @@ describe('Definition Service', () => { revision: '282bdb12f8ce48a34b4b768863c4e44c310c4bd8', url: 'https://gitlab.com/cznic/sqlite/-/tree/282bdb12f8ce48a34b4b768863c4e44c310c4bd8' } + ], + [ + 'sourcearchive/mavencentral/com.azure/azure-storage-blob/12.20.0', + { + type: 'sourcearchive', + provider: 'mavencentral', + namespace: 'com.azure', + name: 'azure-storage-blob', + revision: '12.20.0', + url: 'https://search.maven.org/remotecontent?filepath=com/azure/azure-storage-blob/12.20.0/azure-storage-blob-12.20.0-sources.jar' + } ] ]) From b1da6d70e7098f79dfbe62db4b9bf4b0ccca9a2a Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Mon, 29 Apr 2024 12:04:22 -0700 Subject: [PATCH 4/4] Update aggregatorTest.js --- test/business/aggregatorTest.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/business/aggregatorTest.js b/test/business/aggregatorTest.js index 5de3b0f09..49dbdfbb5 100644 --- a/test/business/aggregatorTest.js +++ b/test/business/aggregatorTest.js @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation and others. Licensed under the MIT license. +// (c) Copyright 2024, SAP SE and ClearlyDefined contributors. Licensed under the MIT license. // SPDX-License-Identifier: MIT const AggregatorService = require('../../business/aggregator')