Merge branch 'master' into qt/fix-pypi-404

docs/rampup.md

@@ -80,7 +80,7 @@ GitHub webhook is set to put message on queue for any change in PR status, this
 ### Database
 What do we put in mongo / Cosmos DB? ( I see config for curations and definitions)
 *	Curations: looks like pr info, coordinates. Guess: used for curation info in relation to a coordinate (in UI)
-*	Definitions: mapping between attachments and definitions, finding the attachment blobs in azure storage.
+*	Definitions: mapping between attachments and definitions, finding the attachment blobs in azure storage
 
 ### Definitions
 business/definitionservice.js: code to compute definition, recomputed after harvest or curation

lib/utils.js

@@ -321,7 +321,8 @@ function isLicenseFile(filePath, coordinates) {
   if (_licenseFileNames.includes(basePath)) return true
   if (!coordinates) return false
   for (const prefix of getLicenseLocations(coordinates) || []) {
-    if (_licenseFileNames.includes(filePath.replace(prefix, ''))) return true
+    const prefixLowered = prefix.toLowerCase()
+    if (_licenseFileNames.includes(filePath.replace(prefixLowered, ''))) return true
   }
   return false
 }
@@ -338,7 +339,13 @@ function isDeclaredLicense(identifier) {
 }
 
 function getLicenseLocations(coordinates) {
-  const map = { npm: ['package/'], maven: ['meta-inf/'], pypi: [`${coordinates.name}-${coordinates.revision}/`], go: [goLicenseLocations(coordinates)] }
+  const map = {
+    npm: ['package/'],
+    maven: ['META-INF/'],
+    pypi: [`${coordinates.name}-${coordinates.revision}/`],
+    go: [goLicenseLocations(coordinates)]
+  }
+  map.sourcearchive = map.maven
   return map[coordinates.type]
 }
 

providers/summary/scancode.js

@@ -5,6 +5,7 @@ const { get, flatten, uniq } = require('lodash')
 const SPDX = require('@clearlydefined/spdx')
 const {
   extractDate,
+  isDeclaredLicense,
   getLicenseLocations,
   isLicenseFile,
   setIfValue,
@@ -32,9 +33,9 @@ class ScanCodeSummarizer {
     if (!scancodeVersion) throw new Error('Not valid ScanCode data')
     const result = {}
     this.addDescribedInfo(result, harvested)
-    let declaredLicense = this._readDeclaredLicense(scancodeVersion, harvested)
-    if (!declaredLicense || declaredLicense === 'NOASSERTION') {
-      declaredLicense = this._getDeclaredLicense(scancodeVersion, harvested, coordinates)
+    let declaredLicense = this._getDeclaredLicenseFromSummary(scancodeVersion, harvested)
+    if (!isDeclaredLicense(declaredLicense)) {
+      declaredLicense = this._getDeclaredLicenseFromFiles(scancodeVersion, harvested, coordinates) || declaredLicense
     }
     setIfValue(result, 'licensed.declared', declaredLicense)
     result.files = this._summarizeFileInfo(harvested.content.files, coordinates)
@@ -46,7 +47,15 @@ class ScanCodeSummarizer {
     if (releaseDate) result.described = { releaseDate: extractDate(releaseDate.trim()) }
   }
 
-  _readDeclaredLicense(scancodeVersion, harvested) {
+  _getDeclaredLicenseFromSummary(scancodeVersion, harvested) {
+    let declaredLicense = this._readDeclaredLicenseFromSummary(scancodeVersion, harvested)
+    if (!isDeclaredLicense(declaredLicense)) {
+      declaredLicense = this._readLicenseExpressionFromSummary(harvested) || declaredLicense
+    }
+    return declaredLicense
+  }
+
+  _readDeclaredLicenseFromSummary(scancodeVersion, harvested) {
     switch (scancodeVersion) {
       case '2.2.1':
       case '2.9.1':
@@ -61,8 +70,11 @@ class ScanCodeSummarizer {
         // Some Maven packages have this value as an object rather than a string
         // Example: for maven/mavencentral/redis.clients/jedis/4.1.1
         // declared_license would be { "name": "MIT", "url": "http://github.com/redis/jedis/raw/master/LICENSE.txt", "comments": null, "distribution": "repo" }'
+        // Some pypi packages have this value as an object with a license field 
+        // Example: for pypi/pypi/abseil/absl-py/0.9.0
+        // declared_license would be { "license": "Apache 2.0", "classifiers": ["License :: OSI Approved :: Apache Software License"] }
         if (typeof declared_license != 'string' && declared_license != undefined) {
-          declared_license = declared_license['name']
+          declared_license = declared_license['name'] || declared_license['license']
         }
 
         return SPDX.normalize(declared_license)
@@ -72,6 +84,12 @@ class ScanCodeSummarizer {
     }
   }
 
+  _readLicenseExpressionFromSummary(harvested) {
+    const licenseExpression = get(harvested, 'content.summary.packages[0].license_expression')
+    const result = licenseExpression && this._normalizeLicenseExpression(licenseExpression)
+    return result?.includes('NOASSERTION') ? null : result
+  }
+
   // find and return the files that should be considered for as a license determinator for this summarization
   _getRootFiles(coordinates, files) {
     const roots = getLicenseLocations(coordinates) || []
@@ -91,7 +109,7 @@ class ScanCodeSummarizer {
     })
   }
 
-  _getDeclaredLicense(scancodeVersion, harvested, coordinates) {
+  _getDeclaredLicenseFromFiles(scancodeVersion, harvested, coordinates) {
     const rootFile = this._getRootFiles(coordinates, harvested.content.files)
     switch (scancodeVersion) {
       case '2.2.1':
@@ -188,9 +206,13 @@ class ScanCodeSummarizer {
   _createExpressionFromLicense(license) {
     const rule = license.matched_rule
     if (!rule || !rule.license_expression) return SPDX.normalize(license.spdx_license_key)
-    const parsed = SPDX.parse(rule.license_expression, key => SPDX.normalizeSingle(scancodeMap.get(key) || key))
+    return this._normalizeLicenseExpression(rule.license_expression)
+  }
+
+  _normalizeLicenseExpression(licenseExpression) {
+    const parsed = SPDX.parse(licenseExpression, (key) => SPDX.normalizeSingle(scancodeMap.get(key) || key))
     const result = SPDX.stringify(parsed)
-    if (result === 'NOASSERTION') this.logger.info(`ScanCode NOASSERTION from ${rule.license_expression}`)
+    if (result === 'NOASSERTION') this.logger.info(`ScanCode NOASSERTION from ${licenseExpression}`)
     return result
   }
 }

test/business/definitionServiceTest.js

@@ -400,7 +400,9 @@ describe('Aggregation service', () => {
     const summaries = summaryService.summarizeAll(coords, raw)
     const { service } = setupAggregatorWithParams(coordSpec, tools)
     const aggregated = service.process(summaries, coords)
-    expect(aggregated.licensed.declared).to.be.equal('LGPL-2.1-only')
+    expect(aggregated.licensed.declared).to.be.ok
+    // package manifest: LGPL-2.0-or-later, license: LGPL-2.1-only
+    expect(aggregated.licensed.declared).to.be.not.equal('NOASSERTION')
   })
 })