Merge pull request #1056 from lumaxis/updates-new-scancode-version

Add new summarizer for recent ScanCode versions

lib/utils.js

@@ -9,6 +9,7 @@ const { DateTime } = require('luxon')
 const { set, unset, union, sortBy, trim, uniqBy } = require('lodash')
 const extend = require('extend')
 const SPDX = require('@clearlydefined/spdx')
+const scancodeMap = require('./scancodeMap')
 const coordinatesMapper = require('./coordinatesMapper')()
 
 async function toResultCoordinatesFromRequest(request) {
@@ -330,16 +331,17 @@ function updateSourceLocation(spec) {
  * Checks deeper than the root depending on coordinate type
  *
  * @param {string} filePath
- * @param {EntityCoordinates} coordinates - optional to look deeper than the root based on coordinate type
+ * @param {EntityCoordinates} [coordinates] - optional to look deeper than the root based on coordinate type
+ * @param {object} [packages] - optional, to look at package directories
  * @returns {boolean}
  */
-function isLicenseFile(filePath, coordinates) {
+function isLicenseFile(filePath, coordinates, packages) {
   if (!filePath) return false
   filePath = filePath.toLowerCase()
   const basePath = filePath.split('/')[0]
   if (_licenseFileNames.includes(basePath)) return true
   if (!coordinates) return false
-  for (const prefix of getLicenseLocations(coordinates) || []) {
+  for (const prefix of getLicenseLocations(coordinates, packages) || []) {
     const prefixLowered = prefix.toLowerCase()
     if (_licenseFileNames.includes(filePath.replace(prefixLowered, ''))) return true
   }
@@ -357,25 +359,81 @@ function isDeclaredLicense(identifier) {
   return identifier && identifier !== 'NOASSERTION' && identifier !== 'NONE'
 }
 
-function getLicenseLocations(coordinates) {
+function getLicenseLocations(coordinates, packages) {
   const map = {
     npm: ['package/'],
     maven: ['META-INF/'],
     pypi: [`${coordinates.name}-${coordinates.revision}/`],
-    go: [goLicenseLocations(coordinates)]
+    go: [goLicenseLocation(coordinates)],
+    debsrc: packages ? debsrcLicenseLocations(packages) : []
   }
   map.sourcearchive = map.maven
   return map[coordinates.type]
 }
 
-function goLicenseLocations(coordinates) {
+function goLicenseLocation(coordinates) {
   if (coordinates.namespace && coordinates.namespace.toLowerCase().includes('%2f')) {
     return `${deCodeSlashes(coordinates.namespace)}/${coordinates.name}@${coordinates.revision}/`
   } else {
     return `${coordinates.namespace}/${coordinates.name}@${coordinates.revision}/`
   }
 }
 
+function debsrcLicenseLocations(packages) {
+  const licenseLocations = []
+
+  // Split packages of `type: deb` and other packages
+  const [debPackages, otherPackages] = packages.reduce(
+    ([debPackages, otherPackages], pkg) => {
+      if (pkg.type === 'deb') {
+        debPackages.push(pkg)
+      } else {
+        otherPackages.push(pkg)
+      }
+      return [debPackages, otherPackages]
+    },
+    [[], []]
+  )
+
+  // Add default location for debian packages
+  if (debPackages.length) {
+    licenseLocations.push('debian/')
+  }
+
+  // Add license locations based on package name and version for other packages
+  return licenseLocations.concat(
+    otherPackages.map(otherPackage =>
+      otherPackage.version ? `${otherPackage.name}-${otherPackage.version}/` : `${otherPackage.name}/`
+    )
+  )
+}
+
+function joinExpressions(expressions) {
+  if (!expressions) return null
+  const list = setToArray(expressions)
+  if (!list) return null
+  const joinedExpressionString = `(${list.join(') AND (')})`
+  return SPDX.normalize(joinedExpressionString)
+}
+
+function normalizeLicenseExpression(licenseExpression, logger) {
+  if (!licenseExpression) return null
+
+  const licenseVisitor = rawLicenseExpression => {
+    const mappedLicenseExpression = scancodeMap.get(rawLicenseExpression)
+    const licenseExpression = mappedLicenseExpression ? mappedLicenseExpression : rawLicenseExpression
+
+    return SPDX.normalizeSingle(licenseExpression)
+  }
+
+  const parsed = SPDX.parse(licenseExpression, licenseVisitor)
+  const result = SPDX.stringify(parsed)
+
+  if (result === 'NOASSERTION') logger.info(`ScanCode NOASSERTION from ${licenseExpression}`)
+
+  return result
+}
+
 function _normalizeVersion(version) {
   if (version == '1') return '1.0.0' // version '1' is not semver valid see https://github.com/clearlydefined/crawler/issues/124
   return semver.valid(version) ? version : null
@@ -552,6 +610,8 @@ module.exports = {
   addArrayToSet,
   extractLicenseFromLicenseUrl,
   getLicenseLocations,
+  joinExpressions,
+  normalizeLicenseExpression,
   mergeDefinitions,
   buildSourceUrl,
   deCodeSlashes,