Add LicenseRef support

lib/utils.js

@@ -427,13 +427,38 @@ function normalizeLicenseExpression(rawLicenseExpression, logger) {
   // therefore use the mapped license expression as an argument if it was found
   const mappedLicenseExpression = scancodeMap.get(rawLicenseExpression)
   const parsed = SPDX.parse(mappedLicenseExpression || rawLicenseExpression || '', licenseVisitor)
-  const result = SPDX.stringify(parsed)
 
+  // normalize the parsed license expression will recursively normalize the parsed license expression
+  const normalizedParsed = _normalizeParsedLicenseExpression(parsed, logger)
+
+  const result = SPDX.stringify(normalizedParsed)
   if (result === 'NOASSERTION') logger.info(`ScanCode NOASSERTION from ${rawLicenseExpression}`)
 
   return result
 }
 
+function _normalizeParsedLicenseExpression(parsedLicenseExpression, logger) {
+  if (parsedLicenseExpression.left) {
+    if (parsedLicenseExpression.left.hasOwnProperty('left')) {
+      parsedLicenseExpression.left = _normalizeParsedLicenseExpression(parsedLicenseExpression.left, logger)
+    } else if (parsedLicenseExpression.left.hasOwnProperty('noassertion')) {
+      const new_left = normalizeLicenseExpression(parsedLicenseExpression.left['noassertion'], logger)
+      if (new_left.toLowerCase() === 'noassertion') parsedLicenseExpression.right = { 'noassertion': new_left }
+      else parsedLicenseExpression.left = { license: new_left }
+    }
+  }
+  if (parsedLicenseExpression.right) {
+    if (parsedLicenseExpression.right.hasOwnProperty('left')) {
+      parsedLicenseExpression.right = _normalizeParsedLicenseExpression(parsedLicenseExpression.right, logger)
+    } else if (parsedLicenseExpression.right.hasOwnProperty('noassertion')) {
+      const new_right = normalizeLicenseExpression(parsedLicenseExpression.right['noassertion'], logger)
+      if (new_right.toLowerCase() === 'noassertion') parsedLicenseExpression.right = { 'noassertion': new_right }
+      else parsedLicenseExpression.right = { license: new_right }
+    }
+  }
+  return parsedLicenseExpression
+}
+
 function _normalizeVersion(version) {
   if (version == '1') return '1.0.0' // version '1' is not semver valid see https://github.com/clearlydefined/crawler/issues/124
   return semver.valid(version) ? version : null

test/lib/util.js

@@ -880,3 +880,45 @@ describe('Utils buildSourceUrl', () => {
     expect(result).to.eq('https://pypi.org/project/zuul/3.3.0/')
   })
 })
+
+describe('normalizeLicenseExpression', () => {
+  it('should normalize license', () => {
+    const expression = 'MIT AND GPL-3.0'
+    const result = utils.normalizeLicenseExpression(expression)
+    expect(result).to.eq('MIT AND GPL-3.0')
+  })
+  it('should normalize single licenseRef', () => {
+    const expression = 'afpl-9.0'
+    const result = utils.normalizeLicenseExpression(expression)
+    expect(result).to.eq('LicenseRef-scancode-afpl-9.0')
+  })
+  it('should normalize license and licenseRef', () => {
+    const expression = 'afl-1.1 AND afpl-9.0'
+    const result = utils.normalizeLicenseExpression(expression)
+    expect(result).to.eq('AFL-1.1 AND LicenseRef-scancode-afpl-9.0')
+  })
+  it('should normalize licenseRef and license', () => {
+    const expression = 'afpl-9.0 AND MIT'
+    const result = utils.normalizeLicenseExpression(expression)
+    expect(result).to.eq('LicenseRef-scancode-afpl-9.0 AND MIT')
+  })
+  it('should normalize licenseRef and licenseRef', () => {
+    const expression = 'afpl-9.0 AND activestate-community'
+    const result = utils.normalizeLicenseExpression(expression)
+    expect(result).to.eq('LicenseRef-scancode-afpl-9.0 AND LicenseRef-scancode-activestate-community')
+  })
+  it('should normalize licenseRef and licenseRef or licenseRef', () => {
+    const expression = 'afpl-9.0 AND activestate-community OR ac3filter'
+    const result = utils.normalizeLicenseExpression(expression)
+    expect(result).to.eq('LicenseRef-scancode-afpl-9.0 AND LicenseRef-scancode-activestate-community OR LicenseRef-scancode-ac3filter')
+  })
+  it('should normalize INVALID to NOASSERTION', () => {
+    const mockLogger = {
+      info: (message) => {
+        console.log(message);
+      }
+    };  const expression = 'INVALID'
+    const result = utils.normalizeLicenseExpression(expression, mockLogger)
+    expect(result).to.eq('NOASSERTION')
+  })
+})