Merge branch 'clearlydefined:master' into yk/sourcearchive-declared-l…

…icense

.github/workflows/build-and-deploy-dev.yml

@@ -23,7 +23,7 @@ jobs:
   build-and-deploy:
     name: Build and Deploy
     needs: upload-package-lock-json
-    uses: clearlydefined/operations/.github/workflows/app-build-and-deploy.yml@v2.0.0
+    uses: clearlydefined/operations/.github/workflows/app-build-and-deploy.yml@v3.1.2
     secrets: 
       AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }}
       AZURE_WEBAPP_PUBLISH_PROFILE: ${{ secrets.AZURE_WEBAPP_PUBLISH_PROFILE_DEV }}

.github/workflows/build-and-deploy-prod.yml

@@ -7,8 +7,22 @@ on:
     types: [published]
 
 jobs:
+  upload-package-lock-json:
+    name: Upload package-lock.json from this repo
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4.1.1
+
+      - name: Upload package-lock.json
+        uses: actions/upload-artifact@v4
+        with:
+          name: package-lock.json
+          path: package-lock.json
+
   build-and-deploy-prod:
-    uses: clearlydefined/operations/.github/workflows/app-build-and-deploy.yml@v1.1.0
+    needs: upload-package-lock-json
+    uses: clearlydefined/operations/.github/workflows/app-build-and-deploy.yml@v3.1.2
     secrets: 
       AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }}
       AZURE_WEBAPP_PUBLISH_PROFILE: ${{ secrets.AZURE_WEBAPP_PUBLISH_PROFILE_PROD }}

.github/workflows/test.yml

@@ -1,12 +1,14 @@
 name: Run Docker build and tests
 
 on:
+  workflow_dispatch:
   push:
     branches:
       - master
   pull_request:
     branches:
       - master
+      - prod
 
 permissions:
   contents: read

.npmrc

@@ -1,2 +1,4 @@
 unsafe-perm = true
 save-exact = true
+
+@clearlydefined:registry=https://npm.pkg.github.com

DevDockerfile

@@ -15,8 +15,12 @@ ENV APPDIR=/opt/service
 # COPY init_container.sh /bin/
 # RUN chmod 755 /bin/init_container.sh
 # CMD ["/bin/init_container.sh"]
-ARG BUILD_NUMBER=0
-ENV BUILD_NUMBER=$BUILD_NUMBER
+
+# Set environment variables from build arguments
+ARG APP_VERSION="UNKNOWN"
+ENV APP_VERSION=$APP_VERSION
+ARG BUILD_SHA="UNKNOWN"
+ENV BUILD_SHA=$BUILD_SHA
 
 COPY patches /tmp/patches
 COPY .npmrc package*.json /tmp/

Dockerfile

@@ -15,8 +15,12 @@ ENV APPDIR=/opt/service
 # COPY init_container.sh /bin/
 # RUN chmod 755 /bin/init_container.sh
 # CMD ["/bin/init_container.sh"]
-ARG BUILD_NUMBER=0
-ENV BUILD_NUMBER=$BUILD_NUMBER
+
+# Set environment variables from build arguments
+ARG APP_VERSION="UNKNOWN"
+ENV APP_VERSION=$APP_VERSION
+ARG BUILD_SHA="UNKNOWN"
+ENV BUILD_SHA=$BUILD_SHA
 
 COPY patches /tmp/patches
 COPY .npmrc package*.json /tmp/

app.js

@@ -225,7 +225,7 @@ function createApp(config) {
         // Bit of trick for local hosting. Preload search if using an in-memory search service
         // Commenting out because I believe this is broken
         // if (searchService.constructor.name === 'MemorySearch') await definitionService.reload('definitions')
-        logger.info('Service initialized', { buildNumber: process.env.BUILD_NUMBER })
+        logger.info('Service initialized', { appVersion: process.env.APP_VERSION })
 
         // kick off the queue processors
         require('./providers/curation/process')(curationQueue, curationService, logger)

business/definitionService.js

@@ -70,6 +70,7 @@ class DefinitionService {
     const existing = await this._cacheExistingAside(coordinates, force)
     let result
     if (get(existing, '_meta.schemaVersion') === currentSchema) {
+      // Log line used for /status page insights
       this.logger.info('computed definition available', { coordinates: coordinates.toString() })
       result = existing
     } else result = force ? await this.computeAndStore(coordinates) : await this.computeStoreAndCurate(coordinates)
@@ -83,10 +84,10 @@ class DefinitionService {
    */
   async getStored(coordinates) {
     const cacheKey = this._getCacheKey(coordinates)
-    this.logger.info('1:Redis:start', { ts: new Date().toISOString(), coordinates: coordinates.toString() })
+    this.logger.debug('1:Redis:start', { ts: new Date().toISOString(), coordinates: coordinates.toString() })
     const cached = await this.cache.get(cacheKey)
     if (cached) return cached
-    this.logger.info('2:blob+mongoDB:start', { ts: new Date().toISOString(), coordinates: coordinates.toString() })
+    this.logger.debug('2:blob+mongoDB:start', { ts: new Date().toISOString(), coordinates: coordinates.toString() })
     const stored = await this.definitionStore.get(coordinates)
     if (stored) this._setDefinitionInCache(cacheKey, stored)
     return stored
@@ -100,7 +101,7 @@ class DefinitionService {
   async _setDefinitionInCache(cacheKey, itemToStore) {
     // 1000 is a magic number here -- we don't want to cache very large definitions, as it can impact redis ops
     if (itemToStore.files && itemToStore.files.length > 1000) {
-      this.logger.info('Skipping caching for key', { coordinates: itemToStore.coordinates.toString() })
+      this.logger.debug('Skipping caching for key', { coordinates: itemToStore.coordinates.toString() })
       return
     }
 
@@ -131,9 +132,9 @@ class DefinitionService {
     const result = {}
     const promises = coordinatesList.map(
       throat(10, async coordinates => {
-        this.logger.info(`1:1:notice_generate:get_single_start:${coordinates}`, { ts: new Date().toISOString() })
+        this.logger.debug(`1:1:notice_generate:get_single_start:${coordinates}`, { ts: new Date().toISOString() })
         const definition = await this.get(coordinates, null, force, expand)
-        this.logger.info(`1:1:notice_generate:get_single_end:${coordinates}`, { ts: new Date().toISOString() })
+        this.logger.debug(`1:1:notice_generate:get_single_end:${coordinates}`, { ts: new Date().toISOString() })
         if (!definition) return
         const key = definition.coordinates.toString()
         result[key] = definition
@@ -217,7 +218,7 @@ class DefinitionService {
 
   async computeStoreAndCurate(coordinates) {
     // one coordinate a time through this method so no duplicate auto curation will be created.
-    this.logger.info('3:memory_lock:start', { ts: new Date().toISOString(), coordinates: coordinates.toString() })
+    this.logger.debug('3:memory_lock:start', { ts: new Date().toISOString(), coordinates: coordinates.toString() })
     while (computeLock.get(coordinates.toString())) await new Promise(resolve => setTimeout(resolve, 500))
     try {
       computeLock.set(coordinates.toString(), true)
@@ -245,20 +246,22 @@ class DefinitionService {
     // Note that curation is a tool so no tools really means there the definition is effectively empty.
     const tools = get(definition, 'described.tools')
     if (!tools || tools.length === 0) {
+      // Log line used for /status page insights
       this.logger.info('definition not available', { coordinates: coordinates.toString() })
       this._harvest(coordinates) // fire and forget
       return definition
     }
+    // Log line used for /status page insights
     this.logger.info('recomputed definition available', { coordinates: coordinates.toString() })
     await this._store(definition)
     return definition
   }
 
   async _harvest(coordinates) {
     try {
-      this.logger.info('trigger_harvest:start', { ts: new Date().toISOString(), coordinates: coordinates.toString() })
+      this.logger.debug('trigger_harvest:start', { ts: new Date().toISOString(), coordinates: coordinates.toString() })
       await this.harvestService.harvest({ tool: 'component', coordinates }, true)
-      this.logger.info('trigger_harvest:end', { ts: new Date().toISOString(), coordinates: coordinates.toString() })
+      this.logger.debug('trigger_harvest:end', { ts: new Date().toISOString(), coordinates: coordinates.toString() })
     } catch (error) {
       this.logger.info('failed to harvest from definition service', {
         crawlerError: error,
@@ -282,26 +285,35 @@ class DefinitionService {
    * @returns {Definition} The fully rendered definition
    */
   async compute(coordinates, curationSpec) {
-    this.logger.info('4:compute:blob:start', { ts: new Date().toISOString(), coordinates: coordinates.toString() })
+    this.logger.debug('4:compute:blob:start', { ts: new Date().toISOString(), coordinates: coordinates.toString() })
     const raw = await this.harvestStore.getAll(coordinates)
-    this.logger.info('4:compute:blob:end', { ts: new Date().toISOString(), coordinates: coordinates.toString() })
+    this.logger.debug('4:compute:blob:end', { ts: new Date().toISOString(), coordinates: coordinates.toString() })
     coordinates = this._getCasedCoordinates(raw, coordinates)
-    this.logger.info('5:compute:summarize:start', { ts: new Date().toISOString(), coordinates: coordinates.toString() })
+    this.logger.debug('5:compute:summarize:start', {
+      ts: new Date().toISOString(),
+      coordinates: coordinates.toString()
+    })
     const summaries = await this.summaryService.summarizeAll(coordinates, raw)
-    this.logger.info('6:compute:aggregate:start', { ts: new Date().toISOString(), coordinates: coordinates.toString() })
+    this.logger.debug('6:compute:aggregate:start', {
+      ts: new Date().toISOString(),
+      coordinates: coordinates.toString()
+    })
     const aggregatedDefinition = (await this.aggregationService.process(summaries, coordinates)) || {}
-    this.logger.info('6:compute:aggregate:end', { ts: new Date().toISOString(), coordinates: coordinates.toString() })
+    this.logger.debug('6:compute:aggregate:end', { ts: new Date().toISOString(), coordinates: coordinates.toString() })
     aggregatedDefinition.coordinates = coordinates
     this._ensureToolScores(coordinates, aggregatedDefinition)
     const definition = await this.curationService.apply(coordinates, curationSpec, aggregatedDefinition)
-    this.logger.info('9:compute:calculate:start', { ts: new Date().toISOString(), coordinates: coordinates.toString() })
+    this.logger.debug('9:compute:calculate:start', {
+      ts: new Date().toISOString(),
+      coordinates: coordinates.toString()
+    })
     this._finalizeDefinition(coordinates, definition)
     this._ensureCuratedScores(definition)
     this._ensureFinalScores(definition)
     // protect against any element of the compute producing an invalid definition
     this._ensureNoNulls(definition)
     this._validate(definition)
-    this.logger.info('9:compute:calculate:end', { ts: new Date().toISOString(), coordinates: coordinates.toString() })
+    this.logger.debug('9:compute:calculate:end', { ts: new Date().toISOString(), coordinates: coordinates.toString() })
     return definition
   }
 

lib/curation.js

@@ -77,8 +77,10 @@ class Curation {
 
     sourceLicenseList.forEach(({ source, license }) => {
       const parsed = SPDX.normalize(license)
-      if (parsed !== license || parsed === 'NOASSERTION') {
+      if (!parsed || parsed.includes('NOASSERTION')) {
         errors.push(`${source} with value "${license}" is not SPDX compliant`)
+      } else if (parsed !== license) {
+        errors.push(`${source} with value "${license}" is not normalized. Suggest using "${parsed}"`)
       }
     })
 

lib/scancodeMap.js

@@ -585,7 +585,8 @@ module.exports = new Map([
   ['dmalloc', 'LicenseRef-scancode-dmalloc'],
   ['dmtf-2017', 'LicenseRef-scancode-dmtf-2017'],
   ['do-no-harm-0.1', 'LicenseRef-scancode-do-no-harm-0.1'],
-  ['docbook', 'LicenseRef-scancode-docbook'],
+  ['docbook', 'DocBook-XML'],
+  ['docbook-schema', 'DocBook-Schema'],
   ['dom4j', 'Plexus'],
   ['dos32a-extender', 'LicenseRef-scancode-dos32a-extender'],
   ['dotseqn', 'Dotseqn'],
@@ -661,6 +662,7 @@ module.exports = new Map([
   ['epl-2.0', 'EPL-2.0'],
   ['epo-osl-2005.1', 'LicenseRef-scancode-epo-osl-2005.1'],
   ['eric-glass', 'LicenseRef-scancode-eric-glass'],
+  ['erlang-otp-linking-exception', 'erlang-otp-linking-exception'],
   ['erlangpl-1.1', 'ErlPL-1.1'],
   ['errbot-exception', 'LicenseRef-scancode-errbot-exception'],
   ['esri', 'LicenseRef-scancode-esri'],
@@ -689,6 +691,8 @@ module.exports = new Map([
   ['fatfs', 'LicenseRef-scancode-fatfs'],
   ['fawkes-runtime-exception', 'Fawkes-Runtime-exception'],
   ['fbm', 'FBM'],
+  ['fcl-1.0-apache-2.0', 'LicenseRef-scancode-fcl-1.0-apache-2.0'],
+  ['fcl-1.0-mit', 'LicenseRef-scancode-fcl-1.0-mit'],
   ['ferguson-twofish', 'Ferguson-Twofish'],
   ['ffsl-1', 'LicenseRef-scancode-ffsl-1'],
   ['fftpack-2004', 'NCL'],
@@ -732,6 +736,8 @@ module.exports = new Map([
   ['fsfap-no-warranty-disclaimer', 'FSFAP-no-warranty-disclaimer'],
   ['fsl-1.0-apache-2.0', 'LicenseRef-scancode-fsl-1.0-apache-2.0'],
   ['fsl-1.0-mit', 'LicenseRef-scancode-fsl-1.0-mit'],
+  ['fsl-1.1-apache-2.0', 'LicenseRef-scancode-fsl-1.1-apache-2.0'],
+  ['fsl-1.1-mit', 'LicenseRef-scancode-fsl-1.1-mit'],
   ['ftdi', 'LicenseRef-scancode-ftdi'],
   ['ftpbean', 'LicenseRef-scancode-ftpbean'],
   ['fujion-exception-to-apache-2.0', 'LicenseRef-scancode-fujion-exception-to-apache-2.0'],
@@ -750,6 +756,7 @@ module.exports = new Map([
   ['gco-v3.0', 'LicenseRef-scancode-gco-v3.0'],
   ['gcr-docs', 'GCR-docs'],
   ['gdcl', 'LicenseRef-scancode-gdcl'],
+  ['generic-amiwm', 'LicenseRef-scancode-generic-amiwm'],
   ['generic-cla', 'LicenseRef-scancode-generic-cla'],
   ['generic-exception', 'LicenseRef-scancode-generic-exception'],
   ['generic-export-compliance', 'LicenseRef-scancode-generic-export-compliance'],
@@ -934,7 +941,7 @@ module.exports = new Map([
   ['here-proprietary', 'LicenseRef-scancode-here-proprietary'],
   ['hessla', 'LicenseRef-scancode-hessla'],
   ['hfoil-1.0', 'LicenseRef-scancode-hfoil-1.0'],
-  ['hidapi', 'LicenseRef-scancode-hidapi'],
+  ['hidapi', 'HIDAPI'],
   ['hippocratic-1.0', 'LicenseRef-scancode-hippocratic-1.0'],
   ['hippocratic-1.1', 'LicenseRef-scancode-hippocratic-1.1'],
   ['hippocratic-1.2', 'LicenseRef-scancode-hippocratic-1.2'],
@@ -963,6 +970,7 @@ module.exports = new Map([
   ['hpnd-fenneberg-livingston', 'HPND-Fenneberg-Livingston'],
   ['hpnd-inria-imag', 'HPND-INRIA-IMAG'],
   ['hpnd-mit-disclaimer', 'HPND-MIT-disclaimer'],
+  ['hpnd-netrek', 'HPND-Netrek'],
   ['hpnd-pbmplus', 'HPND-Pbmplus'],
   ['hpnd-sell-mit-disclaimer-xserver', 'HPND-sell-MIT-disclaimer-xserver'],
   ['hpnd-sell-regexpr', 'HPND-sell-regexpr'],
@@ -1205,7 +1213,7 @@ module.exports = new Map([
   ['lilo', 'LicenseRef-scancode-lilo'],
   ['linking-exception-2.0-plus', 'LicenseRef-scancode-linking-exception-2.0-plus'],
   ['linking-exception-2.1-plus', 'LicenseRef-scancode-linking-exception-2.1-plus'],
-  ['linking-exception-agpl-3.0', 'LicenseRef-scancode-linking-exception-agpl-3.0'],
+  ['linking-exception-agpl-3.0', 'romic-exception'],
   ['linking-exception-lgpl-2.0-plus', 'LicenseRef-scancode-linking-exception-lgpl-2.0plus'],
   ['linking-exception-lgpl-3.0', 'null'],
   ['linotype-eula', 'LicenseRef-scancode-linotype-eula'],
@@ -1219,6 +1227,7 @@ module.exports = new Map([
   ['linuxbios', 'LicenseRef-scancode-linuxbios'],
   ['linuxhowtos', 'LicenseRef-scancode-linuxhowtos'],
   ['llama-2-license-2023', 'LicenseRef-scancode-llama-2-license-2023'],
+  ['llama-3.1-license-2024', 'LicenseRef-scancode-llama-3.1-license-2024'],
   ['llama-license-2023', 'LicenseRef-scancode-llama-license-2023'],
   ['llgpl', 'LLGPL'],
   ['llnl', 'LicenseRef-scancode-llnl'],
@@ -1268,6 +1277,7 @@ module.exports = new Map([
   ['matthew-kwan', 'LicenseRef-scancode-matthew-kwan'],
   ['matthew-welch-font-license', 'LicenseRef-scancode-matthew-welch-font-license'],
   ['mattkruse', 'LicenseRef-scancode-mattkruse'],
+  ['max-mojo-community-20240828', 'LicenseRef-scancode-max-mojo-community-20240828'],
   ['maxmind-geolite2-eula-2019', 'LicenseRef-scancode-maxmind-geolite2-eula-2019'],
   ['maxmind-odl', 'LicenseRef-scancode-maxmind-odl'],
   ['mcafee-tou', 'LicenseRef-scancode-mcafee-tou'],
@@ -1411,6 +1421,7 @@ module.exports = new Map([
   ['ms-remote-ndis-usb-kit', 'LicenseRef-scancode-ms-remote-ndis-usb-kit'],
   ['ms-research-shared-source', 'LicenseRef-scancode-ms-research-shared-source'],
   ['ms-rl', 'MS-RL'],
+  ['ms-rndis', 'LicenseRef-scancode-ms-rndis'],
   ['ms-rsl', 'LicenseRef-scancode-ms-rsl'],
   ['ms-silverlight-3', 'LicenseRef-scancode-ms-silverlight-3'],
   ['ms-specification', 'LicenseRef-scancode-ms-specification'],
@@ -1716,6 +1727,7 @@ module.exports = new Map([
   ['otn-early-adopter-2018', 'LicenseRef-scancode-otn-early-adopter-2018'],
   ['otn-early-adopter-development', 'LicenseRef-scancode-otn-early-adopter-development'],
   ['otn-standard-2014-09', 'LicenseRef-scancode-otn-standard-2014-09'],
+  ['otnla-2016-11-30', 'LicenseRef-scancode-otnla-2016-11-30'],
   ['owal-1.0', 'LicenseRef-scancode-owal-1.0'],
   ['owf-cla-1.0-copyright', 'LicenseRef-scancode-owf-cla-1.0-copyright'],
   ['owf-cla-1.0-copyright-patent', 'LicenseRef-scancode-owf-cla-1.0-copyright-patent'],
@@ -1889,6 +1901,7 @@ module.exports = new Map([
   ['rsalv2', 'LicenseRef-scancode-rsalv2'],
   ['rtools-util', 'LicenseRef-scancode-rtools-util'],
   ['ruby', 'Ruby'],
+  ['ruby-pty', 'Ruby-pty'],
   ['rubyencoder-commercial', 'LicenseRef-scancode-rubyencoder-commercial'],
   ['rubyencoder-loader', 'LicenseRef-scancode-rubyencoder-loader'],
   ['rute', 'LicenseRef-scancode-rute'],
@@ -2158,7 +2171,7 @@ module.exports = new Map([
   ['u-boot-exception-2.0', 'u-boot-exception-2.0'],
   ['ubc', 'LicenseRef-scancode-ubc'],
   ['ubdl', 'UBDL-exception'],
-  ['ubuntu-font-1.0', 'LicenseRef-scancode-ubuntu-font-1.0'],
+  ['ubuntu-font-1.0', 'Ubuntu-font-1.0'],
   ['ucar', 'UCAR'],
   ['ucl-1.0', 'UCL-1.0'],
   ['ugui', 'LicenseRef-scancode-ugui'],
@@ -2295,6 +2308,7 @@ module.exports = new Map([
   ['x11-realmode', 'LicenseRef-scancode-x11-realmode'],
   ['x11-sg', 'LicenseRef-scancode-x11-sg'],
   ['x11-stanford', 'LicenseRef-scancode-x11-stanford'],
+  ['x11-swapped', 'X11-swapped'],
   ['x11-tektronix', 'LicenseRef-scancode-x11-tektronix'],
   ['x11-tiff', 'libtiff'],
   ['x11-x11r5', 'LicenseRef-scancode-x11-x11r5'],

lib/utils.js

@@ -416,20 +416,18 @@ function joinExpressions(expressions) {
   return SPDX.normalize(joinedExpressionString)
 }
 
-function normalizeLicenseExpression(licenseExpression, logger) {
-  if (!licenseExpression) return null
-
-  const licenseVisitor = rawLicenseExpression => {
-    const mappedLicenseExpression = scancodeMap.get(rawLicenseExpression)
-    const licenseExpression = mappedLicenseExpression ? mappedLicenseExpression : rawLicenseExpression
-
-    return SPDX.normalizeSingle(licenseExpression)
-  }
-
-  const parsed = SPDX.parse(licenseExpression, licenseVisitor)
+function normalizeLicenseExpression(
+  rawLicenseExpression,
+  logger,
+  licenseRefLookup = token => token && scancodeMap.get(token)
+) {
+  if (!rawLicenseExpression) return null
+
+  const licenseVisitor = licenseExpression =>
+    scancodeMap.get(licenseExpression) || SPDX.normalizeSingle(licenseExpression)
+  const parsed = SPDX.parse(rawLicenseExpression, licenseVisitor, licenseRefLookup)
   const result = SPDX.stringify(parsed)
-
-  if (result === 'NOASSERTION') logger.info(`ScanCode NOASSERTION from ${licenseExpression}`)
+  if (result === 'NOASSERTION') logger.info(`ScanCode NOASSERTION from ${rawLicenseExpression}`)
 
   return result
 }