Merge branch 'master' into add-licenseref-support

.github/workflows/build-and-deploy-dev.yml

@@ -7,8 +7,23 @@ on:
     branches: [master]
 
 jobs:
+  upload-package-lock-json:
+    name: Upload package-lock.json from this repo
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4.1.1
+
+      - name: Upload package-lock.json
+        uses: actions/upload-artifact@v4
+        with:
+          name: package-lock.json
+          path: package-lock.json
+
   build-and-deploy:
-    uses: clearlydefined/operations/.github/workflows/app-build-and-deploy.yml@v1.1.0
+    name: Build and Deploy
+    needs: upload-package-lock-json
+    uses: clearlydefined/operations/.github/workflows/app-build-and-deploy.yml@v3.0.0
     secrets: 
       AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }}
       AZURE_WEBAPP_PUBLISH_PROFILE: ${{ secrets.AZURE_WEBAPP_PUBLISH_PROFILE_DEV }}

DevDockerfile

@@ -15,8 +15,12 @@ ENV APPDIR=/opt/service
 # COPY init_container.sh /bin/
 # RUN chmod 755 /bin/init_container.sh
 # CMD ["/bin/init_container.sh"]
-ARG BUILD_NUMBER=0
-ENV BUILD_NUMBER=$BUILD_NUMBER
+
+# Set environment variables from build arguments
+ARG APP_VERSION="UNKNOWN"
+ENV APP_VERSION=$APP_VERSION
+ARG BUILD_SHA="UNKNOWN"
+ENV BUILD_SHA=$BUILD_SHA
 
 COPY patches /tmp/patches
 COPY .npmrc package*.json /tmp/

Dockerfile

@@ -15,8 +15,12 @@ ENV APPDIR=/opt/service
 # COPY init_container.sh /bin/
 # RUN chmod 755 /bin/init_container.sh
 # CMD ["/bin/init_container.sh"]
-ARG BUILD_NUMBER=0
-ENV BUILD_NUMBER=$BUILD_NUMBER
+
+# Set environment variables from build arguments
+ARG APP_VERSION="UNKNOWN"
+ENV APP_VERSION=$APP_VERSION
+ARG BUILD_SHA="UNKNOWN"
+ENV BUILD_SHA=$BUILD_SHA
 
 COPY patches /tmp/patches
 COPY .npmrc package*.json /tmp/

SECURITY.md

@@ -0,0 +1,71 @@
+# Vulnerability Disclosure and Embargo Policy
+
+The ClearlyDefined project welcomes the responsible disclosure of vulnerabilities, including those discovered in:
+
+- [ClearlyDefined website](https://github.com/clearlydefined/website/security/advisories/new)
+- [ClearlyDefined service](https://github.com/clearlydefined/service/security/advisories/new)
+- [ClearlyDefined crawler](https://github.com/clearlydefined/crawler/security/advisories/new)
+- [ClearlyDefined documentation](https://github.com/clearlydefined/clearlydefined/security/advisories/new)
+
+## Initial Contact
+
+All security bugs in ClearlyDefined should be reported to the security team.
+To do so, please reach out in the form of a
+[Github Security Advisory](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities).
+
+You will be invited to join this private area to discuss specifics. Doing so
+allows us to start with a high level of confidentiality and relax it if the
+issue is less critical, moving to work on the fix in the open.
+
+Your initial contact will be acknowledged within 48 hours, and you’ll receive
+a more detailed response within 96 hours indicating the next steps in handling
+your report.
+
+After the initial reply to your report, the security team will endeavor to
+keep you informed of the progress being made towards a fix and full
+announcement. As recommended by
+[RFPolicy](https://dl.packetstormsecurity.net/papers/general/rfpolicy-2.0.txt),
+these updates will be sent at least every five working days.
+
+## Disclosure Policy
+
+The ClearlyDefined project has a 5 step disclosure process.
+
+1. Contact is established, a private channel created, and the security report
+   is received and is assigned a primary handler. This person will coordinate
+   the fix and release process.
+2. The problem is confirmed and a list of all affected versions is determined.
+   If an embargo is needed (see below), details of the embargo are decided.
+3. Code is audited to find any potential similar problems.
+4. Fixes are prepared for all releases which are still under maintenance. In
+   case of embargo, these fixes are not committed to the public repository but
+   rather held in a private fork pending the announcement.
+5. The changes are pushed to the public repository and new builds are deployed.
+
+This process can take some time, especially when coordination is required
+with maintainers of other projects. Every effort will be made to handle the bug
+in as timely a manner as possible, however it is important that we follow the
+release process above to ensure that the disclosure is handled in a consistent
+manner.
+
+## Embargoes
+
+While the ClearlyDefined project aims to follow the highest standards of
+transparency and openness, handling some security issues may pose such an
+immediate threat to various stakeholders and require coordination between
+various actors that it cannot be made immediately public.
+
+In this case, security issues will fall under an embargo.
+
+An embargo can be called for in various cases:
+
+- when disclosing the issue without simultaneously providing a mitigation
+  would seriously endanger users,
+- when producing a fix requires coordinating between multiple actors (such as
+  upstream or downstream/dependency projects), or simply
+- when proper analysis of the issue and its ramifications demands time.
+
+If we determine that an issue you report requires an embargo, we will discuss
+this with you and try to find a reasonable expiry date (aka “embargo
+completion date”), as well as who should be included in the list of
+need-to-know people.

app.js

@@ -225,7 +225,7 @@ function createApp(config) {
         // Bit of trick for local hosting. Preload search if using an in-memory search service
         // Commenting out because I believe this is broken
         // if (searchService.constructor.name === 'MemorySearch') await definitionService.reload('definitions')
-        logger.info('Service initialized', { buildNumber: process.env.BUILD_NUMBER })
+        logger.info('Service initialized', { appVersion: process.env.APP_VERSION })
 
         // kick off the queue processors
         require('./providers/curation/process')(curationQueue, curationService, logger)

business/definitionService.js

@@ -34,7 +34,7 @@ const SPDX = require('@clearlydefined/spdx')
 const parse = require('spdx-expression-parse')
 const computeLock = require('../providers/caching/memory')({ defaultTtlSeconds: 60 * 5 /* 5 mins */ })
 
-const currentSchema = '1.6.1'
+const currentSchema = '1.7.0'
 
 const weights = { declared: 30, discovered: 25, consistency: 15, spdx: 15, texts: 15, date: 30, source: 70 }
 

business/statusService.js

@@ -1,7 +1,7 @@
 // Copyright (c) Microsoft Corporation and others. Licensed under the MIT license.
 // SPDX-License-Identifier: MIT
 
-const requestPromise = require('request-promise-native')
+const { callFetch: requestPromise } = require('../lib/fetch')
 const logger = require('../providers/logging/logger')
 
 class StatusService {

lib/fetch.js

@@ -0,0 +1,51 @@
+// (c) Copyright 2024, SAP SE and ClearlyDefined contributors. Licensed under the MIT license.
+// SPDX-License-Identifier: MIT
+
+const axios = require('axios')
+
+function buildRequestOptions(request) {
+  let responseType = 'text'
+  if (request.json) {
+    responseType = 'json'
+  } else if (request.encoding === null) {
+    responseType = 'stream'
+  }
+
+  const validateOptions = {}
+  if (request.simple === false) {
+    validateOptions.validateStatus = () => true
+  }
+
+  return {
+    method: request.method,
+    url: request.url || request.uri,
+    responseType,
+    headers: request.headers,
+    data: request.body,
+    withCredentials: request.withCredentials,
+    ...validateOptions
+  }
+}
+
+async function callFetch(request, axiosInstance = axios) {
+  try {
+    const options = buildRequestOptions(request)
+    // @ts-ignore
+    const response = await axiosInstance(options)
+    if (!request.resolveWithFullResponse) return response.data
+    response.statusCode = response.status
+    response.statusMessage = response.statusText
+    return response
+  } catch (error) {
+    error.statusCode = error.response?.status
+    throw error
+  }
+}
+
+function withDefaults(opts) {
+  // @ts-ignore
+  const axiosInstance = axios.create(opts)
+  return request => callFetch(request, axiosInstance)
+}
+
+module.exports = { callFetch, withDefaults }

lib/gradleCoordinatesMapper.js

@@ -1,7 +1,7 @@
 // (c) Copyright 2021, SAP SE and ClearlyDefined contributors. Licensed under the MIT license.
 // SPDX-License-Identifier: MIT
 
-const requestPromise = require('request-promise-native')
+const { callFetch: requestPromise } = require('../lib/fetch')
 const { promisify } = require('util')
 const parseXml = promisify(require('xml2js').parseString)
 const { get } = require('lodash')

lib/pypiCoordinatesMapper.js

@@ -1,7 +1,7 @@
 // (c) Copyright 2021, SAP SE and ClearlyDefined contributors. Licensed under the MIT license.
 // SPDX-License-Identifier: MIT
 
-const requestPromise = require('request-promise-native')
+const { callFetch: requestPromise } = require('../lib/fetch')
 const EntityCoordinates = require('./entityCoordinates')
 
 class PypiCoordinatesMapper {