Merge branch 'conda-support' of https://github.com/lamarrr/clearlydef…

…ined-service into conda-support

.github/dependabot.yml

@@ -0,0 +1,11 @@
+version: 2
+updates:
+  - package-ecosystem: 'npm'
+    directory: '/'
+    schedule:
+      interval: weekly
+
+  - package-ecosystem: 'github-actions'
+    directory: '/'
+    schedule:
+      interval: 'weekly'

.github/workflows/build.yml

@@ -13,9 +13,9 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - uses: actions/setup-node@v2
+      - uses: actions/setup-node@v4
         with:
           node-version: 18
           cache: 'npm'

.github/workflows/build_and_deploy_dev.yml

@@ -0,0 +1,105 @@
+# This workflow will build a docker image, push it to ghcr.io, and deploy it to an Azure WebApp.
+name: Build and Deploy to dev service app
+
+# Update the triggers based on the environment that is being deployed to.
+#   Triggers for dev deployments:  1) manually triggered, 2) push to branch `master`
+#   Triggers for prod deployments: 1) manually triggered, 2) release created
+on:
+  workflow_dispatch:
+  push:
+    branches: [master]
+
+# There are secrets and environment variables that need to be set that control what is pushed to
+# ghcr and Azure.
+#
+# Secrets:
+#   AZURE_CREDENTIALS:                service principal that has access to the Azure WebApp
+#   AZURE_WEBAPP_PUBLISH_PROFILE_DEV: publish profile for the Azure WebApp  NOTE: The name of the secret changes.  For dev, it ends in `_DEV`.  Production does not have an extension.
+#
+# Environment Variables:
+#   APPLICATION_TYPE:   type of application that is being deployed; used to add a label to the Docker image (values: api | web | worker)
+#   AZURE_WEBAPP_NAME:  name of the Azure WebApp being deployed
+#   DEPLOY_ENVIRONMENT:   environment that the code is being deployed to; used to add a label to the Docker image (values: dev | prod)
+#   DEPLOY_DOCKER_TAG:  the tag used for deploying a specific Docker image to Azure.  For dev, use the `github.sha`.  For production, use the SEMVER
+#                       version of the release.  Make sure to add this tag to the `DOCKER_TAGS` in the `Build and push Docker image` step.
+#   DOCKER_IMAGE_NAME:  name of the Docker image that is being built and pushed to ghcr.io.
+
+env:
+  APPLICATION_TYPE: api
+  AZURE_WEBAPP_NAME: clearlydefined-api-dev
+  DEPLOY_ENVIRONMENT: dev
+  DEPLOY_DOCKER_TAG: ${{ github.sha }}
+  DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/${{ github.repository }}-dev
+
+jobs:
+  build-and-deploy:
+    name: Build and Deploy
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Log into ghcr registry
+        uses: docker/login-action@v3.0.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }} # user that kicked off the action
+          password: ${{ secrets.GITHUB_TOKEN }} # token created when the action launched (short lived)
+
+      - name: Build and push Docker image
+        env:
+          DOCKER_TAGS: |
+            ${{ env.DOCKER_IMAGE_NAME }}:${{ env.DEPLOY_DOCKER_TAG }}
+        uses: docker/build-push-action@v5.1.0
+        with:
+          context: .
+          push: true
+          file: Dockerfile
+          tags: ${{ env.DOCKER_TAGS }}
+          labels: |
+            env=${{ env.DEPLOY_ENVIRONMENT }}
+            type=${{ env.APPLICATION_TYPE }}
+
+      - name: Login for Azure cli commands
+        uses: azure/login@v1.6.1
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS }}
+
+      - name: Set DOCKER configs in Azure web app
+        uses: azure/appservice-settings@v1.1.1
+        with:
+          app-name: ${{ env.AZURE_WEBAPP_NAME }}
+          app-settings-json: |
+            [
+              {
+                    "name": "DOCKER_CUSTOM_IMAGE_NAME",
+                    "value": "${{ env.DOCKER_IMAGE_NAME }}:${{ env.DEPLOY_DOCKER_TAG }}",
+                    "slotSetting": false
+              },
+              {
+                    "name": "DOCKER_REGISTRY_SERVER_URL",
+                    "value": "https://ghcr.io",
+                    "slotSetting": false
+                },
+                {
+                    "name": "DOCKER_REGISTRY_SERVER_USERNAME",
+                    "value": "${{ secrets.REGISTRY_USERNAME  }}",
+                    "slotSetting": false
+                },
+                {
+                    "name": "DOCKER_REGISTRY_SERVER_PASSWORD",
+                    "value": "${{ secrets.REGISTRY_PASSWORD }}",
+                    "slotSetting": false
+                },
+                {
+                    "name": "BUILD_SHA",
+                    "value": "${{ github.sha }}",
+                    "slotSetting": false
+                }
+            ]
+
+      - name: Deploy to Azure WebApp
+        uses: azure/webapps-deploy@v3.0.0
+        with:
+          app-name: ${{ env.AZURE_WEBAPP_NAME }}
+          publish-profile: ${{ secrets.AZURE_WEBAPP_PUBLISH_PROFILE_DEV }}
+          images: '${{ env.DOCKER_IMAGE_NAME }}:${{ env.DEPLOY_DOCKER_TAG }}'

.github/workflows/codeql.yml

@@ -0,0 +1,37 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+  schedule:
+    - cron: '27 10 * * 1'
+
+permissions:
+  contents: read
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    steps:
+    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      with:
+          fetch-depth: 0
+
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@e8893c57a1f3a2b659b6b55564fdfdbbd2982911 # v3.24.0
+      with:
+        languages: 'javascript'
+
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@e8893c57a1f3a2b659b6b55564fdfdbbd2982911 # v3.24.0
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@e8893c57a1f3a2b659b6b55564fdfdbbd2982911 # v3.24.0

app.js

@@ -181,7 +181,7 @@ function createApp(config) {
 
   app.use(require('./middleware/querystring'))
 
-  app.use('/', require('./routes/index'))
+  app.use('/', require('./routes/index')(config.buildsha))
   app.use('/origins/github', require('./routes/originGitHub')())
   app.use('/origins/crate', require('./routes/originCrate')())
   app.use('/origins/conda', require('./routes/originConda')())

bin/config.js

@@ -91,5 +91,6 @@ module.exports = {
     serviceKey: config.get('APPINSIGHTS_SERVICE_APIKEY'),
     crawlerId: config.get('APPINSIGHTS_CRAWLER_APPLICATIONID'),
     crawlerKey: config.get('APPINSIGHTS_CRAWLER_APIKEY')
-  }
+  },
+  buildsha : config.get('BUILD_SHA')
 }

docs/rampup.md

@@ -80,7 +80,7 @@ GitHub webhook is set to put message on queue for any change in PR status, this
 ### Database
 What do we put in mongo / Cosmos DB? ( I see config for curations and definitions)
 *	Curations: looks like pr info, coordinates. Guess: used for curation info in relation to a coordinate (in UI)
-*	Definitions: mapping between attachments and definitions, finding the attachment blobs in azure storage.
+*	Definitions: mapping between attachments and definitions, finding the attachment blobs in azure storage
 
 ### Definitions
 business/definitionservice.js: code to compute definition, recomputed after harvest or curation

lib/pypiCoordinatesMapper.js

@@ -23,8 +23,17 @@ class PypiCoordinatesMapper {
 
   async _resolve(coordinates) {
     const url = `${this.baseUrl}/pypi/${coordinates.name}/json`
-    const answer = await requestPromise({ url, method: 'GET', json: true })
-    return answer?.info?.name && { name: answer.info.name }
+    try {
+      const answer = await this._handleRequest(url)
+      return answer?.info?.name && { name: answer.info.name }  
+    } catch (error) {
+      if (error.statusCode === 404) return null
+      throw error
+    }
+  }
+
+  _handleRequest(url) {
+    return requestPromise({ url, method: 'GET', json: true })
   }
 }