From 247b0763150ff3c9a24f8c3d794eaa1c248440c0 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 7 May 2024 11:14:42 +0100 Subject: [PATCH 1/4] [conda] fixed service component bug --- routes/originConda.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/routes/originConda.js b/routes/originConda.js index 843b3e6b2..a3ef9c3d4 100644 --- a/routes/originConda.js +++ b/routes/originConda.js @@ -41,7 +41,7 @@ router.get( let { channel, subdir, name } = request.params channel = encodeURIComponent(channel) subdir = encodeURIComponent(subdir) - name = encodeURIComponent(channel) + name = encodeURIComponent(name) if (!condaChannels[channel]) { return response.status(404).send(`Unrecognized Conda channel ${channel}`) } @@ -50,7 +50,7 @@ router.get( return response.status(404).send(`Package ${name} not found in Conda channel ${channel}`) } if (subdir !== '-' && !channelData.subdirs.find(x => x == subdir)) { - return response.status(404).send(`Subdir ${subdir} is non-existent in Conda channel ${channel}`) + return response.status(404).send(`Subdir ${subdir} is non-existent in Conda channel ${channel}, subdirs: ${channelData.subdirs}`) } let revisions = [] let subdirs = subdir === '-' ? channelData.packages[name].subdirs : [subdir] @@ -82,7 +82,7 @@ router.get( channel = encodeURIComponent(channel) name = encodeURIComponent(name) if (!condaChannels[channel]) { - return response.status(404).send([]) + return response.status(404).send(`Unrecognized Conda channel ${channel}`) } let channelData = await fetchCondaChannelData(channel) let matches = Object.entries(channelData.packages) From 2a936ebb42c17f7bc32d8d377a2761f171201a58 Mon Sep 17 00:00:00 2001 From: Basit Ayantunde Date: Tue, 7 May 2024 11:28:44 +0100 Subject: [PATCH 2/4] [conda] fixed service component bug --- routes/originConda.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/routes/originConda.js b/routes/originConda.js index a3ef9c3d4..874a19f90 100644 --- a/routes/originConda.js +++ b/routes/originConda.js @@ -50,7 +50,9 @@ router.get( return response.status(404).send(`Package ${name} not found in Conda channel ${channel}`) } if (subdir !== '-' && !channelData.subdirs.find(x => x == subdir)) { - return response.status(404).send(`Subdir ${subdir} is non-existent in Conda channel ${channel}, subdirs: ${channelData.subdirs}`) + return response + .status(404) + .send(`Subdir ${subdir} is non-existent in Conda channel ${channel}, subdirs: ${channelData.subdirs}`) } let revisions = [] let subdirs = subdir === '-' ? channelData.packages[name].subdirs : [subdir] From e35340b7ce7cb6d6cf63f6cf8675ebeb5a1b2ea7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20Spie=C3=9F?= Date: Tue, 7 May 2024 13:59:38 +0000 Subject: [PATCH 3/4] Add a simple test for conda revisions route --- routes/originConda.js | 119 +++++++++++++++++++++-------------------- test/routes/origins.js | 33 ++++++++++++ 2 files changed, 94 insertions(+), 58 deletions(-) create mode 100644 test/routes/origins.js diff --git a/routes/originConda.js b/routes/originConda.js index 874a19f90..b13479e63 100644 --- a/routes/originConda.js +++ b/routes/originConda.js @@ -35,68 +35,71 @@ async function fetchCondaRepoData(channel, subdir) { return repoData } -router.get( - '/:channel/:subdir/:name/revisions', - asyncMiddleware(async (request, response) => { - let { channel, subdir, name } = request.params - channel = encodeURIComponent(channel) - subdir = encodeURIComponent(subdir) - name = encodeURIComponent(name) - if (!condaChannels[channel]) { - return response.status(404).send(`Unrecognized Conda channel ${channel}`) - } - let channelData = await fetchCondaChannelData(channel) - if (!channelData.packages[name]) { - return response.status(404).send(`Package ${name} not found in Conda channel ${channel}`) - } - if (subdir !== '-' && !channelData.subdirs.find(x => x == subdir)) { - return response - .status(404) - .send(`Subdir ${subdir} is non-existent in Conda channel ${channel}, subdirs: ${channelData.subdirs}`) - } - let revisions = [] - let subdirs = subdir === '-' ? channelData.packages[name].subdirs : [subdir] - for (let subdir of subdirs) { - const repoData = await fetchCondaRepoData(channel, subdir) - if (repoData['packages']) { - Object.entries(repoData['packages']).forEach(([, packageData]) => { - if (packageData.name === name) { - revisions.push(`${subdir}:${packageData.version}-${packageData.build}`) - } - }) - } - if (repoData['packages.conda']) { - Object.entries(repoData['packages.conda']).forEach(([, packageData]) => { - if (packageData.name === name) { - revisions.push(`${subdir}:${packageData.version}-${packageData.build}`) - } - }) - } - } - return response.status(200).send(uniq(revisions)) - }) -) +router.get('/:channel/:subdir/:name/revisions', asyncMiddleware(getOriginCondaRevisions())) -router.get( - '/:channel/:name', - asyncMiddleware(async (request, response) => { - let { channel, name } = request.params - channel = encodeURIComponent(channel) - name = encodeURIComponent(name) - if (!condaChannels[channel]) { - return response.status(404).send(`Unrecognized Conda channel ${channel}`) +async function getOriginCondaRevisions(request, response) { + let { channel, subdir, name } = request.params + channel = encodeURIComponent(channel) + subdir = encodeURIComponent(subdir) + name = encodeURIComponent(name) + if (!condaChannels[channel]) { + return response.status(404).send(`Unrecognized Conda channel ${channel}`) + } + let channelData = await fetchCondaChannelData(channel) + if (!channelData.packages[name]) { + return response.status(404).send(`Package ${name} not found in Conda channel ${channel}`) + } + if (subdir !== '-' && !channelData.subdirs.find(x => x == subdir)) { + return response + .status(404) + .send(`Subdir ${subdir} is non-existent in Conda channel ${channel}, subdirs: ${channelData.subdirs}`) + } + let revisions = [] + let subdirs = subdir === '-' ? channelData.packages[name].subdirs : [subdir] + for (let subdir of subdirs) { + const repoData = await fetchCondaRepoData(channel, subdir) + if (repoData['packages']) { + Object.entries(repoData['packages']).forEach(([, packageData]) => { + if (packageData.name === name) { + revisions.push(`${subdir}:${packageData.version}-${packageData.build}`) + } + }) } - let channelData = await fetchCondaChannelData(channel) - let matches = Object.entries(channelData.packages) - .filter(([packageName]) => packageName.includes(name)) - .map(([packageName]) => { - return { id: packageName } + if (repoData['packages.conda']) { + Object.entries(repoData['packages.conda']).forEach(([, packageData]) => { + if (packageData.name === name) { + revisions.push(`${subdir}:${packageData.version}-${packageData.build}`) + } }) - return response.status(200).send(matches) - }) -) + } + } + return response.status(200).send(uniq(revisions)) +} + +router.get('/:channel/:name', asyncMiddleware(getOriginConda)) + +async function getOriginConda(request, response) { + let { channel, name } = request.params + channel = encodeURIComponent(channel) + name = encodeURIComponent(name) + if (!condaChannels[channel]) { + return response.status(404).send(`Unrecognized Conda channel ${channel}`) + } + let channelData = await fetchCondaChannelData(channel) + let matches = Object.entries(channelData.packages) + .filter(([packageName]) => packageName.includes(name)) + .map(([packageName]) => { + return { id: packageName } + }) + return response.status(200).send(matches) +} + +function setup(testFlag = false) { + if (testFlag) { + router._getOriginConda = getOriginConda + router._getOriginCondaRevisions = getOriginCondaRevisions + } -function setup() { return router } diff --git a/test/routes/origins.js b/test/routes/origins.js new file mode 100644 index 000000000..bacc7a8b2 --- /dev/null +++ b/test/routes/origins.js @@ -0,0 +1,33 @@ +// Copyright (c) The Linux Foundation and others. Licensed under the MIT license. +// SPDX-License-Identifier: MIT + +const { expect } = require('chai') +const httpMocks = require('node-mocks-http') +const originCondaRoutes = require('../../routes/originConda') + +describe('Conda origin routes', () => { + it('accepts a good revisions GET request', async () => { + const request = createGetOriginCondaRevisionsRequest() + const response = httpMocks.createResponse() + const router = createRoutes() + await router._getOriginCondaRevisions(request, response) + expect(response.statusCode).to.be.eq(200) + }) +}) + +function createGetOriginCondaRevisionsRequest() { + return httpMocks.createRequest({ + method: 'GET', + url: 'origins/conda/conda-forge/linux-64/tensorflow/revisions', + baseUrl: 'https://dev.clearlydefined.io', + params: { + channel: 'conda-forge', + subdir: 'linux-64', + name: 'tensorflow', + } + }) +} + +function createRoutes() { + return originCondaRoutes(true) +} From 112645cf3ed77e07a2f63f854695dc2b43ff323d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20Spie=C3=9F?= Date: Tue, 7 May 2024 15:08:09 +0000 Subject: [PATCH 4/4] Stub out network responses in test --- routes/originConda.js | 7 +++++-- test/routes/origins.js | 41 +++++++++++++++++++++++++++++++++++++---- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/routes/originConda.js b/routes/originConda.js index b13479e63..763997b95 100644 --- a/routes/originConda.js +++ b/routes/originConda.js @@ -11,7 +11,6 @@ const condaChannels = { 'anaconda-r': 'https://repo.anaconda.com/pkgs/r', 'conda-forge': 'https://conda.anaconda.org/conda-forge' } -const condaCache = new Cache() async function fetchCondaChannelData(channel) { const key = `${channel}-channelData` @@ -94,7 +93,11 @@ async function getOriginConda(request, response) { return response.status(200).send(matches) } -function setup(testFlag = false) { +let condaCache + +function setup(cache = new Cache(), testFlag = false) { + condaCache = cache + if (testFlag) { router._getOriginConda = getOriginConda router._getOriginCondaRevisions = getOriginCondaRevisions diff --git a/test/routes/origins.js b/test/routes/origins.js index bacc7a8b2..367f6726f 100644 --- a/test/routes/origins.js +++ b/test/routes/origins.js @@ -3,15 +3,36 @@ const { expect } = require('chai') const httpMocks = require('node-mocks-http') +const sinon = require('sinon') const originCondaRoutes = require('../../routes/originConda') describe('Conda origin routes', () => { it('accepts a good revisions GET request', async () => { const request = createGetOriginCondaRevisionsRequest() const response = httpMocks.createResponse() - const router = createRoutes() + const stubCondaCache = createStubCondaCache({ + 'conda-forge-linux-64-repoData': { + packages: [ + { + name: 'tensorflow', + version: '2.15.0', + build: 'cuda120py39hb94c71b_3' + } + ], + subdirs: ['linux-64'] + }, + 'conda-forge-channelData': { + packages: { + tensorflow: {} + }, + subdirs: ['linux-64'] + } + }) + const router = createRoutes(stubCondaCache) await router._getOriginCondaRevisions(request, response) expect(response.statusCode).to.be.eq(200) + expect(response._getData()).to.be.deep.equal(['linux-64:2.15.0-cuda120py39hb94c71b_3']) + expect(stubCondaCache.get.calledTwice).to.be.true }) }) @@ -23,11 +44,23 @@ function createGetOriginCondaRevisionsRequest() { params: { channel: 'conda-forge', subdir: 'linux-64', - name: 'tensorflow', + name: 'tensorflow' } }) } -function createRoutes() { - return originCondaRoutes(true) +function createRoutes(condaCache) { + return originCondaRoutes(condaCache, true) +} + +function createStubCondaCache(cacheData) { + let getStub = sinon.stub() + + for (const [key, value] of Object.entries(cacheData)) { + getStub.withArgs(key).returns(value) + } + + return { + get: getStub + } }