From 1bd5cb2a70b065ce0827d803ad0ee14abd241b4f Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Sat, 3 Feb 2024 14:54:10 -0700 Subject: [PATCH] Add github action for integration test --- .github/workflows/integration-test.yml | 28 ++++++ integration/test/harvestTest.js | 36 ++++++++ integration/test/serviceTest.js | 61 +++++++++++++ integration/test/testConfig.js | 43 ++++++++++ integration/test/toolTest.js | 113 +++++++++++++++++++++++++ integration/tools/fetch.js | 20 +++++ integration/tools/harvester.js | 77 +++++++++++++++++ integration/tools/poller.js | 20 +++++ package-lock.json | 91 ++++++++++++++++++++ package.json | 1 + 10 files changed, 490 insertions(+) create mode 100644 .github/workflows/integration-test.yml create mode 100644 integration/test/harvestTest.js create mode 100644 integration/test/serviceTest.js create mode 100644 integration/test/testConfig.js create mode 100644 integration/test/toolTest.js create mode 100644 integration/tools/fetch.js create mode 100644 integration/tools/harvester.js create mode 100644 integration/tools/poller.js diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml new file mode 100644 index 000000000..57270f08e --- /dev/null +++ b/.github/workflows/integration-test.yml @@ -0,0 +1,28 @@ +name: Integration Test + +on: + workflow_dispatch: + ## The tests take a long time to run and can be potentially quite costly, so set it to run manually + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4.1.1 + + - uses: actions/setup-node@v4.0.1 + with: + node-version: 18 + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Trigger harvest and verify completion + run: npx mocha integration/test/harvestTest.js + + - name: Verify computed definitions + run: npx mocha integration/test/serviceTest.js diff --git a/integration/test/harvestTest.js b/integration/test/harvestTest.js new file mode 100644 index 000000000..7d964e59e --- /dev/null +++ b/integration/test/harvestTest.js @@ -0,0 +1,36 @@ +const { components, devApiBaseUrl, harvest } = require('./testConfig') +const Poller = require('../tools/poller') +const Harvester = require('../tools/harvester') +const assert = require('assert') + +describe('Tests for Harvester', function () { + it('should verify all harvests are complete', async function () { + this.timeout(harvest.timeout) + console.time('Harvest Test') + const status = await harvestTillCompletion(components) + for (const [coordinates, isHarvested] of status) { + assert.strictEqual(isHarvested, true, `Harvest for ${coordinates} is not complete`) + } + console.timeEnd('Harvest Test') + }) +}) + +async function harvestTillCompletion(components) { + const { harvestToolVersions, poll } = harvest + const harvester = new Harvester(devApiBaseUrl, harvestToolVersions) + const poller = new Poller(poll.interval, poll.maxTime) + + //make sure that we have one entire set of harvest results (old or new) + console.log('Ensure harvest results exit before starting tests') + const previousHarvests = await harvester.pollForCompletion(components, poller) + const previousHarvestsComplete = Array.from(previousHarvests.values()).every(v => v) + if (!previousHarvestsComplete) { + await harvester.harvest(components) + await harvester.pollForCompletion(components, poller) + } + + //trigger a reharvest to overwrite the old result + console.log('Trigger reharvest to overwrite old results') + await harvester.harvest(components, true) + return harvester.pollForCompletion(components, poller, Date.now()) +} diff --git a/integration/test/serviceTest.js b/integration/test/serviceTest.js new file mode 100644 index 000000000..7001a655b --- /dev/null +++ b/integration/test/serviceTest.js @@ -0,0 +1,61 @@ +const { omit, isEqual } = require('lodash') +const expect = require('chai').expect +const { callFetch } = require('../tools/fetch') +const { devApiBaseUrl, prodApiBaseUrl, components, definition } = require('./testConfig') + +describe('Validate Definition between dev and prod', function () { + it('should get definition for a component and compare to production', async function () { + this.timeout(definition.timeout) + for (const coordinates of components) { + console.log(coordinates) + await compareDefintion(coordinates) + } + }) +}) + +async function compareDefintion(coordinates) { + const recomputedDef = await callFetch(`${devApiBaseUrl}/definitions/${coordinates}?force=true`).then(r => r.json()) + const expectedDef = await callFetch(`${prodApiBaseUrl}/definitions/${coordinates}`).then(r => r.json()) + expect(recomputedDef.coordinates).to.be.deep.equals(expectedDef.coordinates) + compareLicensed(recomputedDef, expectedDef) + compareDescribed(recomputedDef, expectedDef) + compareFiles(recomputedDef, expectedDef) + expect(recomputedDef.score).to.be.deep.equal(expectedDef.score) +} + +function compareLicensed(result, expectation) { + const actual = omit(result.licensed, ['facets']) + const expected = omit(expectation.licensed, ['facets']) + expect(actual).to.be.deep.equals(expected) +} + +function compareDescribed(result, expectation) { + const actual = omit(result.described, ['tools']) + const expected = omit(expectation.described, ['tools']) + expect(actual).to.be.deep.equals(expected) +} + +function compareFiles(result, expectation) { + const resultFiles = filesToMap(result) + const expectedFiles = filesToMap(expectation) + const extraInResult = result.files.filter(f => !expectedFiles.has(f.path)) + const missingInResult = expectation.files.filter(f => !resultFiles.has(f.path)) + const differentEntries = result.files.filter(f => expectedFiles.has(f.path) && !isEqual(expectedFiles.get(f.path), f)) + + const differences = [...extraInResult, ...missingInResult, ...differentEntries] + differences.forEach(f => logFiles(expectedFiles.get(f.path), resultFiles.get(f.path))) + + expect(missingInResult.length).to.be.equal(0, 'Some files are missing in the result') + expect(extraInResult.length).to.be.equal(0, 'There are extra files in the result') + expect(differentEntries.length).to.be.equal(0, 'Some files are different between the result and the expectation') +} + +function logFiles(expected, actual) { + console.log('-------------------') + console.log(`expected: ${JSON.stringify(expected || {})}`) + console.log(`actual: ${JSON.stringify(actual || {})}`) +} + +function filesToMap(result) { + return new Map(result.files.map(f => [f.path, f])) +} diff --git a/integration/test/testConfig.js b/integration/test/testConfig.js new file mode 100644 index 000000000..682b93148 --- /dev/null +++ b/integration/test/testConfig.js @@ -0,0 +1,43 @@ +const devApiBaseUrl = 'https://dev-api.clearlydefined.io' +const prodApiBaseUrl = 'https://api.clearlydefined.io' + +const pollingInterval = 1000 * 60 * 5 // 5 minutes +const pollingMaxTime = 1000 * 60 * 30 // 30 minutes + +const harvestToolVersions = [ + ['licensee', '9.14.0'], + ['scancode', '30.3.0'], + ['reuse', '3.2.1'] +] + +const components = [ + 'maven/mavencentral/org.apache.httpcomponents/httpcore/4.4.16', + 'maven/mavengoogle/android.arch.lifecycle/common/1.0.1', + 'maven/gradleplugin/io.github.lognet/grpc-spring-boot-starter-gradle-plugin/4.6.0', + 'crate/cratesio/-/ratatui/0.26.0', + 'npm/npmjs/-/redis/0.1.0', + 'git/github/ratatui-org/ratatui/bcf43688ec4a13825307aef88f3cdcd007b32641', + 'gem/rubygems/-/sorbet/0.5.11226', + 'pypi/pypi/-/platformdirs/4.2.0', + 'go/golang/rsc.io/quote/v1.3.0', + 'nuget/nuget/-/HotChocolate/13.8.1' + // 'composer/packagist/symfony/polyfill-mbstring/1.11.0', + // 'pod/cocoapods/-/SoftButton/0.1.0', + // 'deb/debian/-/mini-httpd/1.30-0.2_arm64' + // 'debsrc/debian/-/mini-httpd/1.30-0.2_arm64', + // 'sourcearchive/mavencentral/org.apache.httpcomponents/httpcore/4.1' +] + +module.exports = { + devApiBaseUrl, + prodApiBaseUrl, + components, + harvest: { + poll: { interval: pollingInterval, maxTime: pollingMaxTime }, + harvestToolVersions, + timeout: 1000 * 60 * 60 * 2 // 2 hours + }, + definition: { + timeout: 1000 * 20 // 20 seconds + } +} diff --git a/integration/test/toolTest.js b/integration/test/toolTest.js new file mode 100644 index 000000000..512889ea5 --- /dev/null +++ b/integration/test/toolTest.js @@ -0,0 +1,113 @@ +const expect = require('chai').expect +const { callFetch } = require('../tools/fetch') +const Poller = require('../tools/poller') +const Harvester = require('../tools/harvester') +const { devApiBaseUrl } = require('./testConfig') +const sinon = require('sinon') + +describe('Integration test against dev deployment', function () { + it('should get harvest for a component', async function () { + const coordinates = 'nuget/nuget/-/HotChocolate/13.8.1' + const result = await callFetch(`${devApiBaseUrl}/harvest/${coordinates}?form=list`).then(r => r.json()) + expect(result.length).to.be.greaterThan(0) + }) + + it.skip('should harvest a component', async function () { + const coordinates = 'nuget/nuget/-/HotChocolate/13.8.1' + const harvester = new Harvester(devApiBaseUrl) + const result = await harvester.harvest([coordinates]) + expect(result.status).to.be.equal(201) + }) +}) + +describe('Tests for Harvester', function () { + const coordinates = 'nuget/nuget/-/HotChocolate/13.8.1' + let harvester + beforeEach(function () { + harvester = new Harvester(devApiBaseUrl) + }) + + it('should detect when a scan tool result for component is available', async function () { + sinon.stub(harvester, 'fetchHarvestResult').resolves(metadata()) + const result = await harvester.isHarvestedbyTool(coordinates, 'licensee', '9.14.0') + expect(result).to.be.equal(true) + }) + + it('should detect when component is completely harvested', async function () { + sinon.stub(harvester, 'fetchHarvestResult').resolves(metadata()) + const result = await harvester.isHarvestComplete(coordinates) + expect(result).to.be.equal(true) + }) + + it('should detect whether component is harvested after a timestamp', async function () { + const date = '2023-01-01T00:00:00.000Z' + sinon.stub(harvester, 'fetchHarvestResult').resolves(metadata(date)) + const result = await harvester.isHarvestComplete(coordinates, Date.now()) + expect(result).to.be.equal(false) + }) +}) + +describe('Integration Tests for Harvester and Poller', function () { + const coordinates = 'nuget/nuget/-/HotChocolate/13.8.1' + const interval = 10 * 1 + const maxTime = 10 * 2 + let poller + let harvester + + beforeEach(function () { + harvester = new Harvester(devApiBaseUrl) + poller = new Poller(interval, maxTime) + }) + + it('should poll until max time is reached', async function () { + sinon.stub(harvester, 'fetchHarvestResult').resolves({}) + const result = await poller.poll(async () => await harvester.isHarvestComplete(coordinates, Date.now())) + expect(result).to.be.equal(false) + }) + + it('should poll for completion if results exist', async function () { + sinon.stub(harvester, 'fetchHarvestResult').resolves(metadata()) + const status = await harvester.pollForCompletion([coordinates], poller) + expect(status.get(coordinates)).to.be.equal(true) + }) + + it('should poll for completion if results are stale', async function () { + const date = '2023-01-01T00:00:00.000Z' + sinon.stub(harvester, 'fetchHarvestResult').resolves(metadata(date)) + const status = await harvester.pollForCompletion([coordinates], poller, Date.now()) + expect(status.get(coordinates)).to.be.equal(false) + }) +}) + +describe('Unit Tests for Poller', function () { + const interval = 10 * 1 + const maxTime = 10 * 2 + let poller + + beforeEach(function () { + poller = new Poller(interval, maxTime) + }) + + it('should poll until max time reached', async function () { + const activity = sinon.stub().resolves(false) + const result = await poller.poll(activity) + expect(activity.callCount).to.be.equal(3) + expect(result).to.be.equal(false) + }) + + it('should handle when activity is done', async function () { + const activity = sinon.stub().resolves(true) + const result = await poller.poll(activity) + expect(activity.callCount).to.be.equal(1) + expect(result).to.be.equal(true) + }) + + it('should continue to poll until activity is done', async function () { + const activity = sinon.stub().resolves(false).onCall(1).resolves(true) + const result = await poller.poll(activity) + expect(activity.callCount).to.be.equal(2) + expect(result).to.be.equal(true) + }) +}) + +const metadata = date => ({ _metadata: { fetchedAt: date || new Date().toISOString() } }) diff --git a/integration/tools/fetch.js b/integration/tools/fetch.js new file mode 100644 index 000000000..62d40135a --- /dev/null +++ b/integration/tools/fetch.js @@ -0,0 +1,20 @@ +const fetch = (...args) => import('node-fetch').then(({ default: fetch }) => fetch(...args)) + +function buildPostOpts(json) { + return { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(json) + } +} + +async function callFetch(url, fetchOpts) { + console.log(url, fetchOpts) + const response = await fetch(url, fetchOpts) + if (!response.ok) { + const { status, statusText } = response + throw new Error(`Error fetching ${url}: ${status}, ${statusText}`) + } + return response +} +module.exports = { callFetch, buildPostOpts } diff --git a/integration/tools/harvester.js b/integration/tools/harvester.js new file mode 100644 index 000000000..138ce40b2 --- /dev/null +++ b/integration/tools/harvester.js @@ -0,0 +1,77 @@ +const { callFetch, buildPostOpts } = require('./fetch') + +const defaultToolChecks = [ + ['licensee', '9.14.0'], + ['scancode', '30.3.0'], + ['reuse', '3.2.1'] +] + +class Harvester { + constructor(apiBaseUrl, harvestToolChecks) { + this.apiBaseUrl = apiBaseUrl + this.harvestToolChecks = harvestToolChecks || defaultToolChecks + } + + async harvest(components, reharvest = false) { + return await callFetch(`${this.apiBaseUrl}/harvest`, buildPostOpts(this._buildPostJson(components, reharvest))) + } + + _buildPostJson(components, reharvest = false) { + return components.map(coordinates => { + const result = { tool: 'component', coordinates } + if (reharvest) result.policy = 'always' + return result + }) + } + + async pollForCompletion(components, poller, startTime) { + const status = new Map() + for (const coordinates of components) { + const completed = await this._pollForOneCompletion(coordinates, poller, startTime) + status.set(coordinates, completed) + } + + for (const coordinates of components) { + const completed = status.get(coordinates) || (await this.isHarvestComplete(coordinates, startTime)) + status.set(coordinates, completed) + } + return status + } + + async _pollForOneCompletion(coordinates, poller, startTime) { + try { + const completed = await poller.poll(async () => this.isHarvestComplete(coordinates, startTime)) + console.log(`Completed ${coordinates}: ${completed}`) + return completed + } catch (e) { + console.error(`Failed to wait for harvest completion ${coordinates}: ${e.message}`) + return false + } + } + + async isHarvestComplete(coordinates, startTime) { + const harvestChecks = this.harvestToolChecks.map(([tool, toolVersion]) => + this.isHarvestedbyTool(coordinates, tool, toolVersion, startTime) + ) + + return Promise.all(harvestChecks) + .then(results => results.every(r => r)) + .catch(() => false) + } + + async isHarvestedbyTool(coordinates, tool, toolVersion, startTime = 0) { + const harvested = await this.fetchHarvestResult(coordinates, tool, toolVersion) + if (!harvested._metadata) return false + const fetchedAt = new Date(harvested._metadata.fetchedAt) + console.log(`${coordinates} ${tool}, ${toolVersion} fetched at ${fetchedAt}`) + return fetchedAt.getTime() > startTime + } + + async fetchHarvestResult(coordinates, tool, toolVersion) { + return callFetch(`${this.apiBaseUrl}/harvest/${coordinates}/${tool}/${toolVersion}?form=raw`) + .then(r => r.json()) + .catch(() => ({})) + } +} + +module.exports = Harvester diff --git a/integration/tools/poller.js b/integration/tools/poller.js new file mode 100644 index 000000000..0a2554329 --- /dev/null +++ b/integration/tools/poller.js @@ -0,0 +1,20 @@ +class Poller { + constructor(interval, maxTime) { + this.interval = interval + this.maxTime = maxTime + } + + async poll(activity) { + let counter = 0 + while (counter * this.interval <= this.maxTime) { + console.log(`Polling ${counter}`) + const isDone = await activity() + if (isDone) return true + await new Promise(resolve => setTimeout(resolve, this.interval)) + counter++ + } + return false + } +} + +module.exports = Poller diff --git a/package-lock.json b/package-lock.json index 00da0fc4b..87279a08d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -74,6 +74,7 @@ "eslint": "^7.5", "mocha": "^8.2.1", "mongodb-memory-server": "^8.11.2", + "node-fetch": "3.3.2", "node-mocks-http": "^1.6.7", "nodemon": "^2.0.3", "nyc": "^15.0.0", @@ -3866,6 +3867,15 @@ "resolved": "https://registry.npmjs.org/dasherize/-/dasherize-2.0.0.tgz", "integrity": "sha1-bYCcnNDPe7iVLYD8hPoT1H3bEwg=" }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "dev": true, + "engines": { + "node": ">= 12" + } + }, "node_modules/debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -4994,6 +5004,29 @@ "node": ">=4.0.0" } }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, "node_modules/file-entry-cache": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", @@ -5203,6 +5236,18 @@ "node": ">= 0.12" } }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "dev": true, + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, "node_modules/formidable": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/formidable/-/formidable-1.2.2.tgz", @@ -7832,6 +7877,43 @@ "node": ">=4.0.0" } }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "dev": true, + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, "node_modules/node-mocks-http": { "version": "1.6.7", "resolved": "https://registry.npmjs.org/node-mocks-http/-/node-mocks-http-1.6.7.tgz", @@ -10853,6 +10935,15 @@ "node": ">=0.10.0" } }, + "node_modules/web-streams-polyfill": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.2.tgz", + "integrity": "sha512-3pRGuxRF5gpuZc0W+EpwQRmCD7gRqcDOMt688KmdlDAgAyaB1XlN0zq2njfDNm44XVdIouE7pZ6GzbdyH47uIQ==", + "dev": true, + "engines": { + "node": ">= 8" + } + }, "node_modules/webidl-conversions": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", diff --git a/package.json b/package.json index c5c78f5ea..c19261fc6 100644 --- a/package.json +++ b/package.json @@ -78,6 +78,7 @@ "eslint": "^7.5", "mocha": "^8.2.1", "mongodb-memory-server": "^8.11.2", + "node-fetch": "3.3.2", "node-mocks-http": "^1.6.7", "nodemon": "^2.0.3", "nyc": "^15.0.0",