From ac2f7a8745f161a5151f02c2fc8ae8eb25d137ba Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Tue, 23 Apr 2024 15:20:29 -0700 Subject: [PATCH 1/2] Reorganize fixtures for autoloading Group fixtures by endpoints. This ensures that the json response format is consistent within the folder, allowing for coordinates parsing. --- tools/integration/README.md | 1 + .../test/e2e-test-service/definitionTest.js | 29 ++++++++++++++----- .../pod-cocoapods---softbutton-0.1.0.json} | 0 tools/integration/test/testConfig.js | 6 ---- 4 files changed, 23 insertions(+), 13 deletions(-) rename tools/integration/test/fixtures/{softbutton-0.1.0.json => definitions/pod-cocoapods---softbutton-0.1.0.json} (100%) diff --git a/tools/integration/README.md b/tools/integration/README.md index 69625c9..a8b1c3e 100644 --- a/tools/integration/README.md +++ b/tools/integration/README.md @@ -17,5 +17,6 @@ - Mock responses when the production system does not have the response or needs an override, - Current harvest schema versions. This is for polling harvest results to check whether the harvest is complete. When scan tool versions are updated, this needs to be updated as well. +1. Test fixtures are grouped by endpoints at [./test/fixtures](./test/fixtures). 1. The classes used in the integration tests are located at ./lib. Tests for those tooling classes are located at ./test/lib. Run `npm test` to test the tooling classes. 1. Sample API test calls to the production deployment can be found at ./api-test. The [Insomnia collection](./api-test/clearlydefined_prod_api_test_insomnia_collection.json) is organized by endpoints (definitions, harvest, and notices). Refer to the [Swagger UI](https://api.clearlydefined.io/api-docs/#/) for detailed documentation. The `Ping/health check` can be used as the first check to see if the service is up and running. diff --git a/tools/integration/test/e2e-test-service/definitionTest.js b/tools/integration/test/e2e-test-service/definitionTest.js index 57f2b8a..93c43a6 100644 --- a/tools/integration/test/e2e-test-service/definitionTest.js +++ b/tools/integration/test/e2e-test-service/definitionTest.js @@ -4,22 +4,23 @@ const { omit, isEqual } = require('lodash') const { deepStrictEqual, strictEqual } = require('assert') const { callFetch, buildPostOpts } = require('../../lib/fetch') -const { devApiBaseUrl, prodApiBaseUrl, expectedResponses, components, definition } = require('../testConfig') +const { devApiBaseUrl, prodApiBaseUrl, components, definition } = require('../testConfig') const nock = require('nock') +const fs = require('fs') describe('Validation definitions between dev and prod', function () { this.timeout(definition.timeout) - before(() => { - expectedResponses.forEach(({ url, response }) => - nock(prodApiBaseUrl, { allowUnmocked: true }).get(url).reply(200, response) - ) - }) - //Rest a bit to avoid overloading the servers afterEach(() => new Promise(resolve => setTimeout(resolve, definition.timeout / 2))) describe('Validation between dev and prod', function () { + before(() => { + loadFixtures().forEach(([url, definition]) => + nock(prodApiBaseUrl, { allowUnmocked: true }).get(url).reply(200, definition) + ) + }) + components.forEach(coordinates => { it(`should return the same definition as prod for ${coordinates}`, () => fetchAndCompareDefinition(coordinates)) }) @@ -120,3 +121,17 @@ async function findDefinition(coordinates) { ).then(r => r.json()) return response.data.find(d => d.coordinates.revision === revision) } + +function loadFixtures() { + const location = 'test/fixtures/definitions' + return fs + .readdirSync(location) + .filter(f => f.endsWith('.json')) + .map(jsonFile => JSON.parse(fs.readFileSync(`${location}/${jsonFile}`))) + .map(definition => { + const { coordinates } = definition + const namespace = coordinates.namespace || '-' + const coordinatesString = `${coordinates.type}/${coordinates.provider}/${namespace}/${coordinates.name}/${coordinates.revision}` + return [`/definitions/${coordinatesString}`, definition] + }) +} diff --git a/tools/integration/test/fixtures/softbutton-0.1.0.json b/tools/integration/test/fixtures/definitions/pod-cocoapods---softbutton-0.1.0.json similarity index 100% rename from tools/integration/test/fixtures/softbutton-0.1.0.json rename to tools/integration/test/fixtures/definitions/pod-cocoapods---softbutton-0.1.0.json diff --git a/tools/integration/test/testConfig.js b/tools/integration/test/testConfig.js index 9721659..d720645 100644 --- a/tools/integration/test/testConfig.js +++ b/tools/integration/test/testConfig.js @@ -35,15 +35,9 @@ const components = [ // 'sourcearchive/mavencentral/org.apache.httpcomponents/httpcore/4.1' // Dev and prod have different license and scores. See https://github.com/clearlydefined/crawler/issues/533 ] -//When production response is not available or needs to be corrected, stub response from production service for testing -const expectedResponses = [ - { url: '/definitions/pod/cocoapods/-/SoftButton/0.1.0', response: require('./fixtures/softbutton-0.1.0.json') } -] - module.exports = { devApiBaseUrl, prodApiBaseUrl, - expectedResponses, components, harvest: { poll: { interval: pollingInterval, maxTime: pollingMaxTime }, From 15b78d4f6d6bc9cf3e563b9e00f09fb75405d387 Mon Sep 17 00:00:00 2001 From: Qing Tomlinson Date: Mon, 29 Apr 2024 15:21:24 -0700 Subject: [PATCH 2/2] Separate integration test from tool test and update Readme --- tools/integration/README.md | 19 +++++++++---------- tools/integration/package.json | 6 +++--- .../e2e-test-service/attachmentTest.js | 2 +- .../e2e-test-service/curationTest.js | 2 +- .../e2e-test-service/definitionTest.js | 4 ++-- .../pod-cocoapods---softbutton-0.1.0.json | 0 .../test/{ => integration}/harvestTest.js | 4 ++-- .../test/{ => integration}/testConfig.js | 0 tools/integration/test/lib/harvesterTest.js | 5 +++-- 9 files changed, 21 insertions(+), 21 deletions(-) rename tools/integration/test/{ => integration}/e2e-test-service/attachmentTest.js (97%) rename tools/integration/test/{ => integration}/e2e-test-service/curationTest.js (98%) rename tools/integration/test/{ => integration}/e2e-test-service/definitionTest.js (97%) rename tools/integration/test/{ => integration}/fixtures/definitions/pod-cocoapods---softbutton-0.1.0.json (100%) rename tools/integration/test/{ => integration}/harvestTest.js (94%) rename tools/integration/test/{ => integration}/testConfig.js (100%) diff --git a/tools/integration/README.md b/tools/integration/README.md index a8b1c3e..03a014e 100644 --- a/tools/integration/README.md +++ b/tools/integration/README.md @@ -2,21 +2,20 @@ ## Resources -1. Integration tests are located at ./test to support the integration-test.yml GitHub Action. +1. To execute integration tests, you can use the integration-test.yml GitHub Action. When running on the main branch, all tests committed to that branch will be executed. By default, the integration test suite compares the results from development with the production deployment. You can find the integration test suite at [./test/integration](./test/integration), and you can configure the development and production deployment in [./test/integration/testConfig.js](./test/integration/testConfig.js). 1. npm scripts triggered in the GitHub Action include: - - e2e-test-harvest: re-triggers harvest for the components in testConfig.js - - e2e-test-service: runs the tests in [./test/e2e-test-service](./test/e2e-test-service) folder. + - e2e-test-harvest: re-triggers and verifies the completion of the harvest for the components specified in [./test/integration/testConfig.js](./test/integration/testConfig.js) + - e2e-test-service: runs the suite of tests in [./test/integration/e2e-test-service](./test/integration/e2e-test-service) folder. These tests are organized based on the endpoints documented in [Swagger UI](https://api.clearlydefined.io/api-docs/#/). -1. The test configuration is located at [./test/testConfig.js](./test/testConfig.js). +1. The test configuration is located at [./test/integration/testConfig.js](./test/integration/testConfig.js). It contains: - Components to be harvested, - - Base URLs to the development and production systems, polling interval, and timeout, - - Mock responses when the production system does not have the response or needs an override, - - Current harvest schema versions. This is for polling harvest results to check whether the harvest is complete. When scan tool versions are updated, this needs to be updated as well. + - Base URLs for the development and production systems, along with polling interval and timeout settings, + - Current harvest schema versions. This is for polling harvest results to check whether the harvest is complete. When scan tool versions are updated, these need to be updated as well. -1. Test fixtures are grouped by endpoints at [./test/fixtures](./test/fixtures). -1. The classes used in the integration tests are located at ./lib. Tests for those tooling classes are located at ./test/lib. Run `npm test` to test the tooling classes. -1. Sample API test calls to the production deployment can be found at ./api-test. The [Insomnia collection](./api-test/clearlydefined_prod_api_test_insomnia_collection.json) is organized by endpoints (definitions, harvest, and notices). Refer to the [Swagger UI](https://api.clearlydefined.io/api-docs/#/) for detailed documentation. The `Ping/health check` can be used as the first check to see if the service is up and running. +1. Test fixtures are grouped by endpoints at [./test/integration/fixtures](./test/integration/fixtures). You can use these fixtures to override responses from the production system when necessary. +1. The classes used in the integration tests are located at [./lib](./lib). Tests for those tooling classes are located at ./test/lib. Run `npm test` to test the tooling classes. +1. Sample API test calls to the production deployment can be found at [./api-test](./api-test). The [Insomnia collection](./api-test/clearlydefined_prod_api_test_insomnia_collection.json) is organized by endpoints (definitions, harvest, and notices). Refer to the [Swagger UI](https://api.clearlydefined.io/api-docs/#/) for detailed documentation. The `Ping/health check` can be used as the first check to see if the service is up and running. diff --git a/tools/integration/package.json b/tools/integration/package.json index bf0833f..c2420d5 100644 --- a/tools/integration/package.json +++ b/tools/integration/package.json @@ -4,8 +4,8 @@ "description": "Integration tests and related tools for ClearlyDefined", "scripts": { "test": "npm run mocha && npm run lint", - "e2e-test-harvest": "mocha test/harvestTest.js", - "e2e-test-service": "mocha --exit \"test/e2e-test-service/**/*.js\"", + "e2e-test-harvest": "mocha test/integration/harvestTest.js", + "e2e-test-service": "mocha --exit \"test/integration/e2e-test-service/**/*.js\"", "mocha": "mocha --exit \"test/lib/**/*.js\"", "lint": "npm run prettier:check && npm run eslint", "lint:fix": "npm run prettier:write && npm run eslint:fix", @@ -26,4 +26,4 @@ "prettier": "^3.2.5", "sinon": "^17.0.1" } -} +} \ No newline at end of file diff --git a/tools/integration/test/e2e-test-service/attachmentTest.js b/tools/integration/test/integration/e2e-test-service/attachmentTest.js similarity index 97% rename from tools/integration/test/e2e-test-service/attachmentTest.js rename to tools/integration/test/integration/e2e-test-service/attachmentTest.js index c2eabcf..295cc82 100644 --- a/tools/integration/test/e2e-test-service/attachmentTest.js +++ b/tools/integration/test/integration/e2e-test-service/attachmentTest.js @@ -1,7 +1,7 @@ // (c) Copyright 2024, SAP SE and ClearlyDefined contributors. Licensed under the MIT license. // SPDX-License-Identifier: MIT -const { callFetch } = require('../../lib/fetch') +const { callFetch } = require('../../../lib/fetch') const { devApiBaseUrl, prodApiBaseUrl, components, definition } = require('../testConfig') const { strictEqual } = require('assert') diff --git a/tools/integration/test/e2e-test-service/curationTest.js b/tools/integration/test/integration/e2e-test-service/curationTest.js similarity index 98% rename from tools/integration/test/e2e-test-service/curationTest.js rename to tools/integration/test/integration/e2e-test-service/curationTest.js index 69515a1..fd95362 100644 --- a/tools/integration/test/e2e-test-service/curationTest.js +++ b/tools/integration/test/integration/e2e-test-service/curationTest.js @@ -2,7 +2,7 @@ // SPDX-License-Identifier: MIT const { deepStrictEqual, strictEqual, ok } = require('assert') -const { callFetch, buildPostOpts } = require('../../lib/fetch') +const { callFetch, buildPostOpts } = require('../../../lib/fetch') const { devApiBaseUrl, components, definition } = require('../testConfig') describe('Validate curation', function () { diff --git a/tools/integration/test/e2e-test-service/definitionTest.js b/tools/integration/test/integration/e2e-test-service/definitionTest.js similarity index 97% rename from tools/integration/test/e2e-test-service/definitionTest.js rename to tools/integration/test/integration/e2e-test-service/definitionTest.js index 93c43a6..1b454fc 100644 --- a/tools/integration/test/e2e-test-service/definitionTest.js +++ b/tools/integration/test/integration/e2e-test-service/definitionTest.js @@ -3,7 +3,7 @@ const { omit, isEqual } = require('lodash') const { deepStrictEqual, strictEqual } = require('assert') -const { callFetch, buildPostOpts } = require('../../lib/fetch') +const { callFetch, buildPostOpts } = require('../../../lib/fetch') const { devApiBaseUrl, prodApiBaseUrl, components, definition } = require('../testConfig') const nock = require('nock') const fs = require('fs') @@ -123,7 +123,7 @@ async function findDefinition(coordinates) { } function loadFixtures() { - const location = 'test/fixtures/definitions' + const location = 'test/integration/fixtures/definitions' return fs .readdirSync(location) .filter(f => f.endsWith('.json')) diff --git a/tools/integration/test/fixtures/definitions/pod-cocoapods---softbutton-0.1.0.json b/tools/integration/test/integration/fixtures/definitions/pod-cocoapods---softbutton-0.1.0.json similarity index 100% rename from tools/integration/test/fixtures/definitions/pod-cocoapods---softbutton-0.1.0.json rename to tools/integration/test/integration/fixtures/definitions/pod-cocoapods---softbutton-0.1.0.json diff --git a/tools/integration/test/harvestTest.js b/tools/integration/test/integration/harvestTest.js similarity index 94% rename from tools/integration/test/harvestTest.js rename to tools/integration/test/integration/harvestTest.js index 0af34f9..ba04dc4 100644 --- a/tools/integration/test/harvestTest.js +++ b/tools/integration/test/integration/harvestTest.js @@ -2,8 +2,8 @@ // SPDX-License-Identifier: MIT const { components, devApiBaseUrl, harvest } = require('./testConfig') -const Poller = require('../lib/poller') -const Harvester = require('../lib/harvester') +const Poller = require('../../lib/poller') +const Harvester = require('../../lib/harvester') const { strictEqual } = require('assert') describe('Tests for harvesting different components', function () { diff --git a/tools/integration/test/testConfig.js b/tools/integration/test/integration/testConfig.js similarity index 100% rename from tools/integration/test/testConfig.js rename to tools/integration/test/integration/testConfig.js diff --git a/tools/integration/test/lib/harvesterTest.js b/tools/integration/test/lib/harvesterTest.js index eb4d048..6eb61ba 100644 --- a/tools/integration/test/lib/harvesterTest.js +++ b/tools/integration/test/lib/harvesterTest.js @@ -4,9 +4,10 @@ const { strictEqual, ok, deepStrictEqual } = require('assert') const Poller = require('../../lib/poller') const Harvester = require('../../lib/harvester') -const { devApiBaseUrl, harvestToolVersions } = require('../testConfig') const sinon = require('sinon') +const devApiBaseUrl = 'localhost:4000' + describe('Tests for Harvester', function () { const coordinates = 'nuget/nuget/-/NuGet.Protocol/6.7.1' @@ -14,7 +15,7 @@ describe('Tests for Harvester', function () { let fetchStub beforeEach(function () { fetchStub = sinon.stub() - harvester = new Harvester(devApiBaseUrl, harvestToolVersions, fetchStub) + harvester = new Harvester(devApiBaseUrl, undefined, fetchStub) }) describe('Verify api calls in harvest and fetchHarvestResult', function () {