diff --git a/tools/integration/README.md b/tools/integration/README.md index 03a014e..96990f1 100644 --- a/tools/integration/README.md +++ b/tools/integration/README.md @@ -14,7 +14,7 @@ - Components to be harvested, - Base URLs for the development and production systems, along with polling interval and timeout settings, - - Current harvest schema versions. This is for polling harvest results to check whether the harvest is complete. When scan tool versions are updated, these need to be updated as well. + - Current harvest tools. This is used for polling harvest results to check whether the harvest is complete. When scan tools are added or removed during the harvest process, this list needs to be updated as well. 1. Test fixtures are grouped by endpoints at [./test/integration/fixtures](./test/integration/fixtures). You can use these fixtures to override responses from the production system when necessary. 1. The classes used in the integration tests are located at [./lib](./lib). Tests for those tooling classes are located at ./test/lib. Run `npm test` to test the tooling classes. diff --git a/tools/integration/lib/harvestResultFetcher.js b/tools/integration/lib/harvestResultFetcher.js new file mode 100644 index 0000000..a4626b3 --- /dev/null +++ b/tools/integration/lib/harvestResultFetcher.js @@ -0,0 +1,78 @@ +// (c) Copyright 2024, SAP SE and ClearlyDefined contributors. Licensed under the MIT license. +// SPDX-License-Identifier: MIT +const { callFetch } = require('./fetch') + +const defaultTools = ['licensee', 'reuse', 'scancode'] + +class HarvestResultFetcher { + constructor(apiBaseUrl, coordinates, fetch = callFetch) { + this.apiBaseUrl = apiBaseUrl + this._fetch = fetch + this._coordinates = coordinates + } + + async fetchToolVersions(tools = defaultTools) { + const listHarvestResultApi = `${this.apiBaseUrl}/harvest/${this._coordinates}?form=list` + const harvestResultUrls = await this._fetch(listHarvestResultApi).then(r => r.json()) + return tools.flatMap(tool => + harvestResultUrls + .filter(url => url.includes(`/${tool}/`)) + .map(url => url.substring(`${this._coordinates}/${tool}/`.length)) + .map(version => [tool, version]) + ) + } + + async _pollForCompletion(poller) { + try { + const completed = await poller.poll() + console.log(`Completed ${this._coordinates}: ${completed}`) + return completed + } catch (error) { + if (error.code === 'ECONNREFUSED') throw error + console.log(`Error polling for ${this._coordinates}: ${error}`) + return false + } + } + + async pollForToolVersionsComplete(poller, startTime, tools) { + const statuses = new Map() + poller.with(async () => { + const toolVersions = await this.fetchToolVersions(tools) + return this.isHarvestComplete(toolVersions, startTime, statuses) + }) + const completed = await this._pollForCompletion(poller) + if (!completed) throw new Error(`Schema versions not detected`) + return [...statuses.entries()].map(([k, v]) => [k, v.toolVersion]) + } + + async pollForHarvestComplete(poller, toolVersions, startTime) { + const statuses = new Map() + poller.with(async () => this.isHarvestComplete(toolVersions, startTime, statuses)) + return this._pollForCompletion(poller) + } + + async isHarvestComplete(toolVersions, startTime, statuses = new Map()) { + const harvestChecks = (toolVersions || []).map(async ([tool, toolVersion]) => { + const completed = statuses.get(tool)?.completed || (await this.isHarvestedbyTool(tool, toolVersion, startTime)) + if (completed) statuses.set(tool, { toolVersion, completed }) + return tool + }) + return Promise.all(harvestChecks).then(tools => tools.every(tool => statuses.get(tool)?.completed)) + } + + async isHarvestedbyTool(tool, toolVersion, startTime = 0) { + const harvested = await this.fetchHarvestResult(tool, toolVersion) + if (!harvested._metadata) return false + const fetchedAt = new Date(harvested._metadata.fetchedAt) + console.log(`${this._coordinates} ${tool}, ${toolVersion} fetched at ${fetchedAt}`) + return fetchedAt.getTime() > startTime + } + + async fetchHarvestResult(tool, toolVersion) { + return this._fetch(`${this.apiBaseUrl}/harvest/${this._coordinates}/${tool}/${toolVersion}?form=raw`).then(r => + r.headers.get('Content-Length') === '0' ? Promise.resolve({}) : r.json() + ) + } +} + +module.exports = HarvestResultFetcher diff --git a/tools/integration/lib/harvester.js b/tools/integration/lib/harvester.js index d5754e5..2b8dd56 100644 --- a/tools/integration/lib/harvester.js +++ b/tools/integration/lib/harvester.js @@ -2,28 +2,22 @@ // SPDX-License-Identifier: MIT const { callFetch, buildPostOpts } = require('./fetch') - -//The versions correspond to the schema versions of the tools which are used in /harvest/{type}/{provider}/{namespace}/{name}/{revision}/{tool}/{toolVersion} -//See https://api.clearlydefined.io/api-docs/#/harvest/get_harvest__type___provider___namespace___name___revision___tool___toolVersion_ -const defaultToolChecks = [ - ['licensee', '9.14.0'], - ['scancode', '32.3.0'], - ['reuse', '3.2.1'] -] +const HarvestResultFetcher = require('./harvestResultFetcher') class Harvester { constructor(apiBaseUrl, harvestToolChecks, fetch = callFetch) { this.apiBaseUrl = apiBaseUrl - this.harvestToolChecks = harvestToolChecks || defaultToolChecks + this._harvestToolChecks = harvestToolChecks this._fetch = fetch } async harvest(components, reharvest = false) { + if (components.length === 0) return return await this._fetch(`${this.apiBaseUrl}/harvest`, buildPostOpts(this._buildPostJson(components, reharvest))) } _buildPostJson(components, reharvest) { - const tool = this.harvestToolChecks.length === 1 ? this.harvestToolChecks[0][0] : 'component' + const tool = this._harvestToolChecks?.length === 1 ? this._harvestToolChecks[0][0] : 'component' return components.map(coordinates => { const result = { tool, coordinates } if (reharvest) result.policy = 'always' @@ -32,6 +26,7 @@ class Harvester { } async pollForCompletion(components, poller, startTime) { + if (!this._harvestToolChecks) throw new Error('Harvest tool checks not set') const status = new Map() for (const coordinates of components) { const completed = await this._pollForOneCompletion(coordinates, poller, startTime) @@ -39,45 +34,44 @@ class Harvester { } for (const coordinates of components) { - const completed = - status.get(coordinates) || (await this.isHarvestComplete(coordinates, startTime).catch(() => false)) + const completed = status.get(coordinates) || (await this._isHarvestComplete(coordinates, startTime)) status.set(coordinates, completed) } return status } async _pollForOneCompletion(coordinates, poller, startTime) { - try { - const completed = await poller.poll(async () => this.isHarvestComplete(coordinates, startTime)) - console.log(`Completed ${coordinates}: ${completed}`) - return completed - } catch (error) { - if (error.code === 'ECONNREFUSED') throw error - console.log(`Error polling for ${coordinates}: ${error}`) - return false - } + return this.resultChecker(coordinates).pollForHarvestComplete(poller, this._harvestToolChecks, startTime) } - async isHarvestComplete(coordinates, startTime) { - const harvestChecks = this.harvestToolChecks.map(([tool, toolVersion]) => - this.isHarvestedbyTool(coordinates, tool, toolVersion, startTime) - ) - - return Promise.all(harvestChecks).then(results => results.every(r => r)) + async _isHarvestComplete(coordinates, startTime) { + return this.resultChecker(coordinates) + .isHarvestComplete(this._harvestToolChecks, startTime) + .catch(error => { + console.log(`Error polling for ${coordinates} completion: ${error}`) + return false + }) } - async isHarvestedbyTool(coordinates, tool, toolVersion, startTime = 0) { - const harvested = await this.fetchHarvestResult(coordinates, tool, toolVersion) - if (!harvested._metadata) return false - const fetchedAt = new Date(harvested._metadata.fetchedAt) - console.log(`${coordinates} ${tool}, ${toolVersion} fetched at ${fetchedAt}`) - return fetchedAt.getTime() > startTime - } + async detectSchemaVersions(component, poller, tools) { + if (!component) throw new Error('Component not set') + const startTime = Date.now() + //make sure that we have one entire set of harvest results (old or new) + await this.harvest([component]) + //trigger a reharvest to overwrite the old result, so we can verify the timestamp is new for completion + await this.harvest([component], true) - async fetchHarvestResult(coordinates, tool, toolVersion) { - return this._fetch(`${this.apiBaseUrl}/harvest/${coordinates}/${tool}/${toolVersion}?form=raw`).then(r => - r.headers.get('Content-Length') === '0' ? Promise.resolve({}) : r.json() + const detectedToolVersions = await this.resultChecker(component).pollForToolVersionsComplete( + poller, + startTime, + tools ) + console.log(`Detected schema versions: ${detectedToolVersions}`) + this._harvestToolChecks = detectedToolVersions + } + + resultChecker(coordinates) { + return new HarvestResultFetcher(this.apiBaseUrl, coordinates, this._fetch) } } diff --git a/tools/integration/lib/poller.js b/tools/integration/lib/poller.js index db013e0..b08ea45 100644 --- a/tools/integration/lib/poller.js +++ b/tools/integration/lib/poller.js @@ -7,11 +7,17 @@ class Poller { this.maxTime = maxTime } - async poll(activity) { + with(activity) { + this._activity = activity + return this + } + + async poll() { + if (typeof this._activity !== 'function') throw new Error('Activity not set') let counter = 0 while (counter * this.interval < this.maxTime) { console.log(`Polling ${counter}`) - const isDone = await activity() + const isDone = await this._activity() if (isDone) return true await new Promise(resolve => setTimeout(resolve, this.interval)) counter++ diff --git a/tools/integration/test/integration/e2e-test-service/curationTest.js b/tools/integration/test/integration/e2e-test-service/curationTest.js index fd95362..08a0122 100644 --- a/tools/integration/test/integration/e2e-test-service/curationTest.js +++ b/tools/integration/test/integration/e2e-test-service/curationTest.js @@ -3,7 +3,7 @@ const { deepStrictEqual, strictEqual, ok } = require('assert') const { callFetch, buildPostOpts } = require('../../../lib/fetch') -const { devApiBaseUrl, components, definition } = require('../testConfig') +const { devApiBaseUrl, definition } = require('../testConfig') describe('Validate curation', function () { this.timeout(definition.timeout) @@ -11,7 +11,7 @@ describe('Validate curation', function () { //Rest a bit to avoid overloading the servers afterEach(() => new Promise(resolve => setTimeout(resolve, definition.timeout / 2))) - const coordinates = components[0] + const coordinates = 'maven/mavencentral/org.apache.httpcomponents/httpcore/4.4.16' describe('Propose curation', function () { const [type, provider, namespace, name, revision] = coordinates.split('/') diff --git a/tools/integration/test/integration/e2e-test-service/definitionTest.js b/tools/integration/test/integration/e2e-test-service/definitionTest.js index c58e0eb..5fc5688 100644 --- a/tools/integration/test/integration/e2e-test-service/definitionTest.js +++ b/tools/integration/test/integration/e2e-test-service/definitionTest.js @@ -118,8 +118,10 @@ function filesToMap(result) { async function findDefinition(coordinates) { const [type, provider, namespace, name, revision] = coordinates.split('/') + let coordinatesString = `type=${type}&provider=${provider}&name=${name}` + coordinatesString += namespace && namespace !== '-' ? `&namespace=${namespace}` : '' const response = await callFetch( - `${devApiBaseUrl}/definitions?type=${type}&provider=${provider}&namespace=${namespace}&name=${name}&sortDesc=true&sort=revision` + `${devApiBaseUrl}/definitions?${coordinatesString}&sortDesc=true&sort=revision` ).then(r => r.json()) return response.data.find(d => d.coordinates.revision === revision) } diff --git a/tools/integration/test/integration/harvestTest.js b/tools/integration/test/integration/harvestTest.js index ba04dc4..cf58422 100644 --- a/tools/integration/test/integration/harvestTest.js +++ b/tools/integration/test/integration/harvestTest.js @@ -19,8 +19,14 @@ describe('Tests for harvesting different components', function () { }) async function harvestTillCompletion(components) { - const { harvestSchemaVersions, poll } = harvest - const harvester = new Harvester(devApiBaseUrl, harvestSchemaVersions) + if (components.length === 0) return new Map() + + const { poll, tools } = harvest + const harvester = new Harvester(devApiBaseUrl) + + const oneComponent = components.shift() + const versionPoller = new Poller(poll.interval / 5, poll.maxTime) + await harvester.detectSchemaVersions(oneComponent, versionPoller, tools) //make sure that we have one entire set of harvest results (old or new) console.log('Ensure harvest results exist before starting tests') diff --git a/tools/integration/test/integration/testConfig.js b/tools/integration/test/integration/testConfig.js index 79f607b..19106c4 100644 --- a/tools/integration/test/integration/testConfig.js +++ b/tools/integration/test/integration/testConfig.js @@ -5,19 +5,14 @@ const devApiBaseUrl = 'https://dev-api.clearlydefined.io' const prodApiBaseUrl = 'https://api.clearlydefined.io' const pollingInterval = 1000 * 60 * 5 // 5 minutes -const pollingMaxTime = 1000 * 60 * 30 // 30 minutes +const pollingMaxTime = 1000 * 60 * 60 // 60 minutes -//Havest results to check for harvest completeness -//The versions correspond to the schema versions of the tools which are used in /harvest/{type}/{provider}/{namespace}/{name}/{revision}/{tool}/{toolVersion} -//See https://api.clearlydefined.io/api-docs/#/harvest/get_harvest__type___provider___namespace___name___revision___tool___toolVersion_ -const harvestSchemaVersions = [ - ['licensee', '9.14.0'], - ['scancode', '32.3.0'], - ['reuse', '3.2.1'] -] +//Havest tools to check for harvest completeness +const harvestTools = ['licensee', 'reuse', 'scancode'] //Components to test const components = [ + 'pypi/pypi/-/platformdirs/4.2.0', //Keep this as the first element to test, it is relatively small 'maven/mavencentral/org.apache.httpcomponents/httpcore/4.4.16', 'maven/mavengoogle/android.arch.lifecycle/common/1.0.1', 'maven/gradleplugin/io.github.lognet/grpc-spring-boot-starter-gradle-plugin/4.6.0', @@ -26,7 +21,6 @@ const components = [ 'npm/npmjs/-/redis/0.1.0', 'git/github/ratatui-org/ratatui/bcf43688ec4a13825307aef88f3cdcd007b32641', 'gem/rubygems/-/sorbet/0.5.11226', - 'pypi/pypi/-/platformdirs/4.2.0', 'pypi/pypi/-/sdbus/0.12.0', 'go/golang/rsc.io/quote/v1.3.0', 'nuget/nuget/-/NuGet.Protocol/6.7.1', @@ -42,9 +36,9 @@ module.exports = { prodApiBaseUrl, components, harvest: { - poll: { interval: pollingInterval, maxTime: pollingMaxTime }, - harvestSchemaVersions, - timeout: 1000 * 60 * 60 * 2 // 2 hours for harvesting all the components + poll: { interval: pollingInterval, maxTime: pollingMaxTime }, // for each component + tools: harvestTools, + timeout: 1000 * 60 * 60 * 4 // 4 hours for harvesting all the components }, definition: { timeout: 1000 * 10 // for each component diff --git a/tools/integration/test/lib/harvesterResultFetcherTest.js b/tools/integration/test/lib/harvesterResultFetcherTest.js new file mode 100644 index 0000000..365ee38 --- /dev/null +++ b/tools/integration/test/lib/harvesterResultFetcherTest.js @@ -0,0 +1,186 @@ +// (c) Copyright 2024, SAP SE and ClearlyDefined contributors. Licensed under the MIT license. +// SPDX-License-Identifier: MIT + +const { strictEqual, ok, deepStrictEqual } = require('assert') +const HarvestResultFetcher = require('../../lib/harvestResultFetcher') +const sinon = require('sinon') +const Poller = require('../../lib/poller') + +const apiBaseUrl = 'http://localhost:4000' +const defaultToolChecks = [ + ['licensee', '9.14.0'], + ['scancode', '30.3.0'], + ['reuse', '3.2.1'] +] +describe('HarvestResultFetcher', function () { + const coordinates = 'pypi/pypi/-/platformdirs/4.2.0' + let resultMonitor + + describe('fetchHarvestResult', function () { + let fetchStub + beforeEach(function () { + fetchStub = sinon.stub() + resultMonitor = new HarvestResultFetcher(apiBaseUrl, coordinates, fetchStub) + }) + + it('should call the correct harvest api in fetchHarvestResult', async function () { + fetchStub.resolves(new Response(JSON.stringify({ test: true }))) + await resultMonitor.fetchHarvestResult('licensee', '9.14.0') + ok(fetchStub.calledOnce) + strictEqual(fetchStub.getCall(0).args[0], `${apiBaseUrl}/harvest/${coordinates}/licensee/9.14.0?form=raw`) + }) + + it('should handle in fetchHarvestResult when harvest result is not ready', async function () { + const response = new Response() + response.headers.set('Content-Length', '0') + fetchStub.resolves(response) + const result = await resultMonitor.fetchHarvestResult('licensee', '9.14.0') + deepStrictEqual(result, {}) + }) + }) + + describe('isHarvested', function () { + beforeEach(function () { + resultMonitor = new HarvestResultFetcher(apiBaseUrl, coordinates) + }) + it('should detect when a scan tool result for component is available', async function () { + sinon.stub(resultMonitor, 'fetchHarvestResult').resolves(metadata()) + const result = await resultMonitor.isHarvestedbyTool('licensee', '9.14.0') + strictEqual(result, true) + }) + + it('should detect when component is completely harvested', async function () { + sinon.stub(resultMonitor, 'fetchHarvestResult').resolves(metadata()) + const result = await resultMonitor.isHarvestComplete(defaultToolChecks) + strictEqual(result, true) + }) + + it('should detect whether component is harvested after a timestamp', async function () { + const date = '2023-01-01T00:00:00.000Z' + sinon.stub(resultMonitor, 'fetchHarvestResult').resolves(metadata(date)) + const result = await resultMonitor.isHarvestComplete(defaultToolChecks, Date.now()) + strictEqual(result, false) + }) + + it('should handle when harvest result is not ready', async function () { + sinon.stub(resultMonitor, 'fetchHarvestResult').resolves({}) + const result = await resultMonitor.isHarvestComplete(defaultToolChecks) + strictEqual(result, false) + }) + + it('should only call harvest if it is not completed', async function () { + const stub = sinon.stub(resultMonitor, 'fetchHarvestResult').rejects(new Error('failed')) + const status = new Map([['licensee', { toolVersion: '9.14.0', completed: true }]]) + const result = await resultMonitor.isHarvestComplete([['licensee', '9.14.0']], undefined, status) + strictEqual(result, true) + ok(!stub.calledOnce) + }) + }) + + describe('pollForHarvestComplete', function () { + const interval = 10 * 1 + const maxTime = 10 * 2 + let poller + + beforeEach(function () { + poller = new Poller(interval, maxTime) + resultMonitor = new HarvestResultFetcher(apiBaseUrl, coordinates) + }) + + it('should poll for completion if results exist', async function () { + sinon.stub(resultMonitor, 'fetchHarvestResult').resolves(metadata()) + const status = await resultMonitor.pollForHarvestComplete(poller, defaultToolChecks) + strictEqual(status, true) + }) + + it('should poll for completion if results are stale', async function () { + const date = '2023-01-01T00:00:00.000Z' + sinon.stub(resultMonitor, 'fetchHarvestResult').resolves(metadata(date)) + const status = await resultMonitor.pollForHarvestComplete(poller, defaultToolChecks, Date.now()) + strictEqual(status, false) + }) + + it('should handle an error', async function () { + sinon.stub(resultMonitor, 'fetchHarvestResult').rejects(new Error('failed')) + const status = await resultMonitor.pollForHarvestComplete(poller, defaultToolChecks, Date.now()) + strictEqual(status, false) + }) + + it('should only call harvest if it is not yet completed', async function () { + let callCount = 0 + const stub = sinon.stub(resultMonitor, 'fetchHarvestResult').callsFake(() => (callCount++ > 0 ? metadata() : {})) + const status = await resultMonitor.pollForHarvestComplete(poller, defaultToolChecks) + strictEqual(status, true) + strictEqual(stub.callCount, 4) + }) + }) + + describe('fetchToolVersions', function () { + let fetchStub + beforeEach(function () { + fetchStub = sinon.stub() + resultMonitor = new HarvestResultFetcher(apiBaseUrl, coordinates, fetchStub) + }) + + it('should call the correct harvest api', async function () { + fetchStub.resolves(new Response(JSON.stringify([]))) + await resultMonitor.fetchToolVersions() + ok(fetchStub.calledOnce) + strictEqual(fetchStub.getCall(0).args[0], `${apiBaseUrl}/harvest/${coordinates}?form=list`) + }) + + it('should process the result correctly', async function () { + const harvestResults = [ + 'pypi/pypi/-/platformdirs/4.2.0/clearlydefined/1.3.1', + 'pypi/pypi/-/platformdirs/4.2.0/licensee/9.14.0', + 'pypi/pypi/-/platformdirs/4.2.0/reuse/3.2.1', + 'pypi/pypi/-/platformdirs/4.2.0/reuse/3.2.2', + 'pypi/pypi/-/platformdirs/4.2.0/scancode/30.3.0' + ] + fetchStub.resolves(new Response(JSON.stringify(harvestResults))) + const toolVersions = await resultMonitor.fetchToolVersions() + deepStrictEqual(toolVersions, [ + ['licensee', '9.14.0'], + ['reuse', '3.2.1'], + ['reuse', '3.2.2'], + ['scancode', '30.3.0'] + ]) + }) + }) + + describe('pollForToolVersionsComplete', function () { + const interval = 10 * 1 + const maxTime = 10 * 2 + let poller + + beforeEach(function () { + poller = new Poller(interval, maxTime) + resultMonitor = new HarvestResultFetcher(apiBaseUrl, coordinates) + }) + + it('should process the result correctly', async function () { + sinon.stub(resultMonitor, 'fetchToolVersions').resolves([ + ['licensee', '9.14.0'], + ['reuse', '3.2.1'], + ['reuse', '3.2.2'], + ['scancode', '30.3.0'] + ]) + sinon + .stub(resultMonitor, 'isHarvestedbyTool') + .withArgs('licensee', '9.14.0') + .resolves(true) + .withArgs('reuse', '3.2.1') + .resolves(true) + .withArgs('scancode', '30.3.0') + .resolves(true) + const toolVersions = await resultMonitor.pollForToolVersionsComplete(poller, Date.now()) + deepStrictEqual(toolVersions, [ + ['licensee', '9.14.0'], + ['reuse', '3.2.1'], + ['scancode', '30.3.0'] + ]) + }) + }) +}) + +const metadata = date => ({ _metadata: { fetchedAt: date || new Date().toISOString() } }) diff --git a/tools/integration/test/lib/harvesterTest.js b/tools/integration/test/lib/harvesterTest.js index 6eb61ba..aad191e 100644 --- a/tools/integration/test/lib/harvesterTest.js +++ b/tools/integration/test/lib/harvesterTest.js @@ -6,19 +6,24 @@ const Poller = require('../../lib/poller') const Harvester = require('../../lib/harvester') const sinon = require('sinon') -const devApiBaseUrl = 'localhost:4000' - +const apiBaseUrl = 'http://localhost:4000' +const defaultToolChecks = [ + ['licensee', '9.14.0'], + ['scancode', '30.3.0'], + ['reuse', '3.2.1'] +] describe('Tests for Harvester', function () { const coordinates = 'nuget/nuget/-/NuGet.Protocol/6.7.1' let harvester let fetchStub - beforeEach(function () { - fetchStub = sinon.stub() - harvester = new Harvester(devApiBaseUrl, undefined, fetchStub) - }) - describe('Verify api calls in harvest and fetchHarvestResult', function () { + describe('Verify api calls in harvest', function () { + beforeEach(function () { + fetchStub = sinon.stub() + harvester = new Harvester(apiBaseUrl, defaultToolChecks, fetchStub) + }) + it('should call correct api with the correct payload in harvest', async function () { await harvester.harvest([coordinates], false) const expectedPayload = { @@ -27,93 +32,41 @@ describe('Tests for Harvester', function () { body: JSON.stringify([{ tool: 'component', coordinates }]) } ok(fetchStub.calledOnce) - strictEqual(fetchStub.getCall(0).args[0], `${devApiBaseUrl}/harvest`) - deepStrictEqual(fetchStub.getCall(0).args[1], expectedPayload) - }) - - it('should call the correct harvest api in fetchHarvestResult', async function () { - fetchStub.resolves(new Response(JSON.stringify({ test: true }))) - await harvester.fetchHarvestResult(coordinates, 'licensee', '9.14.0') - ok(fetchStub.calledOnce) - strictEqual(fetchStub.getCall(0).args[0], `${devApiBaseUrl}/harvest/${coordinates}/licensee/9.14.0?form=raw`) - }) - - it('should handle in fetchHarvestResult when harvest result is not ready', async function () { - const response = new Response() - response.headers.set('Content-Length', '0') - fetchStub.resolves(response) - const result = await harvester.fetchHarvestResult(coordinates, 'licensee', '9.14.0') - deepStrictEqual(result, {}) + strictEqual(fetchStub.getCall(0).args[0], `${apiBaseUrl}/harvest`, 'Incorrect URL') + deepStrictEqual(fetchStub.getCall(0).args[1], expectedPayload, 'Incorrect payload') }) }) - describe('isHarvested', function () { + describe('pollForCompletion', function () { + const coordinates = 'nuget/nuget/-/NuGet.Protocol/6.7.1' + const interval = 10 * 1 + const maxTime = 10 * 2 + let poller + beforeEach(function () { - harvester = new Harvester(devApiBaseUrl) - }) - it('should detect when a scan tool result for component is available', async function () { - sinon.stub(harvester, 'fetchHarvestResult').resolves(metadata()) - const result = await harvester.isHarvestedbyTool(coordinates, 'licensee', '9.14.0') - strictEqual(result, true) + poller = new Poller(interval, maxTime) + harvester = new Harvester(apiBaseUrl, defaultToolChecks, () => fetchStub()) }) - it('should detect when component is completely harvested', async function () { - sinon.stub(harvester, 'fetchHarvestResult').resolves(metadata()) - const result = await harvester.isHarvestComplete(coordinates) - strictEqual(result, true) + it('should poll for completion if results exist', async function () { + fetchStub = () => Promise.resolve(new Response(createBody())) + const status = await harvester.pollForCompletion([coordinates], poller) + strictEqual(status.get(coordinates), true) }) - it('should detect whether component is harvested after a timestamp', async function () { + it('should poll for completion if results are stale', async function () { const date = '2023-01-01T00:00:00.000Z' - sinon.stub(harvester, 'fetchHarvestResult').resolves(metadata(date)) - const result = await harvester.isHarvestComplete(coordinates, Date.now()) - strictEqual(result, false) + fetchStub = () => Promise.resolve(new Response(createBody(date))) + const status = await harvester.pollForCompletion([coordinates], poller, Date.now()) + strictEqual(status.get(coordinates), false) }) - it('should handle when harvest result is not ready', async function () { - sinon.stub(harvester, 'fetchHarvestResult').resolves({}) - const result = await harvester.isHarvestComplete(coordinates) - strictEqual(result, false) + it('should handle an error', async function () { + fetchStub = () => Promise.reject(new Error('failed')) + const status = await harvester.pollForCompletion([coordinates], poller, Date.now()) + strictEqual(status.get(coordinates), false) }) }) }) -describe('Integration tests for Harvester and Poller', function () { - const coordinates = 'nuget/nuget/-/NuGet.Protocol/6.7.1' - const interval = 10 * 1 - const maxTime = 10 * 2 - let poller - let harvester - - beforeEach(function () { - harvester = new Harvester(devApiBaseUrl) - poller = new Poller(interval, maxTime) - }) - - it('should poll until max time is reached', async function () { - sinon.stub(harvester, 'fetchHarvestResult').resolves({}) - const result = await poller.poll(async () => await harvester.isHarvestComplete(coordinates, Date.now())) - strictEqual(result, false) - }) - - it('should poll for completion if results exist', async function () { - sinon.stub(harvester, 'fetchHarvestResult').resolves(metadata()) - const status = await harvester.pollForCompletion([coordinates], poller) - strictEqual(status.get(coordinates), true) - }) - - it('should poll for completion if results are stale', async function () { - const date = '2023-01-01T00:00:00.000Z' - sinon.stub(harvester, 'fetchHarvestResult').resolves(metadata(date)) - const status = await harvester.pollForCompletion([coordinates], poller, Date.now()) - strictEqual(status.get(coordinates), false) - }) - - it('should handle an error', async function () { - sinon.stub(harvester, 'fetchHarvestResult').rejects(new Error('failed')) - const status = await harvester.pollForCompletion([coordinates], poller, Date.now()) - strictEqual(status.get(coordinates), false) - }) -}) - -const metadata = date => ({ _metadata: { fetchedAt: date || new Date().toISOString() } }) +const createBody = date => JSON.stringify({ _metadata: { fetchedAt: date || new Date().toISOString() } }) diff --git a/tools/integration/test/lib/pollerTest.js b/tools/integration/test/lib/pollerTest.js index 1a1d16d..4b3a38f 100644 --- a/tools/integration/test/lib/pollerTest.js +++ b/tools/integration/test/lib/pollerTest.js @@ -16,28 +16,28 @@ describe('Unit tests for Poller', function () { it('should poll until max time reached', async function () { const activity = sinon.stub().resolves(false) - const result = await poller.poll(activity) + const result = await poller.with(activity).poll() strictEqual(activity.callCount, 2) strictEqual(result, false) }) it('should handle when activity is done', async function () { const activity = sinon.stub().resolves(true) - const result = await poller.poll(activity) + const result = await poller.with(activity).poll() strictEqual(activity.callCount, 1) strictEqual(result, true) }) it('should continue to poll until activity is done', async function () { const activity = sinon.stub().resolves(false).onCall(1).resolves(true) - const result = await poller.poll(activity) + const result = await poller.with(activity).poll() strictEqual(activity.callCount, 2) strictEqual(result, true) }) it('should poll once for one time poller', async function () { const activity = sinon.stub().resolves(false) - const result = await new Poller(1, 1).poll(activity) + const result = await new Poller(1, 1).with(activity).poll() strictEqual(activity.callCount, 1) strictEqual(result, false) })