diff --git a/src/scripts/import-projects.ts b/src/scripts/import-projects.ts index 342d1acd..ea255c0a 100644 --- a/src/scripts/import-projects.ts +++ b/src/scripts/import-projects.ts @@ -4,6 +4,7 @@ import { loadFile } from '../load-file'; import { importTargets, pollImportUrls } from '../lib'; import { Project, ImportTarget } from '../lib/types'; import { getLoggingPath } from '../lib/get-logging-path'; +import { getConcurrentImportsNumber } from '../lib/get-concurrent-imports-number'; const debug = debugLib('snyk:import-projects-script'); @@ -48,10 +49,25 @@ export async function ImportProjects( throw new Error(`Failed to parse targets from ${fileName}`); } debug(`Loaded ${targets.length} targets to import`); + const concurrentTargets = getConcurrentImportsNumber(); + const projects: Project[] = []; //TODO: validation? const filteredTargets = await filterOutImportedTargets(targets, loggingPath); - const pollingUrls = await importTargets(filteredTargets, loggingPath); - const projects = await pollImportUrls(pollingUrls); + if (filteredTargets.length === 0) { + return []; + } + for ( + let targetIndex = 0; + targetIndex < filteredTargets.length; + targetIndex = targetIndex + concurrentTargets + ) { + const batch = filteredTargets.slice( + targetIndex, + targetIndex + concurrentTargets, + ); + const pollingUrls = await importTargets(batch, loggingPath); + projects.push(...(await pollImportUrls(pollingUrls))); + } return projects; } diff --git a/test/lib/index.test.ts b/test/lib/index.test.ts index 19627f02..8c9a0b2e 100644 --- a/test/lib/index.test.ts +++ b/test/lib/index.test.ts @@ -5,7 +5,7 @@ import { pollImportUrls, deleteProjects, } from '../../src/lib'; -import { Status, Project } from '../../src/lib/types'; +import { Project } from '../../src/lib/types'; const ORG_ID = 'f0125d9b-271a-4b50-ad23-80e12575a1bf'; const GITHUB_INTEGRATION_ID = 'c4de291b-e083-4c43-a72c-113463e0d268'; @@ -25,15 +25,11 @@ async function deleteTestProjects( describe('Single target', () => { const discoveredProjects: Project[] = []; it('Import & poll a repo', async () => { - const { pollingUrl } = await importTarget( - ORG_ID, - GITHUB_INTEGRATION_ID, - { - name: 'shallow-goof-policy', - owner: 'snyk-fixtures', - branch: 'master', - }, - ); + const { pollingUrl } = await importTarget(ORG_ID, GITHUB_INTEGRATION_ID, { + name: 'shallow-goof-policy', + owner: 'snyk-fixtures', + branch: 'master', + }); expect(pollingUrl).not.toBeNull(); const projects = await pollImportUrl(pollingUrl); expect(projects[0]).toMatchObject({ diff --git a/test/scripts/__snapshots__/import-projects.test.ts.snap b/test/scripts/__snapshots__/import-projects.test.ts.snap index e036e740..0809287a 100644 --- a/test/scripts/__snapshots__/import-projects.test.ts.snap +++ b/test/scripts/__snapshots__/import-projects.test.ts.snap @@ -1,9 +1,5 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`Import projects script Skips any badly formatted targets 1`] = `"f0125d9b-271a-4b50-ad23-80e12575a1bf:c4de291b-e083-4c43-a72c-113463e0d268:shallow-goof-policy:snyk-fixtures:master:https://dev.snyk.io/api/v1/org/f0125d9b-271a-4b50-ad23-80e12575a1bf/integrations/c4de291b-e083-4c43-a72c-113463e0d268/import/63f06690-1485-40cf-b81f-218104cfb277,f0125d9b-271a-4b50-ad23-80e12575a1bf:c4de291b-e083-4c43-a72c-113463e0d268:ruby-with-versions:snyk-fixtures:master:https://dev.snyk.io/api/v1/org/f0125d9b-271a-4b50-ad23-80e12575a1bf/integrations/c4de291b-e083-4c43-a72c-113463e0d268/import/63f06690-1485-40cf-b81f-218104cfb277,f0125d9b-271a-4b50-ad23-80e12575a1bf:c4de291b-e083-4c43-a72c-113463e0d268:composer-with-vulns:snyk-fixtures:master:https://dev.snyk.io/api/v1/org/f0125d9b-271a-4b50-ad23-80e12575a1bf/integrations/c4de291b-e083-4c43-a72c-113463e0d268/import/63f06690-1485-40cf-b81f-218104cfb277,f0125d9b-271a-4b50-ad23-80e12575a1bf:c4de291b-e083-4c43-a72c-113463e0d268:shallow-goof-policy:snyk-fixtures:master:https://dev.snyk.io/api/v1/org/f0125d9b-271a-4b50-ad23-80e12575a1bf/integrations/c4de291b-e083-4c43-a72c-113463e0d268/import/8ba73ea5-e1ff-4513-ad5a-8931ba3bb4d1,"`; - -exports[`Import projects script succeeds to import targets from file 1`] = `"f0125d9b-271a-4b50-ad23-80e12575a1bf:c4de291b-e083-4c43-a72c-113463e0d268:ruby-with-versions:snyk-fixtures:master,f0125d9b-271a-4b50-ad23-80e12575a1bf:c4de291b-e083-4c43-a72c-113463e0d268:composer-with-vulns:snyk-fixtures:master,f0125d9b-271a-4b50-ad23-80e12575a1bf:c4de291b-e083-4c43-a72c-113463e0d268:shallow-goof-policy:snyk-fixtures:master,"`; - exports[`Import skips previously imported succeeds to import targets from file 1`] = ` "f0125d9b-271a-4b50-ad23-80e12575a1bf:c4de291b-e083-4c43-a72c-113463e0d268:composer-with-vulns:snyk-fixtures:master, " diff --git a/test/scripts/fixtures/import-projects.json b/test/scripts/fixtures/import-projects.json index cbf95f84..19699b2a 100644 --- a/test/scripts/fixtures/import-projects.json +++ b/test/scripts/fixtures/import-projects.json @@ -1,5 +1,32 @@ { "targets": [ + { + "orgId": "f0125d9b-271a-4b50-ad23-80e12575a1bf", + "integrationId": "c4de291b-e083-4c43-a72c-113463e0d268", + "target": { + "name": "shallow-goof-policy", + "owner": "snyk-fixtures", + "branch": "master" + } + }, + { + "orgId": "f0125d9b-271a-4b50-ad23-80e12575a1bf", + "integrationId": "c4de291b-e083-4c43-a72c-113463e0d268", + "target": { + "name": "ruby-with-versions", + "owner": "snyk-fixtures", + "branch": "master" + } + }, + { + "orgId": "f0125d9b-271a-4b50-ad23-80e12575a1bf", + "integrationId": "c4de291b-e083-4c43-a72c-113463e0d268", + "target": { + "name": "composer-with-vulns", + "owner": "snyk-fixtures", + "branch": "master" + } + }, { "orgId": "f0125d9b-271a-4b50-ad23-80e12575a1bf", "integrationId": "c4de291b-e083-4c43-a72c-113463e0d268", diff --git a/test/scripts/import-projects.test.ts b/test/scripts/import-projects.test.ts index a28474b3..58986ec0 100644 --- a/test/scripts/import-projects.test.ts +++ b/test/scripts/import-projects.test.ts @@ -8,7 +8,7 @@ describe('Import projects script', () => { afterEach(() => { fs.unlinkSync(logPath); }); - it('succeeds to import targets from file', async () => { + it.only('succeeds to import targets from file', async () => { const projects = await ImportProjects( path.resolve(__dirname + `/fixtures/${IMPORT_PROJECTS_FILE_NAME}`), __dirname,