fix: batch import projects 5 at a time

snyk · May 22, 2020 · 8080478 · 8080478
1 parent d0f58c3
commit 8080478
Show file tree

Hide file tree

Showing 5 changed files with 52 additions and 17 deletions.
diff --git a/src/scripts/import-projects.ts b/src/scripts/import-projects.ts
@@ -4,6 +4,7 @@ import { loadFile } from '../load-file';
 import { importTargets, pollImportUrls } from '../lib';
 import { Project, ImportTarget } from '../lib/types';
 import { getLoggingPath } from '../lib/get-logging-path';
+import { getConcurrentImportsNumber } from '../lib/get-concurrent-imports-number';
 
 const debug = debugLib('snyk:import-projects-script');
 
@@ -48,10 +49,25 @@ export async function ImportProjects(
     throw new Error(`Failed to parse targets from ${fileName}`);
   }
   debug(`Loaded ${targets.length} targets to import`);
+  const concurrentTargets = getConcurrentImportsNumber();
+  const projects: Project[] = [];
   //TODO: validation?
   const filteredTargets = await filterOutImportedTargets(targets, loggingPath);
-  const pollingUrls = await importTargets(filteredTargets, loggingPath);
-  const projects = await pollImportUrls(pollingUrls);
+  if (filteredTargets.length === 0) {
+    return [];
+  }
+  for (
+    let targetIndex = 0;
+    targetIndex < filteredTargets.length;
+    targetIndex = targetIndex + concurrentTargets
+  ) {
+    const batch = filteredTargets.slice(
+      targetIndex,
+      targetIndex + concurrentTargets,
+    );
+    const pollingUrls = await importTargets(batch, loggingPath);
+    projects.push(...(await pollImportUrls(pollingUrls)));
+  }
 
   return projects;
 }
diff --git a/test/lib/index.test.ts b/test/lib/index.test.ts
@@ -5,7 +5,7 @@ import {
   pollImportUrls,
   deleteProjects,
 } from '../../src/lib';
-import { Status, Project } from '../../src/lib/types';
+import { Project } from '../../src/lib/types';
 const ORG_ID = 'f0125d9b-271a-4b50-ad23-80e12575a1bf';
 const GITHUB_INTEGRATION_ID = 'c4de291b-e083-4c43-a72c-113463e0d268';
 
@@ -25,15 +25,11 @@ async function deleteTestProjects(
 describe('Single target', () => {
   const discoveredProjects: Project[] = [];
   it('Import & poll a repo', async () => {
-    const { pollingUrl } = await importTarget(
-      ORG_ID,
-      GITHUB_INTEGRATION_ID,
-      {
-        name: 'shallow-goof-policy',
-        owner: 'snyk-fixtures',
-        branch: 'master',
-      },
-    );
+    const { pollingUrl } = await importTarget(ORG_ID, GITHUB_INTEGRATION_ID, {
+      name: 'shallow-goof-policy',
+      owner: 'snyk-fixtures',
+      branch: 'master',
+    });
     expect(pollingUrl).not.toBeNull();
     const projects = await pollImportUrl(pollingUrl);
     expect(projects[0]).toMatchObject({

diff --git a/test/scripts/__snapshots__/import-projects.test.ts.snap b/test/scripts/__snapshots__/import-projects.test.ts.snap
@@ -1,9 +1,5 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`Import projects script Skips any badly formatted targets 1`] = `"f0125d9b-271a-4b50-ad23-80e12575a1bf:c4de291b-e083-4c43-a72c-113463e0d268:shallow-goof-policy:snyk-fixtures:master:https://dev.snyk.io/api/v1/org/f0125d9b-271a-4b50-ad23-80e12575a1bf/integrations/c4de291b-e083-4c43-a72c-113463e0d268/import/63f06690-1485-40cf-b81f-218104cfb277,f0125d9b-271a-4b50-ad23-80e12575a1bf:c4de291b-e083-4c43-a72c-113463e0d268:ruby-with-versions:snyk-fixtures:master:https://dev.snyk.io/api/v1/org/f0125d9b-271a-4b50-ad23-80e12575a1bf/integrations/c4de291b-e083-4c43-a72c-113463e0d268/import/63f06690-1485-40cf-b81f-218104cfb277,f0125d9b-271a-4b50-ad23-80e12575a1bf:c4de291b-e083-4c43-a72c-113463e0d268:composer-with-vulns:snyk-fixtures:master:https://dev.snyk.io/api/v1/org/f0125d9b-271a-4b50-ad23-80e12575a1bf/integrations/c4de291b-e083-4c43-a72c-113463e0d268/import/63f06690-1485-40cf-b81f-218104cfb277,f0125d9b-271a-4b50-ad23-80e12575a1bf:c4de291b-e083-4c43-a72c-113463e0d268:shallow-goof-policy:snyk-fixtures:master:https://dev.snyk.io/api/v1/org/f0125d9b-271a-4b50-ad23-80e12575a1bf/integrations/c4de291b-e083-4c43-a72c-113463e0d268/import/8ba73ea5-e1ff-4513-ad5a-8931ba3bb4d1,"`;
-
-exports[`Import projects script succeeds to import targets from file 1`] = `"f0125d9b-271a-4b50-ad23-80e12575a1bf:c4de291b-e083-4c43-a72c-113463e0d268:ruby-with-versions:snyk-fixtures:master,f0125d9b-271a-4b50-ad23-80e12575a1bf:c4de291b-e083-4c43-a72c-113463e0d268:composer-with-vulns:snyk-fixtures:master,f0125d9b-271a-4b50-ad23-80e12575a1bf:c4de291b-e083-4c43-a72c-113463e0d268:shallow-goof-policy:snyk-fixtures:master,"`;
-
 exports[`Import skips previously imported succeeds to import targets from file 1`] = `
 "f0125d9b-271a-4b50-ad23-80e12575a1bf:c4de291b-e083-4c43-a72c-113463e0d268:composer-with-vulns:snyk-fixtures:master,
 "

diff --git a/test/scripts/fixtures/import-projects.json b/test/scripts/fixtures/import-projects.json
@@ -1,5 +1,32 @@
 {
   "targets": [
+    {
+      "orgId": "f0125d9b-271a-4b50-ad23-80e12575a1bf",
+      "integrationId": "c4de291b-e083-4c43-a72c-113463e0d268",
+      "target": {
+        "name": "shallow-goof-policy",
+        "owner": "snyk-fixtures",
+        "branch": "master"
+      }
+    },
+    {
+      "orgId": "f0125d9b-271a-4b50-ad23-80e12575a1bf",
+      "integrationId": "c4de291b-e083-4c43-a72c-113463e0d268",
+      "target": {
+        "name": "ruby-with-versions",
+        "owner": "snyk-fixtures",
+        "branch": "master"
+      }
+    },
+    {
+      "orgId": "f0125d9b-271a-4b50-ad23-80e12575a1bf",
+      "integrationId": "c4de291b-e083-4c43-a72c-113463e0d268",
+      "target": {
+        "name": "composer-with-vulns",
+        "owner": "snyk-fixtures",
+        "branch": "master"
+      }
+    },
     {
       "orgId": "f0125d9b-271a-4b50-ad23-80e12575a1bf",
       "integrationId": "c4de291b-e083-4c43-a72c-113463e0d268",

diff --git a/test/scripts/import-projects.test.ts b/test/scripts/import-projects.test.ts
@@ -8,7 +8,7 @@ describe('Import projects script', () => {
   afterEach(() => {
     fs.unlinkSync(logPath);
   });
-  it('succeeds to import targets from file', async () => {
+  it.only('succeeds to import targets from file', async () => {
     const projects = await ImportProjects(
       path.resolve(__dirname + `/fixtures/${IMPORT_PROJECTS_FILE_NAME}`),
       __dirname,