Merge pull request #444 from snyk-tech-services/feat/relax-batch-errors

Feat/relax batch errors
snyk · Apr 21, 2023 · 606529a · 606529a
2 parents 21b7a64 + 64722b9
commit 606529a
Show file tree

Hide file tree

Showing 3 changed files with 45 additions and 16 deletions.
diff --git a/src/lib/api/import/index.ts b/src/lib/api/import/index.ts
@@ -107,6 +107,7 @@ export async function importTarget(
   }
 }
 
+let failedMoreThanOnce = false;
 export async function importTargets(
   requestManager: requestsManager,
   targets: ImportTarget[],
@@ -141,12 +142,16 @@ export async function importTargets(
           { errorMessage: error.message },
           loggingPath,
         );
+
         if (failed % concurrentImports === 0) {
-          console.error(
-            `Every import in this batch failed, stopping as this is unexpected! Please check if everything is configured ok and review the logs located at ${loggingPath}/*. If everything looks OK re-start the import, previously imported targets will be skipped.`,
-          );
-          // die immediately
-          process.exit(1);
+          if (failedMoreThanOnce) {
+            console.error(
+              `Every import in the last few batches failed, stopping as this is unexpected! Please check if everything is configured ok and review the logs located at ${loggingPath}/*. If everything looks OK re-start the import, previously imported targets will be skipped.`,
+            );
+            // die immediately
+            process.exit(1);
+          }
+          failedMoreThanOnce = true;
         }
       }
     },

diff --git a/src/lib/api/poll-import/index.ts b/src/lib/api/poll-import/index.ts
@@ -1,6 +1,6 @@
 import 'source-map-support/register';
 import * as url from 'url';
-import type { requestsManager } from 'snyk-request-manager';
+import { requestsManager } from 'snyk-request-manager';
 import * as sleep from 'sleep-promise';
 import * as debugLib from 'debug';
 import * as _ from 'lodash';
@@ -33,24 +33,28 @@ export async function pollImportUrl(
   }
   try {
     const { pathname = '' } = url.parse(locationUrl);
-    const res = await requestManager.request({
-      verb: 'get',
-      url: (pathname as string).split('/api/v1/')[1],
-      body: JSON.stringify({}),
-    });
+    const res = await tryPolling(
+      requestManager,
+      (pathname as string).split('/api/v1/')[1],
+    );
     const importStatus: PollImportResponse = res.data;
     const statusCode = res.statusCode || res.status;
-    if (!statusCode || statusCode !== 200) {
+
+    if (!statusCode || ![422, 200].includes(statusCode)) {
       throw new Error(
         'Expected a 200 response, instead received: ' +
           JSON.stringify({ data: res.data, status: statusCode }),
       );
     }
+    if (statusCode == 422) {
+      retryCount = 3;
+    }
     debug(`Import task status is "${importStatus.status}"`);
     if (
-      importStatus.status &&
-      importStatus.status !== 'complete' &&
-      retryCount > 0
+      statusCode == 422 ||
+      (importStatus.status &&
+        importStatus.status !== 'complete' &&
+        retryCount > 0)
     ) {
       await sleep(retryWaitTime);
       debug(`Will re-check import task in "${retryWaitTime} ms"`);
@@ -75,6 +79,24 @@ export async function pollImportUrl(
   }
 }
 
+async function tryPolling(requestManager: requestsManager, pollingUrl: string) {
+  try {
+    const res = await requestManager.request({
+      verb: 'get',
+      url: pollingUrl,
+      body: JSON.stringify({}),
+    });
+
+    return res;
+  } catch (e) {
+    if (e.message?.response?.status == 422) {
+      return e.message?.response;
+    }
+
+    throw e;
+  }
+}
+
 export async function pollImportUrls(
   requestManager: requestsManager,
   locationUrls: string[],

diff --git a/test/scripts/import-projects.test.ts b/test/scripts/import-projects.test.ts
@@ -112,6 +112,7 @@ describe('Skips & logs issues', () => {
   let logs: string[];
 
   afterEach(async () => {
+    process.env.CONCURRENT_IMPORTS = '3';
     await deleteFiles(logs);
     process.env = { ...OLD_ENV };
   }, 10000);
@@ -143,11 +144,12 @@ describe('Skips & logs issues', () => {
   }, 50000);
 
   it('Logs failed when API errors', async () => {
+    process.env.CONCURRENT_IMPORTS = '1';
     // this folder does not exist and will be created on run
     const logRoot = __dirname + '/fixtures/failed-batch-log/';
     const logFiles = generateLogsPaths(logRoot, ORG_ID);
     logs = Object.values(logFiles);
-    const exit = jest.spyOn(process, 'exit').mockImplementationOnce(() => {
+    const exit = jest.spyOn(process, 'exit').mockImplementation(() => {
       throw new Error('process.exit() was called.');
     });
     try {