diff --git a/src/lib/api/import/index.ts b/src/lib/api/import/index.ts index d3e3c06f..435f890d 100644 --- a/src/lib/api/import/index.ts +++ b/src/lib/api/import/index.ts @@ -107,6 +107,7 @@ export async function importTarget( } } +let failedMoreThanOnce = false; export async function importTargets( requestManager: requestsManager, targets: ImportTarget[], @@ -141,12 +142,16 @@ export async function importTargets( { errorMessage: error.message }, loggingPath, ); + if (failed % concurrentImports === 0) { - console.error( - `Every import in this batch failed, stopping as this is unexpected! Please check if everything is configured ok and review the logs located at ${loggingPath}/*. If everything looks OK re-start the import, previously imported targets will be skipped.`, - ); - // die immediately - process.exit(1); + if (failedMoreThanOnce) { + console.error( + `Every import in the last few batches failed, stopping as this is unexpected! Please check if everything is configured ok and review the logs located at ${loggingPath}/*. If everything looks OK re-start the import, previously imported targets will be skipped.`, + ); + // die immediately + process.exit(1); + } + failedMoreThanOnce = true; } } }, diff --git a/src/lib/api/poll-import/index.ts b/src/lib/api/poll-import/index.ts index f5c1ee50..e7bd615c 100644 --- a/src/lib/api/poll-import/index.ts +++ b/src/lib/api/poll-import/index.ts @@ -1,6 +1,6 @@ import 'source-map-support/register'; import * as url from 'url'; -import type { requestsManager } from 'snyk-request-manager'; +import { requestsManager } from 'snyk-request-manager'; import * as sleep from 'sleep-promise'; import * as debugLib from 'debug'; import * as _ from 'lodash'; @@ -33,24 +33,28 @@ export async function pollImportUrl( } try { const { pathname = '' } = url.parse(locationUrl); - const res = await requestManager.request({ - verb: 'get', - url: (pathname as string).split('/api/v1/')[1], - body: JSON.stringify({}), - }); + const res = await tryPolling( + requestManager, + (pathname as string).split('/api/v1/')[1], + ); const importStatus: PollImportResponse = res.data; const statusCode = res.statusCode || res.status; - if (!statusCode || statusCode !== 200) { + + if (!statusCode || ![422, 200].includes(statusCode)) { throw new Error( 'Expected a 200 response, instead received: ' + JSON.stringify({ data: res.data, status: statusCode }), ); } + if (statusCode == 422) { + retryCount = 3; + } debug(`Import task status is "${importStatus.status}"`); if ( - importStatus.status && - importStatus.status !== 'complete' && - retryCount > 0 + statusCode == 422 || + (importStatus.status && + importStatus.status !== 'complete' && + retryCount > 0) ) { await sleep(retryWaitTime); debug(`Will re-check import task in "${retryWaitTime} ms"`); @@ -75,6 +79,24 @@ export async function pollImportUrl( } } +async function tryPolling(requestManager: requestsManager, pollingUrl: string) { + try { + const res = await requestManager.request({ + verb: 'get', + url: pollingUrl, + body: JSON.stringify({}), + }); + + return res; + } catch (e) { + if (e.message?.response?.status == 422) { + return e.message?.response; + } + + throw e; + } +} + export async function pollImportUrls( requestManager: requestsManager, locationUrls: string[], diff --git a/test/scripts/import-projects.test.ts b/test/scripts/import-projects.test.ts index 504ef963..3dba5bca 100644 --- a/test/scripts/import-projects.test.ts +++ b/test/scripts/import-projects.test.ts @@ -112,6 +112,7 @@ describe('Skips & logs issues', () => { let logs: string[]; afterEach(async () => { + process.env.CONCURRENT_IMPORTS = '3'; await deleteFiles(logs); process.env = { ...OLD_ENV }; }, 10000); @@ -143,11 +144,12 @@ describe('Skips & logs issues', () => { }, 50000); it('Logs failed when API errors', async () => { + process.env.CONCURRENT_IMPORTS = '1'; // this folder does not exist and will be created on run const logRoot = __dirname + '/fixtures/failed-batch-log/'; const logFiles = generateLogsPaths(logRoot, ORG_ID); logs = Object.values(logFiles); - const exit = jest.spyOn(process, 'exit').mockImplementationOnce(() => { + const exit = jest.spyOn(process, 'exit').mockImplementation(() => { throw new Error('process.exit() was called.'); }); try {