From 6993d44adefb7164d9f6e71e07376556d86672e7 Mon Sep 17 00:00:00 2001 From: Asaf Agami Date: Tue, 18 Jul 2023 15:07:42 +0300 Subject: [PATCH 01/13] fix: added retry and prevented redundant downloads --- snykTask/src/install/index.ts | 43 +++++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/snykTask/src/install/index.ts b/snykTask/src/install/index.ts index e1ef389f..c6ad6a59 100644 --- a/snykTask/src/install/index.ts +++ b/snykTask/src/install/index.ts @@ -54,17 +54,42 @@ export async function downloadExecutable( targetDirectory: string, executable: Executable, ) { - const fileWriter = fs.createWriteStream( - path.join(targetDirectory, executable.filename), - { - mode: 0o766, - }, - ); - return new Promise((resolve, reject) => { + const filePath = path.join(targetDirectory, executable.filename); + + // Check if the file already exists + if (fs.existsSync(filePath)) { + console.log(`File ${executable.filename} already exists, skipping download.`); + return; + } + + const fileWriter = fs.createWriteStream(filePath, { + mode: 0o766, + }); + + // Wrapping the download in a function for easy retrying + const doDownload = () => new Promise((resolve, reject) => { https.get(executable.downloadUrl, (response) => { response.on('end', () => resolve()); - response.on('error', (err) => reject(err)); + response.on('error', (err) => { + console.error(`Download of ${executable.filename} failed: ${err.message}`); + reject(err); + }); response.pipe(fileWriter); }); }); -} + + // Try to download the file, retry once after 5 seconds if the first attempt fails + try { + await doDownload(); + } catch (err) { + console.error(`Download of ${executable.filename} failed: ${err.message}`) + console.log(`Retrying download of ${executable.filename} after 5 seconds...`); + await new Promise(resolve => setTimeout(resolve, 5000)); + try { + await doDownload(); + console.log(`Retry successful for ${executable.filename}`); + } catch (retryErr) { + console.error(`Retry failed for ${executable.filename}: ${retryErr.message}`); + } + } +} \ No newline at end of file From 0a28a266843822c8131c279db08cfe34c01b2f2a Mon Sep 17 00:00:00 2001 From: Asaf Agami Date: Tue, 18 Jul 2023 15:11:37 +0300 Subject: [PATCH 02/13] feat: add CLI debug output when debug is enabled --- snykTask/src/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/snykTask/src/index.ts b/snykTask/src/index.ts index 5f3b0f5f..cd73bb22 100644 --- a/snykTask/src/index.ts +++ b/snykTask/src/index.ts @@ -154,6 +154,7 @@ async function runSnykTest( .argIf(taskArgs.organization, `--org=${taskArgs.organization}`) .argIf(taskArgs.projectName, `--project-name=${projectNameArg}`) .arg(`--json-file-output=${jsonReportOutputPath}`) + .argIf(isDebugMode(), '-d') .line(taskArgs.additionalArguments); const options = getOptionsToExecuteSnykCLICommand( From fefba976a72975d52de6f4b697f1da7492f7ffb4 Mon Sep 17 00:00:00 2001 From: Asaf Agami Date: Tue, 18 Jul 2023 15:16:38 +0300 Subject: [PATCH 03/13] fix: linter warnings --- snykTask/src/install/index.ts | 39 +++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/snykTask/src/install/index.ts b/snykTask/src/install/index.ts index c6ad6a59..a8781b4d 100644 --- a/snykTask/src/install/index.ts +++ b/snykTask/src/install/index.ts @@ -55,10 +55,12 @@ export async function downloadExecutable( executable: Executable, ) { const filePath = path.join(targetDirectory, executable.filename); - + // Check if the file already exists if (fs.existsSync(filePath)) { - console.log(`File ${executable.filename} already exists, skipping download.`); + console.log( + `File ${executable.filename} already exists, skipping download.`, + ); return; } @@ -67,29 +69,36 @@ export async function downloadExecutable( }); // Wrapping the download in a function for easy retrying - const doDownload = () => new Promise((resolve, reject) => { - https.get(executable.downloadUrl, (response) => { - response.on('end', () => resolve()); - response.on('error', (err) => { - console.error(`Download of ${executable.filename} failed: ${err.message}`); - reject(err); + const doDownload = () => + new Promise((resolve, reject) => { + https.get(executable.downloadUrl, (response) => { + response.on('end', () => resolve()); + response.on('error', (err) => { + console.error( + `Download of ${executable.filename} failed: ${err.message}`, + ); + reject(err); + }); + response.pipe(fileWriter); }); - response.pipe(fileWriter); }); - }); // Try to download the file, retry once after 5 seconds if the first attempt fails try { await doDownload(); } catch (err) { - console.error(`Download of ${executable.filename} failed: ${err.message}`) - console.log(`Retrying download of ${executable.filename} after 5 seconds...`); - await new Promise(resolve => setTimeout(resolve, 5000)); + console.error(`Download of ${executable.filename} failed: ${err.message}`); + console.log( + `Retrying download of ${executable.filename} after 5 seconds...`, + ); + await new Promise((resolve) => setTimeout(resolve, 5000)); try { await doDownload(); console.log(`Retry successful for ${executable.filename}`); } catch (retryErr) { - console.error(`Retry failed for ${executable.filename}: ${retryErr.message}`); + console.error( + `Retry failed for ${executable.filename}: ${retryErr.message}`, + ); } } -} \ No newline at end of file +} From 3a210fa8e0f497ebf4360af6535067fb8d656b9e Mon Sep 17 00:00:00 2001 From: Asaf Agami Date: Tue, 18 Jul 2023 15:26:35 +0300 Subject: [PATCH 04/13] fix: more linter warnings --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9327d7e6..44addd13 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,7 +2,7 @@ version: 2.1 orbs: prodsec: snyk/prodsec-orb@1.0 - + jobs: test: docker: From 6b3b08d2ec19a62b1f383b65611db533a0fc5fee Mon Sep 17 00:00:00 2001 From: Asaf Agami Date: Tue, 18 Jul 2023 15:38:21 +0300 Subject: [PATCH 05/13] chore: bump azure-pipelines-task-lib version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 6eae86e3..ccbcaaa3 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "author": "snyk.io", "license": "Apache-2.0", "dependencies": { - "azure-pipelines-task-lib": "3.3.1", + "azure-pipelines-task-lib": "^4.4.0", "jquery": "^3.4.1", "vss-web-extension-sdk": "^5.141.0" }, From 5e8fddf0c027b51a6aa349592f6829b943efd68f Mon Sep 17 00:00:00 2001 From: Asaf Agami Date: Tue, 18 Jul 2023 15:46:38 +0300 Subject: [PATCH 06/13] Revert "chore: bump azure-pipelines-task-lib version" This reverts commit 6b3b08d2ec19a62b1f383b65611db533a0fc5fee. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index ccbcaaa3..6eae86e3 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "author": "snyk.io", "license": "Apache-2.0", "dependencies": { - "azure-pipelines-task-lib": "^4.4.0", + "azure-pipelines-task-lib": "3.3.1", "jquery": "^3.4.1", "vss-web-extension-sdk": "^5.141.0" }, From 452fc2b516c7e6c6cc9343d3f3d39cc94ed1dc0d Mon Sep 17 00:00:00 2001 From: Asaf Agami Date: Tue, 18 Jul 2023 15:53:52 +0300 Subject: [PATCH 07/13] chore: ignore vulns with no available upgrades --- .snyk | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .snyk diff --git a/.snyk b/.snyk new file mode 100644 index 00000000..8434a190 --- /dev/null +++ b/.snyk @@ -0,0 +1,15 @@ +# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities. +version: v1.25.0 +# ignores vulnerabilities until expiry date; change duration by modifying expiry date +ignore: + SNYK-JS-MOCKERY-3043117: + - '*': + reason: No upgrade available + expires: 2024-07-18T00:00:00.000Z + created: 2023-07-18T12:52:43.840Z + SNYK-JS-SEMVER-3247795: + - '*': + reason: No upgrade available + expires: 2024-07-18T00:00:00.000Z + created: 2023-07-18T12:52:32.034Z +patch: {} From f40a5ff91c7867aaa80862d039e599e8d8bcf1e4 Mon Sep 17 00:00:00 2001 From: Asaf Agami Date: Tue, 18 Jul 2023 17:22:45 +0300 Subject: [PATCH 08/13] chore: add logging in test pipelines script --- ops/deploy/run-test-pipelines.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ops/deploy/run-test-pipelines.ts b/ops/deploy/run-test-pipelines.ts index 7039dbb2..edd8fd49 100644 --- a/ops/deploy/run-test-pipelines.ts +++ b/ops/deploy/run-test-pipelines.ts @@ -95,6 +95,10 @@ async function runBuild( ): Promise { let success = false; + console.log( + `Starting build for project: ${testProjectName} with build definition ID: ${testBuildDefinitionId}`, + ); + try { const launchPipelineResult = await launchBuildPipeline( webApi, @@ -150,7 +154,9 @@ async function runBuild( return Promise.reject(); } } catch (err) { - console.log('failed to launching / checking build'); + console.log( + `Failed to launch/check build for project: ${testProjectName} with build definition ID: ${testBuildDefinitionId}`, + ); console.log(err); console.log('\nrejecting - not successful'); return Promise.reject(); From 6ffa175a5100859b64c4c427564fb3f6a0f79240 Mon Sep 17 00:00:00 2001 From: Asaf Agami Date: Tue, 18 Jul 2023 17:39:53 +0300 Subject: [PATCH 09/13] chore: add project name to logs --- ops/deploy/run-test-pipelines.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ops/deploy/run-test-pipelines.ts b/ops/deploy/run-test-pipelines.ts index edd8fd49..28a3cbd0 100644 --- a/ops/deploy/run-test-pipelines.ts +++ b/ops/deploy/run-test-pipelines.ts @@ -135,7 +135,7 @@ async function runBuild( console.log('build succeeded'); success = true; } else { - console.log(`build did not succeed. BuildResult code: ${result}`); + console.log(`build did not succeed for ${testProjectName}. BuildResult code: ${result}`); } } break; From e9d9bfabd075f4bcb560ad492c3b65b2ad5d5266 Mon Sep 17 00:00:00 2001 From: Asaf Agami Date: Tue, 18 Jul 2023 17:44:07 +0300 Subject: [PATCH 10/13] fix: linter --- ops/deploy/run-test-pipelines.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ops/deploy/run-test-pipelines.ts b/ops/deploy/run-test-pipelines.ts index 28a3cbd0..235b1490 100644 --- a/ops/deploy/run-test-pipelines.ts +++ b/ops/deploy/run-test-pipelines.ts @@ -135,7 +135,9 @@ async function runBuild( console.log('build succeeded'); success = true; } else { - console.log(`build did not succeed for ${testProjectName}. BuildResult code: ${result}`); + console.log( + `build did not succeed for ${testProjectName}. BuildResult code: ${result}`, + ); } } break; From 9d2e36b7884f39ce1276ee8f38bb4e8d0821567d Mon Sep 17 00:00:00 2001 From: Asaf Agami Date: Wed, 19 Jul 2023 11:44:07 +0300 Subject: [PATCH 11/13] chore: add more project names in logging statements --- ops/deploy/run-test-pipelines.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ops/deploy/run-test-pipelines.ts b/ops/deploy/run-test-pipelines.ts index 235b1490..45c359f2 100644 --- a/ops/deploy/run-test-pipelines.ts +++ b/ops/deploy/run-test-pipelines.ts @@ -127,12 +127,12 @@ async function runBuild( } if (status === BuildStatus.Completed) { - console.log('build is complete'); + console.log(`build is complete for ${testProjectName}`); const result = checkBuildStatusRes.result; console.log(`build result: ${result}`); if (result) { if (result === BuildResult.Succeeded) { - console.log('build succeeded'); + console.log(`build succeeded for ${testProjectName}`); success = true; } else { console.log( @@ -143,7 +143,7 @@ async function runBuild( break; } else { console.log( - `Still waiting for build ${buildId} to complete. Status: ${status}. Time: ${new Date().getTime()}`, + `Still waiting for build ${buildId} (${testProjectName}) to complete. Status: ${status}. Time: ${new Date().getTime()}`, ); await asyncSleep(10000); } From ee71741102faf90f37391df8901058f86d1d7a68 Mon Sep 17 00:00:00 2001 From: Asaf Agami Date: Wed, 19 Jul 2023 11:44:28 +0300 Subject: [PATCH 12/13] feat: use 5 retries, refactored retry logic to a loop --- snykTask/src/install/index.ts | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/snykTask/src/install/index.ts b/snykTask/src/install/index.ts index a8781b4d..7e22d54e 100644 --- a/snykTask/src/install/index.ts +++ b/snykTask/src/install/index.ts @@ -53,6 +53,7 @@ export function getSnykDownloadInfo(platform: Platform): SnykDownloads { export async function downloadExecutable( targetDirectory: string, executable: Executable, + maxRetries = 5 ) { const filePath = path.join(targetDirectory, executable.filename); @@ -83,22 +84,26 @@ export async function downloadExecutable( }); }); - // Try to download the file, retry once after 5 seconds if the first attempt fails - try { - await doDownload(); - } catch (err) { - console.error(`Download of ${executable.filename} failed: ${err.message}`); - console.log( - `Retrying download of ${executable.filename} after 5 seconds...`, - ); - await new Promise((resolve) => setTimeout(resolve, 5000)); + // Try to download the file, retry up to `maxRetries` times if the attempt fails + for (let attempt = 0; attempt < maxRetries; attempt++) { try { await doDownload(); - console.log(`Retry successful for ${executable.filename}`); - } catch (retryErr) { - console.error( - `Retry failed for ${executable.filename}: ${retryErr.message}`, - ); + console.log(`Download successful for ${executable.filename}`); + break; + } catch (err) { + console.error(`Download of ${executable.filename} failed: ${err.message}`); + + // Don't wait before retrying the last attempt + if (attempt < maxRetries - 1) { + console.log( + `Retrying download of ${executable.filename} after 5 seconds...`, + ); + await new Promise((resolve) => setTimeout(resolve, 5000)); + } else { + console.error( + `All retries failed for ${executable.filename}: ${err.message}`, + ); + } } } -} +} \ No newline at end of file From 35904250e11971a9d99c2d7c8906b1cd658715d0 Mon Sep 17 00:00:00 2001 From: Asaf Agami Date: Wed, 19 Jul 2023 11:47:35 +0300 Subject: [PATCH 13/13] chore: npm run format --- snykTask/src/install/index.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/snykTask/src/install/index.ts b/snykTask/src/install/index.ts index 7e22d54e..099d5a63 100644 --- a/snykTask/src/install/index.ts +++ b/snykTask/src/install/index.ts @@ -53,7 +53,7 @@ export function getSnykDownloadInfo(platform: Platform): SnykDownloads { export async function downloadExecutable( targetDirectory: string, executable: Executable, - maxRetries = 5 + maxRetries = 5, ) { const filePath = path.join(targetDirectory, executable.filename); @@ -91,7 +91,9 @@ export async function downloadExecutable( console.log(`Download successful for ${executable.filename}`); break; } catch (err) { - console.error(`Download of ${executable.filename} failed: ${err.message}`); + console.error( + `Download of ${executable.filename} failed: ${err.message}`, + ); // Don't wait before retrying the last attempt if (attempt < maxRetries - 1) { @@ -106,4 +108,4 @@ export async function downloadExecutable( } } } -} \ No newline at end of file +}