From e5313696be093110ab35f7cd8a11b58d8fcf6ec9 Mon Sep 17 00:00:00 2001 From: Liliana Kastilio Date: Thu, 21 May 2020 12:22:32 +0100 Subject: [PATCH] feat: bulk delete to help clean up after tests --- .gitignore | 1 + src/lib/get-snyk-host.ts | 3 + src/lib/import/index.ts | 8 +- src/lib/index.ts | 1 + src/lib/poll-import/index.ts | 3 +- src/lib/project/index.ts | 57 +++++++++++ src/lib/types.ts | 2 - test/lib/index.test.ts | 187 ++++++++++++++++++++--------------- 8 files changed, 177 insertions(+), 85 deletions(-) create mode 100644 src/lib/get-snyk-host.ts create mode 100644 src/lib/project/index.ts diff --git a/.gitignore b/.gitignore index 02392b5d..b88a660a 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ dist package-lock.json yarn.lock .eslintcache +local.* diff --git a/src/lib/get-snyk-host.ts b/src/lib/get-snyk-host.ts new file mode 100644 index 00000000..5314e356 --- /dev/null +++ b/src/lib/get-snyk-host.ts @@ -0,0 +1,3 @@ +export function getSnykHost(): string { + return process.env.SNYK_HOST || 'https://snyk.io'; +} diff --git a/src/lib/import/index.ts b/src/lib/import/index.ts index f9fe52c8..2e9b7512 100644 --- a/src/lib/import/index.ts +++ b/src/lib/import/index.ts @@ -5,9 +5,9 @@ import * as debugLib from 'debug'; import * as _ from 'lodash'; import { Target, FilePath, ImportTarget } from '../types'; import { getApiToken } from '../get-api-token'; +import { getSnykHost } from '../get-snyk-host'; -const debug = debugLib('snyk:import'); -const SNYK_HOST = process.env.SNYK_HOST || 'https://snyk.io'; +const debug = debugLib('snyk:api-import'); export async function importTarget( orgId: string, @@ -29,6 +29,7 @@ export async function importTarget( target, files, }; + const SNYK_HOST = getSnykHost(); const res = await needle( 'post', @@ -44,7 +45,6 @@ export async function importTarget( }, ); if (res.statusCode && res.statusCode !== 201) { - debug('ERROR:', res.body); throw new Error( 'Expected a 201 response, instead received: ' + JSON.stringify(res.body), ); @@ -65,7 +65,7 @@ export async function importTarget( innerError?: string; } = new Error('Could not complete API import'); err.innerError = error; - debug(`Could not complete API import: ${error}`); + debug(`Could not complete API import: ${error.message}`); throw err; } } diff --git a/src/lib/index.ts b/src/lib/index.ts index 33e00c50..27ac37e5 100644 --- a/src/lib/index.ts +++ b/src/lib/index.ts @@ -1,2 +1,3 @@ export * from './import'; export * from './poll-import'; +export * from './project'; diff --git a/src/lib/poll-import/index.ts b/src/lib/poll-import/index.ts index 6d40ac51..088af056 100644 --- a/src/lib/poll-import/index.ts +++ b/src/lib/poll-import/index.ts @@ -6,8 +6,9 @@ import * as _ from 'lodash'; import * as pMap from 'p-map'; import { PollImportResponse } from '../types'; import { getApiToken } from '../get-api-token'; + const debug = debugLib('snyk:poll-import'); -const MIN_RETRY_WAIT_TIME = 30000; +const MIN_RETRY_WAIT_TIME = 3000; const MAX_RETRY_COUNT = 1000; export async function pollImportUrl( diff --git a/src/lib/project/index.ts b/src/lib/project/index.ts new file mode 100644 index 00000000..8bba0dd0 --- /dev/null +++ b/src/lib/project/index.ts @@ -0,0 +1,57 @@ +import 'source-map-support/register'; +import * as needle from 'needle'; +import * as debugLib from 'debug'; +import { getApiToken } from '../get-api-token'; +import { getSnykHost } from '../get-snyk-host'; +const debug = debugLib('snyk:api-import'); + +export async function deleteProjects( + orgId: string, + projects: string[], +): Promise { + const apiToken = getApiToken(); + if (!(orgId && projects)) { + throw new Error( + `Missing required parameters. Please ensure you have provided: orgId & projectIds.`, + ); + } + debug('Deleting projectIds:', projects.join(', ')); + + try { + const SNYK_HOST = getSnykHost(); + const body = { + projects, + }; + const res = await needle( + 'post', + `${SNYK_HOST}/api/v1/org/${orgId}/projects/bulk-delete`, + body, + { + json: true, + // eslint-disable-next-line @typescript-eslint/camelcase + read_timeout: 30000, + headers: { + 'content-type': 'application/json', + Authorization: `token ${apiToken}`, + }, + }, + ); + if (res.statusCode !== 200) { + throw new Error( + `Expected a 200 response, instead received statusCode: ${ + res.statusCode + }, + body: ${JSON.stringify(res.body)}`, + ); + } + return res.body; + } catch (error) { + debug('Could not delete project:', error.message || error); + const err: { + message?: string | undefined; + innerError?: string; + } = new Error('Could not delete project'); + err.innerError = error; + throw err; + } +} diff --git a/src/lib/types.ts b/src/lib/types.ts index 0b15ae9c..0a9ab8d9 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -1,5 +1,3 @@ -import { Url } from 'url'; - export interface ImportTarget { orgId: string; integrationId: string; diff --git a/test/lib/index.test.ts b/test/lib/index.test.ts index b1f4a5c8..5bbff190 100644 --- a/test/lib/index.test.ts +++ b/test/lib/index.test.ts @@ -3,96 +3,127 @@ import { pollImportUrl, importTargets, pollImportUrls, + deleteProjects, } from '../../src/lib'; -import { Status } from '../../src/lib/types'; +import { Status, Project } from '../../src/lib/types'; +const ORG_ID = 'f0125d9b-271a-4b50-ad23-80e12575a1bf'; +const GITHUB_INTEGRATION_ID = 'c4de291b-e083-4c43-a72c-113463e0d268'; -// TODO: afterEach delete the new projects -test('Import & Poll a repo', async () => { - const { pollingUrl } = await importTarget( - 'f0125d9b-271a-4b50-ad23-80e12575a1bf', - 'c4de291b-e083-4c43-a72c-113463e0d268', - { +async function deleteTestProjects( + discoveredProjects: Project[], +): Promise { + const projectIds: string[] = []; + discoveredProjects.forEach(async (project) => { + if (project.projectUrl) { + const projectId = project.projectUrl.split('/').slice(-1)[0]; + projectIds.push(projectId); + } + }); + await deleteProjects(ORG_ID, projectIds); +} + +describe('Single target', () => { + const discoveredProjects: Project[] = []; + it('Import & poll a repo', async () => { + const { pollingUrl } = await importTarget(ORG_ID, GITHUB_INTEGRATION_ID, { name: 'shallow-goof-policy', owner: 'snyk-fixtures', branch: 'master', - }, - ); - expect(pollingUrl).not.toBeNull(); - const importLog = await pollImportUrl(pollingUrl); - expect(importLog).toMatchObject({ - id: expect.any(String), - status: 'complete', - created: expect.any(String), - }); - expect(importLog.logs.length > 1).toBeTruthy(); - expect(importLog.logs[0]).toMatchObject({ - name: 'snyk-fixtures/shallow-goof-policy', - created: expect.any(String), - status: 'complete', - projects: [ - { - projectUrl: expect.any(String), - success: true, - targetFile: expect.any(String), - }, - ], + }); + expect(pollingUrl).not.toBeNull(); + const importLog = await pollImportUrl(pollingUrl); + expect(importLog).toMatchObject({ + id: expect.any(String), + status: 'complete', + created: expect.any(String), + }); + expect(importLog.logs.length > 1).toBeTruthy(); + expect(importLog.logs[0]).toMatchObject({ + name: expect.any(String), + created: expect.any(String), + status: 'complete', + projects: [ + { + projectUrl: expect.any(String), + success: true, + targetFile: expect.any(String), + }, + ], + }); + // cleanup + importLog.logs.forEach((log) => { + discoveredProjects.push(...log.projects); + }); + }, 30000000); + afterAll(async () => { + await deleteTestProjects(discoveredProjects); }); -}, 30000000); +}); -test('importTargets & pollImportUrls multiple repos', async () => { - const pollingUrls = await importTargets([ - { - orgId: 'f0125d9b-271a-4b50-ad23-80e12575a1bf', - integrationId: 'c4de291b-e083-4c43-a72c-113463e0d268', - target: { - name: 'shallow-goof-policy', - owner: 'snyk-fixtures', - branch: 'master', - }, - }, - { - orgId: 'f0125d9b-271a-4b50-ad23-80e12575a1bf', - integrationId: 'c4de291b-e083-4c43-a72c-113463e0d268', - target: { - name: 'ruby-with-versions', - owner: 'snyk-fixtures', - branch: 'master', +describe('Multiple targets', () => { + const discoveredProjects: Project[] = []; + it('importTargets & pollImportUrls multiple repos', async () => { + const pollingUrls = await importTargets([ + { + orgId: ORG_ID, + integrationId: GITHUB_INTEGRATION_ID, + target: { + name: 'shallow-goof-policy', + owner: 'snyk-fixtures', + branch: 'master', + }, }, - }, - { - orgId: 'f0125d9b-271a-4b50-ad23-80e12575a1bf', - integrationId: 'c4de291b-e083-4c43-a72c-113463e0d268', - target: { - name: 'composer-with-vulns', - owner: 'snyk-fixtures', - branch: 'master', + { + orgId: ORG_ID, + integrationId: GITHUB_INTEGRATION_ID, + target: { + name: 'ruby-with-versions', + owner: 'snyk-fixtures', + branch: 'master', + }, }, - }, - ]); - expect(pollingUrls.length >= 1).toBeTruthy(); - const importLog = await pollImportUrls(pollingUrls); - expect(importLog[0]).toMatchObject({ - id: expect.any(String), - status: 'complete', - created: expect.any(String), - }); - // at least one job successfully finished - expect(importLog[0].logs[0]).toMatchObject({ - name: expect.any(String), - created: expect.any(String), - status: 'complete', - projects: [ { - projectUrl: expect.any(String), - success: true, - targetFile: expect.any(String), + orgId: ORG_ID, + integrationId: GITHUB_INTEGRATION_ID, + target: { + name: 'composer-with-vulns', + owner: 'snyk-fixtures', + branch: 'master', + }, }, - ], + ]); + expect(pollingUrls.length >= 1).toBeTruthy(); + const importLog = await pollImportUrls(pollingUrls); + expect(importLog[0]).toMatchObject({ + id: expect.any(String), + status: 'complete', + created: expect.any(String), + }); + // at least one job successfully finished + expect(importLog[0].logs[0]).toMatchObject({ + name: expect.any(String), + created: expect.any(String), + status: 'complete', + projects: [ + { + projectUrl: expect.any(String), + success: true, + targetFile: expect.any(String), + }, + ], + }); + expect( + importLog[0].logs.every((job) => job.status === Status.COMPLETE), + ).toBeTruthy(); + // cleanup + importLog[0].logs.forEach((log) => { + discoveredProjects.push(...log.projects); + }); + }, 30000000); + afterAll(async () => { + await deleteTestProjects(discoveredProjects); }); - expect( - importLog[0].logs.every((job) => job.status === Status.COMPLETE), - ).toBeTruthy(); -}, 30000000); +}); test.todo('Import & poll in one'); test.todo('Failed import 100%');