From 8149efb5a96d474b83bf903a06323b2bd17c4c90 Mon Sep 17 00:00:00 2001 From: Paulius Kutka Date: Tue, 25 May 2021 16:21:46 +0100 Subject: [PATCH 1/3] feat: expose bundle creation logic --- src/analysis.ts | 128 ++++++++++++++++++++++++++++++++---------------- src/index.ts | 3 +- 2 files changed, 87 insertions(+), 44 deletions(-) diff --git a/src/analysis.ts b/src/analysis.ts index e277f54a..d7d21099 100644 --- a/src/analysis.ts +++ b/src/analysis.ts @@ -18,6 +18,7 @@ import { GetAnalysisResponseDto, AnalysisFailedResponse, AnalysisFinishedResponse, + RemoteBundle, } from './http'; import emitter from './emitter'; import { defaultBaseURL, MAX_PAYLOAD, IGNORES_DEFAULT } from './constants'; @@ -41,6 +42,7 @@ import { import { RequestOptions } from './interfaces/http-options.interface'; import { fromEntries } from './lib/utils'; +import { ISupportedFiles } from './interfaces/files.interface'; const sleep = (duration: number) => new Promise(resolve => setTimeout(resolve, duration)); @@ -259,20 +261,12 @@ export async function analyzeFolders(options: FolderOptions): Promise { const analysisOptions: AnalyzeGitOptions = { ...analyzeGitDefaults, ...options }; - const { - baseURL, - sessionToken, - oAuthToken, - username, - includeLint, - reachability, - severity, - gitUri, - sarif, - source, - } = analysisOptions; + const { baseURL, sessionToken, oAuthToken, username, includeLint, reachability, severity, gitUri, sarif, source } = + analysisOptions; const bundleResponse = await createGitBundle( { baseURL, @@ -482,3 +446,81 @@ export async function analyzeGit(options: GitOptions, requestOptions?: RequestOp return result; } + +interface CreateBundleFromFolders extends FolderOptions { + supportedFiles?: ISupportedFiles; + baseDir?: string; + fileIgnores?: string[]; +} + +let createBundleFromFoldersDefaults = { + baseURL: defaultBaseURL, + sessionToken: '', + symlinksEnabled: false, + maxPayload: MAX_PAYLOAD, + defaultFileIgnores: IGNORES_DEFAULT, + source: '', +}; + +/** + * Creates a remote bundle and returns response from the bundle API + * + * @param {CreateBundleFromFolders} options + * @returns {Promise} + */ +export async function createBundleFromFolders(options: CreateBundleFromFolders): Promise { + const analysisOptions = { ...createBundleFromFoldersDefaults, ...options }; + + const { + baseURL, + source, + paths, + symlinksEnabled, + defaultFileIgnores, + maxPayload, + sessionToken, + supportedFiles = await getSupportedFiles(baseURL, source), + baseDir = determineBaseDir(paths), + fileIgnores = await collectIgnoreRules(paths, symlinksEnabled, defaultFileIgnores), + } = analysisOptions; + + emitter.scanFilesProgress(0); + const bundleFiles = []; + let totalFiles = 0; + const bundleFileCollector = collectBundleFiles( + baseDir, + paths, + supportedFiles, + fileIgnores, + maxPayload, + symlinksEnabled, + ); + for await (const f of bundleFileCollector) { + bundleFiles.push(f); + totalFiles += 1; + emitter.scanFilesProgress(totalFiles); + } + + // Create remote bundle + return bundleFiles.length + ? await remoteBundleFactory(baseURL, sessionToken, bundleFiles, [], baseDir, null, maxPayload, source) + : null; +} + +/** + * Get supported filters and test baseURL for correctness and availability + * + * @param baseURL + * @param source + * @returns + */ +async function getSupportedFiles(baseURL: string, source: string): Promise { + emitter.supportedFilesLoaded(null); + const resp = await getFilters(baseURL, source); + if (resp.type === 'error') { + throw resp.error; + } + const supportedFiles = resp.value; + emitter.supportedFilesLoaded(supportedFiles); + return supportedFiles; +} diff --git a/src/index.ts b/src/index.ts index 462920fa..cac51f12 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,4 @@ -import { analyzeFolders, extendAnalysis, analyzeGit } from './analysis'; +import { analyzeFolders, extendAnalysis, analyzeGit, createBundleFromFolders } from './analysis'; import emitter from './emitter'; import { startSession, checkSession, reportEvent, reportError } from './http'; import * as constants from './constants'; @@ -21,6 +21,7 @@ import { export { getGlobPatterns, analyzeFolders, + createBundleFromFolders, extendAnalysis, analyzeGit, startSession, From 7f1a49f8252f2c35c5e781b4032986c059723492 Mon Sep 17 00:00:00 2001 From: Paulius Kutka Date: Wed, 26 May 2021 09:03:34 +0100 Subject: [PATCH 2/3] chore: move default to a single variable --- src/analysis.ts | 57 ++++++++++++++++++------------------------------- 1 file changed, 21 insertions(+), 36 deletions(-) diff --git a/src/analysis.ts b/src/analysis.ts index d7d21099..fb8b5da1 100644 --- a/src/analysis.ts +++ b/src/analysis.ts @@ -46,6 +46,19 @@ import { ISupportedFiles } from './interfaces/files.interface'; const sleep = (duration: number) => new Promise(resolve => setTimeout(resolve, duration)); +const ANALYSIS_OPTIONS_DEFAULTS = { + baseURL: defaultBaseURL, + sessionToken: '', + includeLint: false, + reachability: false, + severity: AnalysisSeverity.info, + symlinksEnabled: false, + maxPayload: MAX_PAYLOAD, + defaultFileIgnores: IGNORES_DEFAULT, + sarif: false, + source: '', +} + async function pollAnalysis( { baseURL, @@ -239,20 +252,9 @@ function mergeBundleResults(bundle: IFileBundle, analysisData: IBundleResult, li analysisResults, }; } -let analyzeFolderDefaults = { - baseURL: defaultBaseURL, - sessionToken: '', - includeLint: false, - reachability: false, - severity: AnalysisSeverity.info, - symlinksEnabled: false, - maxPayload: MAX_PAYLOAD, - defaultFileIgnores: IGNORES_DEFAULT, - sarif: false, - source: '', -}; + export async function analyzeFolders(options: FolderOptions): Promise { - const analysisOptions: AnalyzeFoldersOptions = { ...analyzeFolderDefaults, ...options }; + const analysisOptions: AnalyzeFoldersOptions = { ...ANALYSIS_OPTIONS_DEFAULTS, ...options }; const { baseURL, sessionToken, @@ -383,18 +385,8 @@ export async function extendAnalysis( ); } -const analyzeGitDefaults = { - baseURL: defaultBaseURL, - sessionToken: '', - includeLint: false, - reachability: false, - severity: AnalysisSeverity.info, - sarif: false, - source: '', -}; - export async function analyzeGit(options: GitOptions, requestOptions?: RequestOptions): Promise { - const analysisOptions: AnalyzeGitOptions = { ...analyzeGitDefaults, ...options }; + const analysisOptions: AnalyzeGitOptions = { ...ANALYSIS_OPTIONS_DEFAULTS, ...options }; const { baseURL, sessionToken, oAuthToken, username, includeLint, reachability, severity, gitUri, sarif, source } = analysisOptions; const bundleResponse = await createGitBundle( @@ -447,29 +439,22 @@ export async function analyzeGit(options: GitOptions, requestOptions?: RequestOp return result; } -interface CreateBundleFromFolders extends FolderOptions { +interface CreateBundleFromFoldersOptions extends FolderOptions { supportedFiles?: ISupportedFiles; baseDir?: string; fileIgnores?: string[]; } -let createBundleFromFoldersDefaults = { - baseURL: defaultBaseURL, - sessionToken: '', - symlinksEnabled: false, - maxPayload: MAX_PAYLOAD, - defaultFileIgnores: IGNORES_DEFAULT, - source: '', -}; + /** * Creates a remote bundle and returns response from the bundle API * - * @param {CreateBundleFromFolders} options + * @param {CreateBundleFromFoldersOptions} options * @returns {Promise} */ -export async function createBundleFromFolders(options: CreateBundleFromFolders): Promise { - const analysisOptions = { ...createBundleFromFoldersDefaults, ...options }; +export async function createBundleFromFolders(options: CreateBundleFromFoldersOptions): Promise { + const analysisOptions = { ...ANALYSIS_OPTIONS_DEFAULTS, ...options }; const { baseURL, From 7ebc3bfc7086ad837f3c09ee2437815e6c861fd0 Mon Sep 17 00:00:00 2001 From: Paulius Kutka Date: Wed, 26 May 2021 09:11:49 +0100 Subject: [PATCH 3/3] test: add tests for createBundleFromFolders --- tests/analysis.spec.ts | 308 +++++++++++++++++++++++------------------ 1 file changed, 170 insertions(+), 138 deletions(-) diff --git a/tests/analysis.spec.ts b/tests/analysis.spec.ts index 972e2792..93cdfa52 100644 --- a/tests/analysis.spec.ts +++ b/tests/analysis.spec.ts @@ -1,7 +1,7 @@ import path from 'path'; import jsonschema from 'jsonschema'; -import { analyzeFolders } from '../src/analysis'; +import { analyzeFolders, createBundleFromFolders } from '../src/analysis'; import { uploadRemoteBundle } from '../src/bundles'; import { baseURL, sessionToken, TEST_TIMEOUT } from './constants/base'; import { sampleProjectPath, bundleFiles, bundleFilesFull } from './constants/sample'; @@ -12,156 +12,188 @@ import { AnalysisSeverity } from '../src/interfaces/analysis-result.interface'; import * as sarifSchema from './sarif-schema-2.1.0.json'; describe('Functional test of analysis', () => { - it( - 'analyze folder', - async () => { - const onSupportedFilesLoaded = jest.fn((data: ISupportedFiles | null) => { - if (data === null) { - // all good - } - }); - emitter.on(emitter.events.supportedFilesLoaded, onSupportedFilesLoaded); + describe('analyzeFolders', () => { + it( + 'analyze folder', + async () => { + const onSupportedFilesLoaded = jest.fn((data: ISupportedFiles | null) => { + if (data === null) { + // all good + } + }); + emitter.on(emitter.events.supportedFilesLoaded, onSupportedFilesLoaded); + + const bFiles = await bundleFilesFull; + const onScanFilesProgress = jest.fn((processed: number) => { + expect(typeof processed).toBe('number'); + expect(processed).toBeGreaterThanOrEqual(0); + expect(processed).toBeLessThanOrEqual(bFiles.length); + }); + emitter.on(emitter.events.scanFilesProgress, onScanFilesProgress); + + const onCreateBundleProgress = jest.fn((processed: number, total: number) => { + expect(typeof processed).toBe('number'); + expect(total).toEqual(2); + + expect(processed).toBeLessThanOrEqual(total); + }); + emitter.on(emitter.events.createBundleProgress, onCreateBundleProgress); + + const onAnalyseProgress = jest.fn((data: AnalysisResponseProgress) => { + expect(['WAITING', 'FETCHING', 'ANALYZING', 'DC_DONE']).toContain(data.status); + expect(typeof data.progress).toBe('number'); + expect(data.progress).toBeGreaterThanOrEqual(0); + expect(data.progress).toBeLessThanOrEqual(100); + }); + emitter.on(emitter.events.analyseProgress, onAnalyseProgress); + + const onAPIRequestLog = jest.fn((message: string) => { + expect(typeof message).toBe('string'); + }); + emitter.on(emitter.events.apiRequestLog, onAPIRequestLog); + + const bundle = await analyzeFolders({ + baseURL, + sessionToken, + includeLint: false, + severity: 1, + paths: [sampleProjectPath], + symlinksEnabled: false, + maxPayload: 1000, + }); + expect(bundle).toHaveProperty('baseURL'); + expect(bundle).toHaveProperty('sessionToken'); + expect(bundle).toHaveProperty('supportedFiles'); + expect(bundle).toHaveProperty('analysisURL'); + expect(Object.keys(bundle.analysisResults.files).length).toEqual(4); + expect( + bundle.analysisResults.files.hasOwnProperty(`${sampleProjectPath}/GitHubAccessTokenScrambler12.java`), + ).toBeTruthy(); + expect(Object.keys(bundle.analysisResults.suggestions).length).toEqual(6); + + expect(bundle.analysisResults.timing.analysis).toBeGreaterThanOrEqual( + bundle.analysisResults.timing.fetchingCode, + ); + expect(bundle.analysisResults.timing.queue).toBeGreaterThanOrEqual(0); + expect(new Set(bundle.analysisResults.coverage)).toEqual( + new Set([ + { + files: 2, + isSupported: true, + lang: 'Java', + }, + { + files: 4, + isSupported: true, + lang: 'JavaScript', + }, + ]), + ); + + // Check if emitter event happened + expect(onSupportedFilesLoaded).toHaveBeenCalledTimes(2); + expect(onScanFilesProgress).toHaveBeenCalledTimes(7); + expect(onCreateBundleProgress).toHaveBeenCalledTimes(3); + expect(onAnalyseProgress).toHaveBeenCalled(); + expect(onAPIRequestLog).toHaveBeenCalled(); + + // Test uploadRemoteBundle with empty list of files + let uploaded = await uploadRemoteBundle(baseURL, sessionToken, bundle.bundleId, []); + // We do nothing in such cases + expect(uploaded).toEqual(true); + + const onUploadBundleProgress = jest.fn((processed: number, total: number) => { + expect(typeof processed).toBe('number'); + expect(total).toEqual(bFiles.length); + + expect(processed).toBeLessThanOrEqual(total); + }); + emitter.on(emitter.events.uploadBundleProgress, onUploadBundleProgress); + + // Forse uploading files one more time + uploaded = await uploadRemoteBundle(baseURL, sessionToken, bundle.bundleId, bFiles); + + expect(uploaded).toEqual(true); + + expect(onUploadBundleProgress).toHaveBeenCalledTimes(2); + expect(onAPIRequestLog).toHaveBeenCalled(); + }, + TEST_TIMEOUT, + ); + + it('analyze folder - with sarif returned', async () => { + const includeLint = false; + const severity = AnalysisSeverity.info; + const paths: string[] = [path.join(sampleProjectPath, 'only_text')]; + const symlinksEnabled = false; + const maxPayload = 1000; + const defaultFileIgnores = undefined; + const sarif = true; - const bFiles = await bundleFilesFull; - const onScanFilesProgress = jest.fn((processed: number) => { - expect(typeof processed).toBe('number'); - expect(processed).toBeGreaterThanOrEqual(0); - expect(processed).toBeLessThanOrEqual(bFiles.length); + const bundle = await analyzeFolders({ + baseURL, + sessionToken, + includeLint, + severity, + paths, + symlinksEnabled, + maxPayload, + defaultFileIgnores, + sarif, }); - emitter.on(emitter.events.scanFilesProgress, onScanFilesProgress); + const validationResult = jsonschema.validate(bundle.sarifResults, sarifSchema); - const onCreateBundleProgress = jest.fn((processed: number, total: number) => { - expect(typeof processed).toBe('number'); - expect(total).toEqual(2); + expect(validationResult.errors.length).toEqual(0); + }); - expect(processed).toBeLessThanOrEqual(total); - }); - emitter.on(emitter.events.createBundleProgress, onCreateBundleProgress); + it('analyze empty folder', async () => { + const includeLint = false; + const severity = AnalysisSeverity.info; + const paths: string[] = [path.join(sampleProjectPath, 'only_text')]; + const symlinksEnabled = false; + const maxPayload = 1000; - const onAnalyseProgress = jest.fn((data: AnalysisResponseProgress) => { - expect(['WAITING', 'FETCHING', 'ANALYZING', 'DC_DONE']).toContain(data.status); - expect(typeof data.progress).toBe('number'); - expect(data.progress).toBeGreaterThanOrEqual(0); - expect(data.progress).toBeLessThanOrEqual(100); + const bundle = await analyzeFolders({ + baseURL, + sessionToken, + includeLint, + severity, + paths, + symlinksEnabled, + maxPayload, }); - emitter.on(emitter.events.analyseProgress, onAnalyseProgress); - const onAPIRequestLog = jest.fn((message: string) => { - expect(typeof message).toBe('string'); - }); - emitter.on(emitter.events.apiRequestLog, onAPIRequestLog); + expect(bundle.analysisResults.files).toEqual({}); + expect(bundle.analysisResults.suggestions).toEqual({}); + expect(bundle.analysisResults.coverage).toEqual([]); + }); + }); - const bundle = await analyzeFolders({ + describe('createBundleFromFolders', () => { + it('should return a bundle with correct parameters', async () => { + const includeLint = false; + const severity = AnalysisSeverity.info; + const paths: string[] = [path.join(sampleProjectPath)]; + const symlinksEnabled = false; + const maxPayload = 1000; + const defaultFileIgnores = undefined; + + const result = await createBundleFromFolders({ baseURL, sessionToken, - includeLint: false, - severity: 1, - paths: [sampleProjectPath], - symlinksEnabled: false, - maxPayload: 1000, + includeLint, + severity, + paths, + symlinksEnabled, + maxPayload, + defaultFileIgnores, }); - expect(bundle).toHaveProperty('baseURL'); - expect(bundle).toHaveProperty('sessionToken'); - expect(bundle).toHaveProperty('supportedFiles'); - expect(bundle).toHaveProperty('analysisURL'); - expect(Object.keys(bundle.analysisResults.files).length).toEqual(4); - expect( - bundle.analysisResults.files.hasOwnProperty(`${sampleProjectPath}/GitHubAccessTokenScrambler12.java`), - ).toBeTruthy(); - expect(Object.keys(bundle.analysisResults.suggestions).length).toEqual(6); - - expect(bundle.analysisResults.timing.analysis).toBeGreaterThanOrEqual(bundle.analysisResults.timing.fetchingCode); - expect(bundle.analysisResults.timing.queue).toBeGreaterThanOrEqual(0); - expect(new Set(bundle.analysisResults.coverage)).toEqual( - new Set([ - { - files: 2, - isSupported: true, - lang: 'Java', - }, - { - files: 4, - isSupported: true, - lang: 'JavaScript', - }, - ]), - ); - - // Check if emitter event happened - expect(onSupportedFilesLoaded).toHaveBeenCalledTimes(2); - expect(onScanFilesProgress).toHaveBeenCalledTimes(7); - expect(onCreateBundleProgress).toHaveBeenCalledTimes(3); - expect(onAnalyseProgress).toHaveBeenCalled(); - expect(onAPIRequestLog).toHaveBeenCalled(); - - // Test uploadRemoteBundle with empty list of files - let uploaded = await uploadRemoteBundle(baseURL, sessionToken, bundle.bundleId, []); - // We do nothing in such cases - expect(uploaded).toEqual(true); - - const onUploadBundleProgress = jest.fn((processed: number, total: number) => { - expect(typeof processed).toBe('number'); - expect(total).toEqual(bFiles.length); - - expect(processed).toBeLessThanOrEqual(total); - }); - emitter.on(emitter.events.uploadBundleProgress, onUploadBundleProgress); - - // Forse uploading files one more time - uploaded = await uploadRemoteBundle(baseURL, sessionToken, bundle.bundleId, bFiles); - - expect(uploaded).toEqual(true); - - expect(onUploadBundleProgress).toHaveBeenCalledTimes(2); - expect(onAPIRequestLog).toHaveBeenCalled(); - }, - TEST_TIMEOUT, - ); - - it('analyze folder - with sarif returned', async () => { - const includeLint = false; - const severity = AnalysisSeverity.info; - const paths: string[] = [path.join(sampleProjectPath, 'only_text')]; - const symlinksEnabled = false; - const maxPayload = 1000; - const defaultFileIgnores = undefined; - const sarif = true; - - const bundle = await analyzeFolders({ - baseURL, - sessionToken, - includeLint, - severity, - paths, - symlinksEnabled, - maxPayload, - defaultFileIgnores, - sarif, - }); - const validationResult = jsonschema.validate(bundle.sarifResults, sarifSchema); - expect(validationResult.errors.length).toEqual(0); - }); + expect(result).not.toBeNull(); + expect(result).toHaveProperty("bundleId"); + expect(result).toHaveProperty("missingFiles"); + expect(result).toHaveProperty("uploadURL"); - it('analyze empty folder', async () => { - const includeLint = false; - const severity = AnalysisSeverity.info; - const paths: string[] = [path.join(sampleProjectPath, 'only_text')]; - const symlinksEnabled = false; - const maxPayload = 1000; - - const bundle = await analyzeFolders({ - baseURL, - sessionToken, - includeLint, - severity, - paths, - symlinksEnabled, - maxPayload, }); - - expect(bundle.analysisResults.files).toEqual({}); - expect(bundle.analysisResults.suggestions).toEqual({}); - expect(bundle.analysisResults.coverage).toEqual([]); }); });