Merge pull request #108 from snyk/feat/legacy-json-support

Feat/legacy json support

development.md

@@ -4,14 +4,14 @@ To use and debug package locally you don't need publish it to NPM registry:
 
 ```shell script
 $ cd <package-location>
-$ npm install && npm run build && npx yalc publish
+$ npm install && npm run build && npm link
 ```
 
 After that you have to create symlink to your package in your project folder:
 
 ```shell script
 $ cd <project-location>
-$ npx yalc add @snyk/code-client
+$ npm link @snyk/code-client
 ```
 
 ## Publishing

package.json

@@ -58,8 +58,7 @@
     "prettier": "^2.1.1",
     "ts-jest": "^26.3.0",
     "typescript": "^4.0.2",
-    "write": "^2.0.0",
-    "yalc": "^1.0.0-pre.53"
+    "write": "^2.0.0"
   },
   "dependencies": {
     "@deepcode/dcignore": "^1.0.4",

src/analysis.ts

@@ -1,6 +1,8 @@
 /* eslint-disable no-await-in-loop */
+import omit from 'lodash.omit';
 
-import { AnalyzeFoldersOptions, prepareExtendingBundle } from './files';
+import { AnalyzeFoldersOptions, prepareExtendingBundle, resolveBundleFilePath } from './files';
+import { POLLING_INTERVAL } from './constants';
 import {
   GetAnalysisErrorCodes,
   getAnalysis,
@@ -12,9 +14,10 @@ import {
   ConnectionOptions,
   GetAnalysisOptions,
 } from './http';
+import { fromEntries } from './lib/utils';
 import { createBundleFromFolders, FileBundle, remoteBundleFactory } from './bundles';
-import emitter from './emitter';
-import { AnalysisResult } from './interfaces/analysis-result.interface';
+import { emitter } from './emitter';
+import { AnalysisResult, AnalysisResultLegacy, AnalysisResultSarif, AnalysisFiles, Suggestion } from './interfaces/analysis-result.interface';
 
 const sleep = (duration: number) => new Promise(resolve => setTimeout(resolve, duration));
 
@@ -56,7 +59,7 @@ async function pollAnalysis(
       return analysisResponse as Result<AnalysisFailedResponse, GetAnalysisErrorCodes>;
     }
 
-    await sleep(500);
+    await sleep(POLLING_INTERVAL);
   }
 }
 
@@ -73,25 +76,25 @@ export async function analyzeBundle(options: GetAnalysisOptions): Promise<Analys
   return analysisData.value;
 }
 
-// function normalizeResultFiles(files: AnalysisFiles, baseDir: string): AnalysisFiles {
-//   if (baseDir) {
-//     return fromEntries(
-//       Object.entries(files).map(([path, positions]) => {
-//         const filePath = resolveBundleFilePath(baseDir, path);
-//         return [filePath, positions];
-//       }),
-//     );
-//   }
-//   return files;
-// }
+function normalizeResultFiles(files: AnalysisFiles, baseDir: string): AnalysisFiles {
+  if (baseDir) {
+    return fromEntries(
+      Object.entries(files).map(([path, positions]) => {
+        const filePath = resolveBundleFilePath(baseDir, path);
+        return [filePath, positions];
+      }),
+    );
+  }
+  return files;
+}
 
 interface FileAnalysisOptions {
   connection: ConnectionOptions;
   analysisOptions: AnalysisOptions;
   fileOptions: AnalyzeFoldersOptions;
 }
 
-interface FileAnalysis extends FileAnalysisOptions {
+export interface FileAnalysis extends FileAnalysisOptions {
   fileBundle: FileBundle;
   analysisResults: AnalysisResult;
 }
@@ -109,18 +112,36 @@ export async function analyzeFolders(options: FileAnalysisOptions): Promise<File
     ...options.connection,
     ...options.analysisOptions,
   });
-  // TODO: expand relative file names to absolute ones
-  // analysisResults.files = normalizeResultFiles(analysisData.analysisResults.files, baseDir);
+
+  if (analysisResults.type === 'legacy') {
+    // expand relative file names to absolute ones only for legacy results
+    analysisResults.files = normalizeResultFiles(analysisResults.files, fileBundle.baseDir);
+  }
 
   return { fileBundle, analysisResults, ...options };
 }
 
-function mergeBundleResults(
-  oldAnalysisResults: AnalysisResult,
+function mergeBundleResults(oldAnalysisResults: AnalysisResult,
   newAnalysisResults: AnalysisResult,
   limitToFiles: string[],
   removedFiles: string[] = [],
+  baseDir: string,
 ): AnalysisResult {
+
+  if (newAnalysisResults.type == 'sarif') {
+    return mergeSarifResults(oldAnalysisResults as AnalysisResultSarif, newAnalysisResults, limitToFiles, removedFiles);
+  }
+
+  return mergeLegacyResults(oldAnalysisResults as AnalysisResultLegacy, newAnalysisResults, limitToFiles, removedFiles, baseDir);
+
+}
+
+function mergeSarifResults(
+  oldAnalysisResults: AnalysisResultSarif,
+  newAnalysisResults: AnalysisResultSarif,
+  limitToFiles: string[],
+  removedFiles: string[] = [],
+): AnalysisResultSarif {
   // Start from the new analysis results
   // For each finding of the old analysis,
   //  if it's location is not part of the limitToFiles or removedFiles (removedFiles should also be checked against condeFlow),
@@ -189,6 +210,57 @@ function mergeBundleResults(
   return newAnalysisResults;
 }
 
+const moveSuggestionIndexes = <T>(
+  suggestionIndex: number,
+  suggestions: { [index: string]: T },
+): { [index: string]: T } => {
+  const entries = Object.entries(suggestions);
+  return fromEntries(
+    entries.map(([i, s]) => {
+      return [`${parseInt(i, 10) + suggestionIndex + 1}`, s];
+    }),
+  );
+};
+
+function mergeLegacyResults(
+  oldAnalysisResults: AnalysisResultLegacy,
+  newAnalysisResults: AnalysisResultLegacy,
+  limitToFiles: string[],
+  removedFiles: string[] = [],
+  baseDir: string,
+  ): AnalysisResultLegacy {
+
+  // expand relative file names to absolute ones only for legacy results
+  newAnalysisResults.files = normalizeResultFiles(newAnalysisResults.files, baseDir);
+
+  // Determine max suggestion index in our data
+  const suggestionIndex = Math.max(...Object.keys(oldAnalysisResults.suggestions).map(i => parseInt(i, 10))) || -1;
+
+  // Addup all new suggestions' indexes
+  const newSuggestions = moveSuggestionIndexes<Suggestion>(suggestionIndex, newAnalysisResults.suggestions);
+  const suggestions = { ...oldAnalysisResults.suggestions, ...newSuggestions };
+
+  const newFiles = fromEntries(
+    Object.entries(newAnalysisResults.files).map(([fn, s]) => {
+      return [fn, moveSuggestionIndexes(suggestionIndex, s)];
+    }),
+  );
+
+  // expand relative file names to absolute ones only for legacy results
+  const changedFiles = [...limitToFiles, ...removedFiles].map(path => resolveBundleFilePath(baseDir, path));
+
+  const files = {
+    ...omit(oldAnalysisResults.files, changedFiles),
+    ...newFiles,
+  };
+
+  return {
+    ...newAnalysisResults,
+    files,
+    suggestions,
+  };
+}
+
 interface ExtendAnalysisOptions extends FileAnalysis {
   files: string[];
 }
@@ -234,10 +306,7 @@ export async function extendAnalysis(options: ExtendAnalysisOptions): Promise<Fi
     limitToFiles,
   });
 
-  // TODO: Transform relative paths into absolute
-  // analysisData.analysisResults.files = normalizeResultFiles(analysisData.analysisResults.files, bundle.baseDir);
-
-  analysisResults = mergeBundleResults(options.analysisResults, analysisResults, limitToFiles, removedFiles);
+  analysisResults = mergeBundleResults(options.analysisResults, analysisResults, limitToFiles, removedFiles, options.fileBundle.baseDir);
 
   return { ...options, fileBundle, analysisResults };
 }

src/bundles.ts

@@ -28,7 +28,7 @@ import {
 } from './http';
 
 import { MAX_PAYLOAD, MAX_UPLOAD_ATTEMPTS, UPLOAD_CONCURRENCY } from './constants';
-import emitter from './emitter';
+import { emitter } from './emitter';
 
 type BundleErrorCodes = CreateBundleErrorCodes | CheckBundleErrorCodes | ExtendBundleErrorCodes;
 

src/constants.ts

@@ -12,6 +12,7 @@ export const EXCLUDED_NAMES = [GIT_FILENAME, GITIGNORE_FILENAME, DCIGNORE_FILENA
 export const CACHE_KEY = '.dccache';
 export const MAX_UPLOAD_ATTEMPTS = 5;
 export const UPLOAD_CONCURRENCY = 5;
+export const POLLING_INTERVAL = 500;
 export const MAX_RETRY_ATTEMPTS = 5; // Request retries on network errors
 export const REQUEST_RETRY_DELAY = 30 * 1000; // 30 seconds delay between retries
 

src/emitter.ts

@@ -46,5 +46,4 @@ export class EmitterDC extends EventEmitter {
   }
 }
 
-const emitter = new EmitterDC();
-export default emitter;
+export const emitter = new EmitterDC();

src/http.ts

@@ -4,7 +4,6 @@ import pick from 'lodash.pick';
 import { ErrorCodes, GenericErrorTypes, DEFAULT_ERROR_MESSAGES } from './constants';
 
 import { BundleFiles, SupportedFiles } from './interfaces/files.interface';
-// import { AnalysisSeverity } from './interfaces/analysis-options.interface';
 import { AnalysisResult } from './interfaces/analysis-result.interface';
 import { makeRequest, Payload } from './needle';
 
@@ -386,6 +385,7 @@ export interface AnalysisOptions {
   readonly severity?: number;
   readonly limitToFiles?: string[];
   readonly prioritized?: boolean;
+  readonly legacy?: boolean;
 }
 
 export interface GetAnalysisOptions extends ConnectionOptions, AnalysisOptions {
@@ -408,8 +408,7 @@ export async function getAnalysis(
         hash: options.bundleHash,
         limitToFiles: options.limitToFiles || [],
       },
-      ...pick(options, ['severity', 'prioritized']),
-      // severity: options.severity || AnalysisSeverity.info,
+      ...pick(options, ['severity', 'prioritized', 'legacy']),
     },
   };
 

src/index.ts

@@ -1,25 +1,33 @@
-import { analyzeFolders } from './analysis';
+import { analyzeFolders, extendAnalysis, FileAnalysis } from './analysis';
 import { createBundleFromFolders } from './bundles';
-import emitter from './emitter';
-import { startSession, checkSession, getAnalysis } from './http';
+import { emitter } from './emitter';
+import { startSession, checkSession, getAnalysis, getIpFamily, IpFamily } from './http';
 import * as constants from './constants';
 import { getGlobPatterns } from './files';
 
 import { SupportedFiles } from './interfaces/files.interface';
 import { AnalysisSeverity } from './interfaces/analysis-options.interface';
-import { AnalysisResult } from './interfaces/analysis-result.interface';
+import { AnalysisResult, AnalysisResultLegacy, FilePath, FileSuggestion, Suggestion, Marker } from './interfaces/analysis-result.interface';
 
 export {
   getGlobPatterns,
   analyzeFolders,
   createBundleFromFolders,
-  // extendAnalysis,
-  getAnalysis,
-  startSession,
-  checkSession,
+  extendAnalysis,
   emitter,
   constants,
   AnalysisSeverity,
   AnalysisResult,
+  AnalysisResultLegacy,
   SupportedFiles,
+  FileAnalysis,
+  FilePath,
+  FileSuggestion,
+  Suggestion,
+  Marker,
+  getAnalysis,
+  startSession,
+  checkSession,
+  getIpFamily,
+  IpFamily,
 };

src/interfaces/analysis-result.interface.ts

@@ -1,13 +1,13 @@
 import { Log } from 'sarif';
+import { AnalysisSeverity } from './analysis-options.interface';
 
 interface Coverage {
   files: number;
   isSupported: boolean;
   lang: string;
 }
 
-export interface AnalysisResult {
-  sarif: Log;
+interface AnalysisResultBase {
   timing: {
     fetchingCode: number;
     analysis: number;
@@ -16,3 +16,74 @@ export interface AnalysisResult {
   coverage: Coverage[];
   status: 'COMPLETE';
 }
+
+export interface AnalysisResultSarif extends AnalysisResultBase {
+  type: 'sarif';
+  sarif: Log;
+}
+
+export interface Position {
+  cols: Point;
+  rows: Point;
+}
+
+export interface MarkerPosition extends Position {
+  file: string;
+}
+
+export type Point = [number, number];
+
+export interface Marker {
+  msg: Point;
+  pos: MarkerPosition[];
+}
+
+export interface FileSuggestion extends Position {
+  markers?: Marker[];
+}
+
+export interface FilePath {
+  [suggestionIndex: string]: FileSuggestion[];
+}
+
+export interface AnalysisFiles {
+  [filePath: string]: FilePath;
+}
+
+interface CommitChangeLine {
+  line: string;
+  lineNumber: number;
+  lineChange: 'removed' | 'added' | 'none';
+}
+
+interface ExampleCommitFix {
+  commitURL: string;
+  lines: CommitChangeLine[];
+}
+
+export interface Suggestion {
+  id: string;
+  message: string;
+  severity: AnalysisSeverity;
+  leadURL?: string;
+  rule: string;
+  tags: string[];
+  categories: string[];
+  repoDatasetSize: number;
+  exampleCommitDescriptions: string[];
+  exampleCommitFixes: ExampleCommitFix[];
+  cwe: string[];
+  title: string;
+  text: string;
+}
+
+export interface Suggestions {
+  [suggestionIndex: string]: Suggestion;
+}
+export interface AnalysisResultLegacy extends AnalysisResultBase {
+  type: 'legacy';
+  suggestions: Suggestions;
+  files: AnalysisFiles;
+}
+
+export type AnalysisResult = AnalysisResultSarif | AnalysisResultLegacy;