Skip to content

Commit

Permalink
fix: write dep-graph payloads to stdout stream
Browse files Browse the repository at this point in the history
  • Loading branch information
mcombuechen committed Aug 9, 2024
1 parent 60c2939 commit c6a175d
Show file tree
Hide file tree
Showing 7 changed files with 135 additions and 45 deletions.
24 changes: 9 additions & 15 deletions src/lib/ecosystems/test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@ import { getPlugin } from './plugins';
import { TestDependenciesResponse } from '../snyk-test/legacy';
import {
assembleQueryString,
depGraphToOutputString,
printDepGraph,
shouldPrintDepGraph,
} from '../snyk-test/common';
import { getAuthHeader } from '../api-token';
import { resolveAndTestFacts } from './resolve-test-facts';
Expand Down Expand Up @@ -46,9 +47,14 @@ export async function testEcosystem(
}
spinner.clearAll();

if (isUnmanagedEcosystem(ecosystem) && options['print-graph']) {
if (isUnmanagedEcosystem(ecosystem) && shouldPrintDepGraph(options)) {
const [result] = await getUnmanagedDepGraph(results);
const depGraph = convertDepGraph(result);
const [target] = paths;
return formatUnmanagedResults(results, target);

await printDepGraph(depGraph, target, process.stdout);

return TestCommandResult.createJsonTestCommandResult('');
}

const [testResults, errors] = await selectAndExecuteTestStrategy(
Expand Down Expand Up @@ -87,18 +93,6 @@ export async function selectAndExecuteTestStrategy(
: await testDependencies(scanResultsByPath, options);
}

export async function formatUnmanagedResults(
results: ScanResultsByPath,
target: string,
): Promise<TestCommandResult> {
const [result] = await getUnmanagedDepGraph(results);
const depGraph = convertDepGraph(result);

return TestCommandResult.createJsonTestCommandResult(
depGraphToOutputString(depGraph, target),
);
}

async function testDependencies(
scans: {
[dir: string]: ScanResult[];
Expand Down
22 changes: 14 additions & 8 deletions src/lib/snyk-test/assemble-payloads.ts
Original file line number Diff line number Diff line change
@@ -1,16 +1,21 @@
import * as path from 'path';
import { DepGraph } from '@snyk/dep-graph';

import config from '../config';
import { isCI } from '../is-ci';
import { getPlugin } from '../ecosystems';
import { Ecosystem, ContainerTarget, ScanResult } from '../ecosystems/types';
import { Options, PolicyOptions, TestOptions } from '../types';
import { Payload } from './types';
import { assembleQueryString, depGraphToOutputString } from './common';
import {
assembleQueryString,
printDepGraph,
shouldPrintDepGraph,
} from './common';
import { spinner } from '../spinner';
import { findAndLoadPolicyForScanResult } from '../ecosystems/policy';
import { getAuthHeader } from '../../lib/api-token';
import { DockerImageNotFoundError } from '../errors';
import { DepGraph } from '@snyk/dep-graph';

export async function assembleEcosystemPayloads(
ecosystem: Ecosystem,
Expand Down Expand Up @@ -53,17 +58,18 @@ export async function assembleEcosystemPayloads(
scanResult.name =
options['project-name'] || config.PROJECT_NAME || scanResult.name;

if (options['print-graph'] && !options['print-deps']) {
if (shouldPrintDepGraph(options)) {
spinner.clear<void>(spinnerLbl)();

// not every scanResult has a 'depGraph' fact, for example the JAR
// fingerprints. I don't think we have another option than to skip
// those.
const dg = scanResult.facts.find((dg) => dg.type === 'depGraph');
if (dg) {
console.log(
depGraphToOutputString(
dg.data.toJSON(),
constructProjectName(scanResult),
),
await printDepGraph(
dg.data.toJSON(),
constructProjectName(scanResult),
process.stdout,
);
}
}
Expand Down
40 changes: 28 additions & 12 deletions src/lib/snyk-test/common.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
import { Readable, Writable } from 'stream';
import { JsonStreamStringify } from 'json-stream-stringify';
import { DepGraphData } from '@snyk/dep-graph';

import config from '../config';
import { color } from '../theme';
import { DepGraphData } from '@snyk/dep-graph';
import { jsonStringifyLargeObject } from '../json';
import { Options } from '../types';
import { ConcatStream } from '../stream';

export function assembleQueryString(options) {
const org = options.org || config.org || null;
Expand Down Expand Up @@ -71,15 +75,27 @@ export type FailOn = 'all' | 'upgradable' | 'patchable';
export const RETRY_ATTEMPTS = 3;
export const RETRY_DELAY = 500;

// depGraphData formats the given depGrahData with the targetName as expected by
// the `depgraph` CLI workflow.
export function depGraphToOutputString(
dg: DepGraphData,
/**
* printDepGraph writes the given dep-graph and target name to the destination
* stream as expected by the `depgraph` CLI workflow.
*/
export async function printDepGraph(
depGraph: DepGraphData,
targetName: string,
): string {
return `DepGraph data:
${jsonStringifyLargeObject(dg)}
DepGraph target:
${targetName}
DepGraph end`;
destination: Writable,
): Promise<void> {
return new Promise((res, rej) => {
new ConcatStream(
Readable.from('DepGraph data:\n'),
new JsonStreamStringify(depGraph),
Readable.from(`\nDepGraph target:\n${targetName}\nDepGraph end\n\n`),
)
.on('end', res)
.on('error', rej)
.pipe(destination);
});
}

export function shouldPrintDepGraph(opts: Options): boolean {
return opts['print-graph'] && !opts['print-deps'];
}
4 changes: 4 additions & 0 deletions src/lib/snyk-test/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,10 @@ async function executeTest(root, options) {
);
}

if (options['print-graph']) {
options.quiet = true;
}

return run(root, options, featureFlags).then((results) => {
for (const res of results) {
if (!res.packageManager) {
Expand Down
22 changes: 12 additions & 10 deletions src/lib/snyk-test/run-test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,13 @@ import {
} from '../errors';
import * as snyk from '../';
import { isCI } from '../is-ci';
import * as common from './common';
import { RETRY_ATTEMPTS, RETRY_DELAY } from './common';
import {
RETRY_ATTEMPTS,
RETRY_DELAY,
printDepGraph,
assembleQueryString,
shouldPrintDepGraph,
} from './common';
import config from '../config';
import * as analytics from '../analytics';
import { maybePrintDepGraph, maybePrintDepTree } from '../print-deps';
Expand Down Expand Up @@ -341,7 +346,7 @@ export async function runTest(
try {
const payloads = await assemblePayloads(root, options, featureFlags);

if (options['print-graph'] && !options['print-deps']) {
if (shouldPrintDepGraph(options)) {
const results: TestResult[] = [];
return results;
}
Expand Down Expand Up @@ -754,8 +759,7 @@ async function assembleLocalPayloads(
? (pkg as depGraphLib.DepGraph).rootPkg.name
: (pkg as DepTree).name;

// print dep graph if `--print-graph` is set
if (options['print-graph'] && !options['print-deps']) {
if (shouldPrintDepGraph(options)) {
spinner.clear<void>(spinnerLbl)();
let root: depGraphLib.DepGraph;
if (scannedProject.depGraph) {
Expand All @@ -768,9 +772,7 @@ async function assembleLocalPayloads(
);
}

console.log(
common.depGraphToOutputString(root.toJSON(), targetFile || ''),
);
await printDepGraph(root.toJSON(), targetFile || '', process.stdout);
}

const body: PayloadBody = {
Expand Down Expand Up @@ -829,7 +831,7 @@ async function assembleLocalPayloads(
'x-is-ci': isCI(),
authorization: getAuthHeader(),
},
qs: common.assembleQueryString(options),
qs: assembleQueryString(options),
body,
};

Expand Down Expand Up @@ -871,7 +873,7 @@ async function assembleRemotePayloads(root, options): Promise<Payload[]> {
{
method: 'GET',
url,
qs: common.assembleQueryString(options),
qs: assembleQueryString(options),
json: true,
headers: {
'x-is-ci': isCI(),
Expand Down
37 changes: 37 additions & 0 deletions src/lib/stream.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
import { Readable } from 'stream';

export class ConcatStream extends Readable {
private current: Readable | undefined;
private queue: Readable[] = [];

constructor(...streams: Readable[]) {
super({ objectMode: false }); // Adjust objectMode if needed
this.queue.push(...streams);
}

append(...streams: Readable[]): void {
this.queue.push(...streams);
if (!this.current) {
this._read();
}
}

_read(size?: number): void {
if (this.current) {
return;
}

this.current = this.queue.shift();
if (!this.current) {
this.push(null);
return;
}

this.current.on('data', (chunk) => this.push(chunk));
this.current.on('end', () => {
this.current = undefined;
this._read(size);
});
this.current.on('error', (err) => this.emit('error', err));
}
}
31 changes: 31 additions & 0 deletions test/jest/unit/lib/stream.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
import { Readable, Writable } from 'stream';

import { ConcatStream } from '../../../../src/lib/stream';

describe('ConcatStream', () => {
it('should create a readable stream', () => {
const stream = new ConcatStream();

expect(stream).toBeInstanceOf(Readable);
});

it('should concatenate readable streams', async () => {
const stream = new ConcatStream();
const chunks = jest.fn();
const out = new Writable({
write: (chunk, enc, done) => {
chunks(chunk.toString());
done();
},
});

stream.append(Readable.from('foo'), Readable.from('bar'));

await new Promise((res) => {
stream.pipe(out).on('finish', res);
});

expect(chunks).toHaveBeenCalledWith('foo');
expect(chunks).toHaveBeenCalledWith('bar');
});
});

0 comments on commit c6a175d

Please sign in to comment.