diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/action/helper/ActionDescriptor.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/action/helper/ActionDescriptor.java index 40b071b8aa..dad1b4f23b 100644 --- a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/action/helper/ActionDescriptor.java +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/action/helper/ActionDescriptor.java @@ -243,6 +243,8 @@ public static final class ActionStepDescriptor implements IActionIfSupplier { private List set; /** Optional write operations */ private List write; + /** Optional forEach operation */ + private ActionStepForEachDescriptor forEach; /** * This method is invoked by the parent element (which may either be another @@ -253,38 +255,7 @@ public final void postLoad(ActionDescriptor action) { if ( requests!=null ) { requests.forEach(d->d.postLoad(action)); } if ( set!=null ) { set.forEach(d->d.postLoad(action)); } if ( write!=null ) { write.forEach(d->d.postLoad(action)); } - } - } - - /** - * This class describes a forEach element, allowing iteration over the output of - * the parent element, like the response of a REST request or the contents of a - * action parameter. - */ - @Reflectable @NoArgsConstructor - @Data - public static final class ActionStepForEachDescriptor implements IActionIfSupplier { - /** Optional if-expression, executing steps only if condition evaluates to true */ - @JsonProperty("if") private SimpleExpression _if; - /** Optional break-expression, terminating forEach if condition evaluates to true */ - private SimpleExpression breakIf; - /** Required name for this step element */ - private String name; - /** Optional requests for which to embed the response in each forEach node */ - private List embed; - /** Steps to be repeated for each value */ - @JsonProperty("do") private List _do; - - /** - * This method is invoked by the {@link ActionStepDescriptor#postLoad()} - * method. It checks that required properties are set, then calls the postLoad() method for - * each sub-step. - */ - public final void postLoad(ActionDescriptor action) { - checkNotBlank("forEach name", name, this); - checkNotNull("forEach do", _do, this); - if ( embed!=null ) { embed.forEach(d->d.postLoad(action)); } - _do.forEach(d->d.postLoad(action)); + if ( forEach!=null ) { forEach.postLoad(action); } } } @@ -316,14 +287,12 @@ public void postLoad(ActionDescriptor action) { ()->"No value template found with name "+valueTemplate); } } - } - - // Ideally, this should be an inner class on ActionStepRequestDescriptor, but this causes - // issues with native images; see https://github.com/formkiq/graalvm-annotations-processor/issues/11 - // TODO: Add other operations like 'merge' for merging two ObjectNodes or ArrayNodes? - @Reflectable - public static enum ActionStepSetOperation { - replace, append; + + // TODO: Add other operations like 'merge' for merging two ObjectNodes or ArrayNodes? + @Reflectable + public static enum ActionStepSetOperation { + replace, append, merge; + } } /** @@ -348,6 +317,45 @@ public void postLoad(ActionDescriptor action) { } } + /** + * This class describes a forEach element, allowing iteration over the output of + * a given input. + */ + @Reflectable @NoArgsConstructor + @Data + public static final class ActionStepForEachDescriptor implements IActionIfSupplier { + /** Processor that runs the forEach steps. This expression must evaluate to an + * IActionStepForEachProcessor instance. */ + private SimpleExpression processor; + /** Optional if-expression, executing steps only if condition evaluates to true */ + @JsonProperty("if") private SimpleExpression _if; + /** Optional break-expression, terminating forEach if condition evaluates to true */ + private SimpleExpression breakIf; + /** Required name for this step element */ + private String name; + /** Steps to be repeated for each value */ + @JsonProperty("do") private List _do; + + /** + * This method is invoked by the {@link ActionStepDescriptor#postLoad()} + * method. It checks that required properties are set, then calls the postLoad() method for + * each sub-step. + */ + public final void postLoad(ActionDescriptor action) { + checkNotBlank("forEach name", name, this); + checkNotNull("forEach do", _do, this); + _do.forEach(d->d.postLoad(action)); + } + + @FunctionalInterface + public static interface IActionStepForEachProcessor { + /** Implementations of this method should invoke the given function for every + * JsonNode to be processed, and terminate processing if the given function + * returns false. */ + public void process(Function consumer); + } + } + /** * This class describes a REST request. */ @@ -377,7 +385,7 @@ public static final class ActionStepRequestDescriptor implements IActionIfSuppli /** Optional steps to be executed on request failure; if not declared, an exception will be thrown */ private List onFail; /** Optional forEach block to be repeated for every response element */ - private ActionStepForEachDescriptor forEach; + private ActionStepRequestForEachDescriptor forEach; /** * This method is invoked by {@link ActionStepDescriptor#postLoad()} @@ -397,23 +405,51 @@ protected final void postLoad(ActionDescriptor action) { forEach.postLoad(action); } } - } - - // Ideally, this should be an inner class on ActionStepRequestDescriptor, but this causes - // issues with native images; see https://github.com/formkiq/graalvm-annotations-processor/issues/11 - @Reflectable - public static enum ActionStepRequestType { - simple, paged - } - - // Ideally, this should be an inner class on ActionStepRequestDescriptor, but this causes - // issues with native images; see https://github.com/formkiq/graalvm-annotations-processor/issues/11 - @Reflectable @NoArgsConstructor - @Data - public static final class ActionStepRequestPagingProgressDescriptor { - private TemplateExpression prePageLoad; - private TemplateExpression postPageLoad; - private TemplateExpression postPageProcess; + + /** + * This class describes a request forEach element, allowing iteration over the output of + * the parent element, like the response of a REST request or the contents of a + * action parameter. + */ + @Reflectable @NoArgsConstructor + @Data + public static final class ActionStepRequestForEachDescriptor implements IActionIfSupplier { + /** Optional if-expression, executing steps only if condition evaluates to true */ + @JsonProperty("if") private SimpleExpression _if; + /** Optional break-expression, terminating forEach if condition evaluates to true */ + private SimpleExpression breakIf; + /** Required name for this step element */ + private String name; + /** Optional requests for which to embed the response in each forEach node */ + private List embed; + /** Steps to be repeated for each value */ + @JsonProperty("do") private List _do; + + /** + * This method is invoked by the {@link ActionStepDescriptor#postLoad()} + * method. It checks that required properties are set, then calls the postLoad() method for + * each sub-step. + */ + public final void postLoad(ActionDescriptor action) { + checkNotBlank("forEach name", name, this); + checkNotNull("forEach do", _do, this); + if ( embed!=null ) { embed.forEach(d->d.postLoad(action)); } + _do.forEach(d->d.postLoad(action)); + } + } + + @Reflectable + public static enum ActionStepRequestType { + simple, paged + } + + @Reflectable @NoArgsConstructor + @Data + public static final class ActionStepRequestPagingProgressDescriptor { + private TemplateExpression prePageLoad; + private TemplateExpression postPageLoad; + private TemplateExpression postPageProcess; + } } /** diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/action/helper/ActionRunner.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/action/helper/ActionRunner.java index 0154e15db2..e31809b918 100644 --- a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/action/helper/ActionRunner.java +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/action/helper/ActionRunner.java @@ -48,11 +48,13 @@ import com.fortify.cli.common.action.helper.ActionDescriptor.ActionRequestTargetDescriptor; import com.fortify.cli.common.action.helper.ActionDescriptor.ActionStepDescriptor; import com.fortify.cli.common.action.helper.ActionDescriptor.ActionStepForEachDescriptor; +import com.fortify.cli.common.action.helper.ActionDescriptor.ActionStepForEachDescriptor.IActionStepForEachProcessor; import com.fortify.cli.common.action.helper.ActionDescriptor.ActionStepRequestDescriptor; -import com.fortify.cli.common.action.helper.ActionDescriptor.ActionStepRequestPagingProgressDescriptor; -import com.fortify.cli.common.action.helper.ActionDescriptor.ActionStepRequestType; +import com.fortify.cli.common.action.helper.ActionDescriptor.ActionStepRequestDescriptor.ActionStepRequestForEachDescriptor; +import com.fortify.cli.common.action.helper.ActionDescriptor.ActionStepRequestDescriptor.ActionStepRequestPagingProgressDescriptor; +import com.fortify.cli.common.action.helper.ActionDescriptor.ActionStepRequestDescriptor.ActionStepRequestType; import com.fortify.cli.common.action.helper.ActionDescriptor.ActionStepSetDescriptor; -import com.fortify.cli.common.action.helper.ActionDescriptor.ActionStepSetOperation; +import com.fortify.cli.common.action.helper.ActionDescriptor.ActionStepSetDescriptor.ActionStepSetOperation; import com.fortify.cli.common.action.helper.ActionDescriptor.ActionStepWriteDescriptor; import com.fortify.cli.common.action.helper.ActionDescriptor.ActionValidationException; import com.fortify.cli.common.action.helper.ActionDescriptor.ActionValueTemplateDescriptor; @@ -155,7 +157,7 @@ public final ActionRunner addRequestHelper(String name, IActionRequestHelper req return this; } - private IActionRequestHelper getRequestHelper(String name) { + public IActionRequestHelper getRequestHelper(String name) { if ( StringUtils.isBlank(name) ) { if ( requestHelpers.size()==1 ) { return requestHelpers.values().iterator().next(); @@ -307,6 +309,7 @@ private final void processStep(ActionStepDescriptor step) { processSupplier(step::getDebug, this::processDebugStep); processSupplier(step::get_throw, this::processThrowStep); processSupplier(step::getRequests, this::processRequestsStep); + processSupplier(step::getForEach, this::processForEachStep); processAll(step::getSet, this::processSetStep); processAll(step::getWrite, this::processWriteStep); } @@ -369,6 +372,15 @@ private JsonNode getTargetValueForSetStep(ActionStepSetDescriptor set, JsonNode throw new IllegalStateException("Cannot append value to existing value of type "+result.getNodeType()); } ((ArrayNode)result).add(value); + } else if ( op==ActionStepSetOperation.merge ) { + result = localData.get(name); + if ( result==null ) { + result = objectMapper.createObjectNode(); + } + if ( !result.isObject() || !value.isObject() ) { + throw new IllegalStateException(String.format("Only ObjectNodes can be merged (existing value type: %s, type of value to be merged: %s) ", result.getNodeType(), value.getNodeType())); + } + ((ObjectNode)result).setAll((ObjectNode)value); } return result; } @@ -438,6 +450,19 @@ private void processThrowStep(TemplateExpression message) { throw new RuntimeException(spelEvaluator.evaluate(message, localData, String.class)); } + private void processForEachStep(ActionStepForEachDescriptor forEach) { + spelEvaluator.evaluate(forEach.getProcessor(), localData, IActionStepForEachProcessor.class) + .process(node->processForEachStepNode(forEach, node)); + } + + private boolean processForEachStepNode(ActionStepForEachDescriptor forEach, JsonNode node) { + setDataValue(forEach.getName(), node); + if ( _if(forEach) ) { + processSteps(forEach.get_do()); + } + return true; + } + private void processRequestsStep(List requests) { if ( requests!=null ) { var requestsProcessor = new ActionStepRequestsProcessor(); @@ -452,7 +477,7 @@ private final void processResponse(ActionStepRequestDescriptor requestDescriptor localData.set(name+"_raw", rawBody); localData.set(name, body); processOnResponse(requestDescriptor); - processForEach(requestDescriptor); + processRequestStepForEach(requestDescriptor); } private final void processFailure(ActionStepRequestDescriptor requestDescriptor, UnirestException e) { @@ -467,15 +492,15 @@ private final void processOnResponse(ActionStepRequestDescriptor requestDescript processSteps(onResponseSteps); } - private final void processForEach(ActionStepRequestDescriptor requestDescriptor) { + private final void processRequestStepForEach(ActionStepRequestDescriptor requestDescriptor) { var forEach = requestDescriptor.getForEach(); if ( forEach!=null ) { var input = localData.get(requestDescriptor.getName()); if ( input!=null ) { if ( input instanceof ArrayNode ) { - updateForEachTotalCount(forEach, (ArrayNode)input); - processForEachEmbed(forEach, (ArrayNode)input); - processForEach(forEach, (ArrayNode)input, this::processForEachEntryDo); + updateRequestStepForEachTotalCount(forEach, (ArrayNode)input); + processRequestStepForEachEmbed(forEach, (ArrayNode)input); + processRequestStepForEach(forEach, (ArrayNode)input, this::processRequestStepForEachEntryDo); } else { throw new ActionValidationException("forEach not supported on node type "+input.getNodeType()); } @@ -483,18 +508,18 @@ private final void processForEach(ActionStepRequestDescriptor requestDescriptor) } } - private final void processForEachEmbed(ActionStepForEachDescriptor forEach, ArrayNode source) { + private final void processRequestStepForEachEmbed(ActionStepRequestForEachDescriptor forEach, ArrayNode source) { var requestExecutor = new ActionStepRequestsProcessor(); - processForEach(forEach, source, getForEachEntryEmbedProcessor(requestExecutor)); + processRequestStepForEach(forEach, source, getRequestForEachEntryEmbedProcessor(requestExecutor)); requestExecutor.executeRequests(); } @FunctionalInterface - private interface IForEachEntryProcessor { - void process(ActionStepForEachDescriptor forEach, JsonNode currentNode, ObjectNode newData); + private interface IRequestStepForEachEntryProcessor { + void process(ActionStepRequestForEachDescriptor forEach, JsonNode currentNode, ObjectNode newData); } - private final void processForEach(ActionStepForEachDescriptor forEach, ArrayNode source, IForEachEntryProcessor entryProcessor) { + private final void processRequestStepForEach(ActionStepRequestForEachDescriptor forEach, ArrayNode source, IRequestStepForEachEntryProcessor entryProcessor) { for ( int i = 0 ; i < source.size(); i++ ) { var currentNode = source.get(i); var newData = JsonHelper.shallowCopy(localData); @@ -510,19 +535,19 @@ private final void processForEach(ActionStepForEachDescriptor forEach, ArrayNode } } - private void updateForEachTotalCount(ActionStepForEachDescriptor forEach, ArrayNode array) { + private void updateRequestStepForEachTotalCount(ActionStepRequestForEachDescriptor forEach, ArrayNode array) { var totalCountName = String.format("total%sCount", StringUtils.capitalize(forEach.getName())); var totalCount = localData.get(totalCountName); if ( totalCount==null ) { totalCount = new IntNode(0); } localData.put(totalCountName, totalCount.asInt()+array.size()); } - private void processForEachEntryDo(ActionStepForEachDescriptor forEach, JsonNode currentNode, ObjectNode newData) { + private void processRequestStepForEachEntryDo(ActionStepRequestForEachDescriptor forEach, JsonNode currentNode, ObjectNode newData) { var processor = new ActionStepsProcessor(newData, this); processor.processSteps(forEach.get_do()); } - private IForEachEntryProcessor getForEachEntryEmbedProcessor(ActionStepRequestsProcessor requestExecutor) { + private IRequestStepForEachEntryProcessor getRequestForEachEntryEmbedProcessor(ActionStepRequestsProcessor requestExecutor) { return (forEach, currentNode, newData) -> { if ( !currentNode.isObject() ) { // TODO Improve exception message? @@ -592,6 +617,7 @@ private void executeRequest(String target, List request } public static interface IActionRequestHelper extends AutoCloseable { + public UnirestInstance getUnirestInstance(); public JsonNode transformInput(JsonNode input); public void executePagedRequest(ActionRequestDescriptor requestDescriptor); public void executeSimpleRequests(List requestDescriptor); diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/action/helper/ActionSpelFunctions.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/action/helper/ActionSpelFunctions.java index 7a35a93106..0d56443a07 100644 --- a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/action/helper/ActionSpelFunctions.java +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/action/helper/ActionSpelFunctions.java @@ -13,6 +13,7 @@ package com.fortify.cli.common.action.helper; import java.time.LocalDateTime; +import java.time.Year; import java.time.ZoneId; import java.time.ZoneOffset; import java.time.ZonedDateTime; @@ -23,6 +24,7 @@ import org.jsoup.Jsoup; import org.jsoup.nodes.Document; +import org.jsoup.nodes.TextNode; import org.jsoup.safety.Safelist; import com.formkiq.graalvm.annotations.Reflectable; @@ -76,12 +78,37 @@ public static final String htmlToText(String html) { if( html==null ) { return null; } Document document = Jsoup.parse(html); document.outputSettings(new Document.OutputSettings().prettyPrint(false));//makes html() preserve linebreaks and spacing + document.select("li").append("\\n"); document.select("br").append("\\n"); document.select("p").prepend("\\n\\n"); String s = document.html().replaceAll("\\\\n", "\n"); return Jsoup.clean(s, "", Safelist.none(), new Document.OutputSettings().prettyPrint(false)); } + public static final String cleanRuleDescription(String description) { + if( description==null ) { return ""; } + Document document = Jsoup.parse(description); + document.outputSettings(new Document.OutputSettings().prettyPrint(false));//makes html() preserve linebreaks and spacing + document.select("li").append("\\n"); + document.select("br").append("\\n"); + document.select("p").prepend("\\n\\n"); + var paragraphs = document.select("Paragraph"); + for ( var p : paragraphs ) { + var altParagraph = p.select("AltParagraph"); + if ( !altParagraph.isEmpty() ) { + p.replaceWith(new TextNode(String.join("\n\n",altParagraph.eachText()))); + } else { + p.remove(); + } + + } + document.select("IfDef").remove(); + document.select("ConditionalText").remove(); + + String s = document.html().replaceAll("\\\\n", "\n"); + return Jsoup.clean(s, "", Safelist.none(), new Document.OutputSettings().prettyPrint(false)); + } + /** * @param html to be converted to plain text * @return Single line of plain text for the given HTML contents @@ -153,5 +180,9 @@ public static final String formatDateTimewithZoneIdAsUTC(String pattern, String LocalDateTime utcDateTime = LocalDateTime.ofInstant(zonedDateTime.toInstant(), ZoneOffset.UTC); return DateTimeFormatter.ofPattern(pattern).format(utcDateTime); } + + public static final String copyright() { + return String.format("Copyright (c) %s Open Text", Year.now().getValue()); + } } diff --git a/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/actions/zip/github-sast-report.yaml b/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/actions/zip/github-sast-report.yaml index 7a160b32e3..943d2d5447 100644 --- a/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/actions/zip/github-sast-report.yaml +++ b/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/actions/zip/github-sast-report.yaml @@ -1,14 +1,3 @@ -# For now, github-sast-report and sarif-report actions are exactly the same, apart from usage -# information and default output file name. The reason for having two similar but separate -# actions is two-fold: -# - We want to explicitly show that fcli supports both GitHub Code Scanning integration (which -# just happens to be based on SARIF) and generic SARIF capabilities. -# - Potentially, outputs for GitHub and generic SARIF may deviate in the future, for example if -# we want to add SARIF properties that are not supported by GitHub. -# Until the latter situation arises, we should make sure though that both actions stay in sync; -# when updating one, the other should also be updated. and ideally we should have functional tests -# that compare the outputs of both actions. - usage: header: Generate a GitHub Code Scanning report listing FoD SAST vulnerabilities. description: | @@ -57,8 +46,11 @@ steps: uri: /api/v3/releases/${parameters.release.releaseId}/vulnerabilities/${issue.vulnId}/traces do: - set: - - name: rules + - if: "ruleCache==null || ruleCache[issue.checkId]==null" + name: rules operation: append + - name: ruleCache + operation: merge - name: results operation: append - write: @@ -70,6 +62,9 @@ steps: Output written to ${parameters.file} valueTemplates: + - name: ruleCache + contents: "${{issue.checkId: true}}" + - name: github-sast-report contents: "$schema": https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json @@ -80,33 +75,49 @@ valueTemplates: name: 'Fortify on Demand' version: SCA ${staticScanSummary?.staticScanSummaryDetails?.engineVersion?:'version unknown'}; Rulepack ${staticScanSummary?.staticScanSummaryDetails?.rulePackVersion?:'version unknown'} rules: ${rules?:{}} + properties: + copyright: ${#copyright()} + applicationName: ${parameters.release.applicationName} + applicationId: ${parameters.release.applicationId} + releaseName: ${parameters.release.releaseName} + releaseId: ${parameters.release.releaseId} results: ${results?:{}} - name: rules contents: - id: ${issue.id+''} + id: ${issue.checkId} shortDescription: text: ${issue.category} fullDescription: - text: ${#htmlToText(issue.details?.summary)} + text: | + ${issue.category} + + ${#cleanRuleDescription(issue.details?.summary)} help: text: | - ${#htmlToText(issue.details?.explanation)} + Explanation + + ${#cleanRuleDescription(issue.details?.explanation)?:'-'} + + Recommendations + + ${#cleanRuleDescription(issue.recommendations?.recommendations)?:'-'} + + Tips + + ${#cleanRuleDescription(issue.recommendations?.tips)?:'-'} - ${#htmlToText(issue.recommendations?.recommendations)} + References + ${#cleanRuleDescription(issue.recommendations?.references)?:'-'} - For more information, see ${#fod.issueBrowserUrl(issue)} - properties: - tags: ${{issue.severityString}} - precision: ${(issue.severityString matches "(Critical|Medium)") ? "high":"low" } - security-severity: ${{Critical:10.0,High:8.9,Medium:6.9,Low:3.9}.get(issue.severityString)+''} + ${#copyright()} - name: results contents: - ruleId: ${issue.id+''} + ruleId: ${issue.checkId} message: - text: ${#htmlToText(issue.details?.summary)} + text: ${#htmlToText(issue.details?.summary)} [More information](${#fod.issueBrowserUrl(issue)}) level: ${(issue.severityString matches "(Critical|High)") ? "warning":"note" } partialFingerprints: issueInstanceId: ${issue.instanceId} diff --git a/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/actions/zip/sarif-sast-report.yaml b/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/actions/zip/sarif-sast-report.yaml index de20420e0a..2550edeaf2 100644 --- a/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/actions/zip/sarif-sast-report.yaml +++ b/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/actions/zip/sarif-sast-report.yaml @@ -1,17 +1,3 @@ -# For now, github-sast-report and sarif-report actions are exactly the same, apart from the -# following: -# - Different usage information -# - Different default output file name -# - The sarif-report doesn't impose a limit of 1000 issues -# The reason for having two similar but separate actions is two-fold: -# - We want to explicitly show that fcli supports both GitHub Code Scanning integration (which -# just happens to be based on SARIF) and generic SARIF capabilities. -# - Potentially, outputs for GitHub and generic SARIF may deviate in the future, for example if -# we want to add SARIF properties that are not supported by GitHub. -# Until the latter situation arises, we should make sure though that both actions stay in sync; -# when updating one, the other should also be updated. and ideally we should have functional tests -# that compare the outputs of both actions. - usage: header: Generate SARIF report listing SSC SAST vulnerabilities. description: | diff --git a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/_common/rest/transfer/SSCFileTransferHelper.java b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/_common/rest/transfer/SSCFileTransferHelper.java index 86d27797fb..4c04900c11 100644 --- a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/_common/rest/transfer/SSCFileTransferHelper.java +++ b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/_common/rest/transfer/SSCFileTransferHelper.java @@ -119,7 +119,7 @@ public static enum SSCFileTransferTokenType { REPORT_FILE } - private static final class SSCFileTransferTokenSupplier implements AutoCloseable, Supplier { + public static final class SSCFileTransferTokenSupplier implements AutoCloseable, Supplier { private final UnirestInstance unirest; private final String token; diff --git a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/action/cli/cmd/SSCActionRunCommand.java b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/action/cli/cmd/SSCActionRunCommand.java index 54a78808b1..70310cccd7 100644 --- a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/action/cli/cmd/SSCActionRunCommand.java +++ b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/action/cli/cmd/SSCActionRunCommand.java @@ -12,31 +12,49 @@ *******************************************************************************/ package com.fortify.cli.ssc.action.cli.cmd; +import java.io.InputStream; import java.util.List; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; import org.springframework.expression.spel.support.SimpleEvaluationContext; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.formkiq.graalvm.annotations.Reflectable; import com.fortify.cli.common.action.cli.cmd.AbstractActionRunCommand; +import com.fortify.cli.common.action.helper.ActionDescriptor.ActionStepForEachDescriptor.IActionStepForEachProcessor; import com.fortify.cli.common.action.helper.ActionDescriptor.ActionValidationException; import com.fortify.cli.common.action.helper.ActionRunner; import com.fortify.cli.common.action.helper.ActionRunner.IActionRequestHelper.BasicActionRequestHelper; import com.fortify.cli.common.action.helper.ActionRunner.ParameterTypeConverterArgs; +import com.fortify.cli.common.json.JsonHelper; import com.fortify.cli.common.output.product.IProductHelper; import com.fortify.cli.common.rest.unirest.IUnirestInstanceSupplier; import com.fortify.cli.common.spring.expression.SpelHelper; import com.fortify.cli.common.util.StringUtils; +import com.fortify.cli.ssc._common.rest.SSCUrls; import com.fortify.cli.ssc._common.rest.bulk.SSCBulkRequestBuilder; import com.fortify.cli.ssc._common.rest.helper.SSCProductHelper; +import com.fortify.cli.ssc._common.rest.transfer.SSCFileTransferHelper.SSCFileTransferTokenSupplier; +import com.fortify.cli.ssc._common.rest.transfer.SSCFileTransferHelper.SSCFileTransferTokenType; import com.fortify.cli.ssc._common.session.cli.mixin.SSCUnirestInstanceSupplierMixin; import com.fortify.cli.ssc.appversion.helper.SSCAppVersionHelper; import com.fortify.cli.ssc.issue.helper.SSCIssueFilterSetHelper; import kong.unirest.HttpRequest; +import kong.unirest.RawResponse; +import kong.unirest.UnirestInstance; import lombok.Getter; import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; import picocli.CommandLine.Command; import picocli.CommandLine.Mixin; @@ -54,13 +72,17 @@ protected void configure(ActionRunner templateRunner, SimpleEvaluationContext co templateRunner .addParameterConverter("appversion_single", this::loadAppVersion) .addParameterConverter("filterset", this::loadFilterSet) - .addRequestHelper("ssc", new SSCDataExtractRequestHelper(unirestInstanceSupplier::getUnirestInstance, SSCProductHelper.INSTANCE)); + .addRequestHelper("ssc", new SSCActionRequestHelper(unirestInstanceSupplier::getUnirestInstance, SSCProductHelper.INSTANCE)); context.setVariable("ssc", new SSCSpelFunctions(templateRunner)); } @RequiredArgsConstructor @Reflectable public final class SSCSpelFunctions { private final ActionRunner templateRunner; + public IActionStepForEachProcessor ruleDescriptionsProcessor(String appVersionId) { + var unirest = templateRunner.getRequestHelper("ssc").getUnirestInstance(); + return new SSCFPRRuleDescriptionProcessor(unirest, appVersionId)::process; + } public String issueBrowserUrl(ObjectNode issue, ObjectNode filterset) { var deepLinkExpression = baseUrl() +"/html/ssc/version/${projectVersionId}/fix/${id}/?engineType=${engineType}&issue=${issueInstanceId}"; @@ -110,8 +132,8 @@ private final JsonNode loadFilterSet(String titleOrId, ParameterTypeConverterArg return filterSetDescriptor.asJsonNode(); } - private static final class SSCDataExtractRequestHelper extends BasicActionRequestHelper { - public SSCDataExtractRequestHelper(IUnirestInstanceSupplier unirestInstanceSupplier, IProductHelper productHelper) { + private static final class SSCActionRequestHelper extends BasicActionRequestHelper { + public SSCActionRequestHelper(IUnirestInstanceSupplier unirestInstanceSupplier, IProductHelper productHelper) { super(unirestInstanceSupplier, productHelper); } @@ -128,10 +150,131 @@ public void executeSimpleRequests(List requestDescripto } private HttpRequest createRequest(ActionRequestDescriptor requestDescriptor) { - var request = getUnirestInstance(). request(requestDescriptor.getMethod(), requestDescriptor.getUri()) + var request = getUnirestInstance().request(requestDescriptor.getMethod(), requestDescriptor.getUri()) .queryString(requestDescriptor.getQueryParams()); var body = requestDescriptor.getBody(); return body==null ? request : request.body(body); } } + + @RequiredArgsConstructor + private final class SSCFPRRuleDescriptionProcessor { + private final UnirestInstance unirest; + private final String appVersionId; + + @SneakyThrows + public final void process(Function consumer) { + try ( SSCFileTransferTokenSupplier tokenSupplier = new SSCFileTransferTokenSupplier(unirest, SSCFileTransferTokenType.DOWNLOAD); ) { + unirest.get(SSCUrls.DOWNLOAD_CURRENT_FPR(appVersionId, false)) + .routeParam("downloadToken", tokenSupplier.get()).asObject(r->processFpr(r, consumer)).getBody(); + } + } + + @SneakyThrows + private final String processFpr(RawResponse r, Function consumer) { + try ( var zis = new ZipInputStream(r.getContent()) ) { + ZipEntry entry; + while ( (entry = zis.getNextEntry())!=null ) { + if ( "audit.fvdl".equals(entry.getName()) ) { + processAuditFvdl(zis, consumer); break; + } + } + } + return null; + } + + private final void processAuditFvdl(InputStream is, Function consumer) throws XMLStreamException { + var factory = XMLInputFactory.newInstance(); + var reader = factory.createXMLStreamReader(is); + while(reader.hasNext()) { + int eventType = reader.next(); + if ( eventType==XMLStreamReader.START_ELEMENT && "Description".equals(reader.getLocalName()) ) { + if (!processDescription(reader, consumer)) { break; } + } + } + } + + private boolean processDescription(XMLStreamReader reader, Function consumer) throws XMLStreamException { + var ruleId = reader.getAttributeValue(null, "classID"); + var entry = JsonHelper.getObjectMapper().createObjectNode() + .put("id", ruleId); + var tips = JsonHelper.getObjectMapper().createArrayNode(); + var references = JsonHelper.getObjectMapper().createArrayNode(); + entry.set("tips", tips); + entry.set("references", references); + processElement(reader, name->{ + switch ( name ) { + case "Abstract": entry.put("abstract", readString(reader)); break; + case "Explanation": entry.put("explanation", readString(reader)); break; + case "Recommendations": entry.put("recommendations", readString(reader)); break; + case "Tips": addTips(reader, tips); break; + case "References": addReferences(reader, references); break; + } + }); + return consumer.apply(entry); + } + + @SneakyThrows + private void addTips(XMLStreamReader reader, ArrayNode tips) { + processElement(reader, name->{ + switch ( name ) { + case "Tip": tips.add(readString(reader)); + } + }); + } + + @SneakyThrows + private void addReferences(XMLStreamReader reader, ArrayNode references) { + processElement(reader, name->{ + switch ( name ) { + case "Reference": references.add(readReference(reader)); + } + }); + } + + @SneakyThrows + private ObjectNode readReference(XMLStreamReader reader) { + var reference = JsonHelper.getObjectMapper().createObjectNode(); + processElement(reader, name->{ + switch ( name ) { + case "Title": reference.put("title", readString(reader)); break; + case "Publisher": reference.put("publisher", readString(reader)); break; + case "Author": reference.put("author", readString(reader)); break; + case "Source": reference.put("source", readString(reader)); break; + } + }); + return reference; + } + + private void processElement(XMLStreamReader reader, Consumer consumer) throws XMLStreamException { + int level = 1; + while(level > 0 && reader.hasNext()) { + int eventType = reader.next(); + switch ( eventType ) { + case XMLStreamReader.START_ELEMENT: + level++; + consumer.accept(reader.getLocalName()); + case XMLStreamReader.END_ELEMENT: + level--; break; + } + } + } + + @SneakyThrows + private String readString(XMLStreamReader reader) { + StringBuilder result = new StringBuilder(); + while (reader.hasNext()) { + int eventType = reader.next(); + switch (eventType) { + case XMLStreamReader.CHARACTERS: + case XMLStreamReader.CDATA: + result.append(reader.getText()); + break; + case XMLStreamReader.END_ELEMENT: + return result.toString(); + } + } + throw new XMLStreamException("Premature end of file"); + } + } } diff --git a/fcli-core/fcli-ssc/src/main/resources/com/fortify/cli/ssc/actions/zip/github-sast-report.yaml b/fcli-core/fcli-ssc/src/main/resources/com/fortify/cli/ssc/actions/zip/github-sast-report.yaml index 440b356ed1..3b29708559 100644 --- a/fcli-core/fcli-ssc/src/main/resources/com/fortify/cli/ssc/actions/zip/github-sast-report.yaml +++ b/fcli-core/fcli-ssc/src/main/resources/com/fortify/cli/ssc/actions/zip/github-sast-report.yaml @@ -1,14 +1,3 @@ -# For now, github-sast-report and sarif-report actions are exactly the same, apart from usage -# information and default Optional output file name. The reason for having two similar but separate -# actions is two-fold: -# - We want to explicitly show that fcli supports both GitHub Code Scanning integration (which -# just happens to be based on SARIF) and generic SARIF capabilities. -# - Potentially, outputs for GitHub and generic SARIF may deviate in the future, for example if -# we want to add SARIF properties that are not supported by GitHub. -# Until the latter situation arises, we should make sure though that both actions stay in sync; -# when updating one, the other should also be updated. and ideally we should have functional tests -# that compare the outputs of both actions. - usage: header: Generate a GitHub Code Scanning report listing SSC SAST vulnerabilities. description: | @@ -68,10 +57,19 @@ steps: uri: /api/v1/issueDetails/${issue.id} do: - set: - - name: rules - operation: append + - name: ruleCategories + operation: merge - name: results operation: append + - progress: Processing rule data + - forEach: + processor: "#ssc.ruleDescriptionsProcessor(parameters.appversion.id)" + name: rule + do: + - set: + - if: "#isNotBlank(ruleCategories[rule.id])" + name: rules + operation: append - write: - to: ${parameters.file} valueTemplate: github-sast-report @@ -81,6 +79,9 @@ steps: Output written to ${parameters.file} valueTemplates: + - name: ruleCategories + contents: "${{issue.primaryRuleGuid: issue.issueName}}" + - name: github-sast-report contents: "$schema": https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json @@ -91,33 +92,52 @@ valueTemplates: name: 'Fortify SCA' version: ${lastStaticScan?.engineVersion?:'unknown'} rules: ${rules?:{}} + properties: + copyright: ${#copyright()} + applicationName: ${parameters.appversion.project.name} + applicationId: ${parameters.appversion.project.id} + versionName: ${parameters.appversion.name} + versionId: ${parameters.appversion.id} results: ${results?:{}} - name: rules contents: - id: ${issue.id+''} + id: ${rule.id} shortDescription: - text: ${issue.issueName} + text: ${ruleCategories[rule.id]} fullDescription: - text: ${issue.details?.brief} + text: | + ${ruleCategories[rule.id]} + + ${rule.abstract} help: text: | - ${#htmlToText(issue.details?.detail)} - - ${#htmlToText(issue.details?.recommendation)} - + Explanation + + ${#htmlToText(rule.explanation)?:'-'} + + Recommendations + + ${#htmlToText(rule.recommendations)?:'-'} + + Tips + + ${#join('\n\n', rule.tips)?:'-'} + + References + + ${#join('\n\n', rule.references.![title + +(#isNotBlank(publisher)?", "+publisher:"") + +(#isNotBlank(author)?", "+author:"") + +(#isNotBlank(source)?", "+source:"")])?:'-'} - For more information, see ${#ssc.issueBrowserUrl(issue,parameters.filterset)} - properties: - tags: ${{issue.friority}} - precision: ${(issue.friority matches "(Critical|Medium)") ? "high":"low" } - security-severity: ${{Critical:10.0,High:8.9,Medium:6.9,Low:3.9}.get(issue.friority)+''} + ${#copyright()} - name: results contents: - ruleId: ${issue.id+''} + ruleId: ${issue.primaryRuleGuid} message: - text: ${issue.details?.brief} + text: ${issue.details?.brief} [More information](${#ssc.issueBrowserUrl(issue,parameters.filterset)}) level: ${(issue.friority matches "(Critical|High)") ? "warning":"note" } partialFingerprints: issueInstanceId: ${issue.issueInstanceId} @@ -126,8 +146,8 @@ valueTemplates: artifactLocation: uri: ${issue.fullFileName} region: - startLine: ${issue.lineNumber==0?1:issue.lineNumber} - endLine: ${issue.lineNumber==0?1:issue.lineNumber} + startLine: ${issue.lineNumber==0||issue.lineNumber==null?1:issue.lineNumber} + endLine: ${issue.lineNumber==0||issue.lineNumber==null?1:issue.lineNumber} startColumn: ${1} # Needs to be specified as an expression in order to end up as integer instead of string in JSON endColumn: ${80} codeFlows: |- @@ -146,7 +166,7 @@ valueTemplates: uri: fullPath }, region: { - startLine: line==0?1:line + startLine: line==0||line==null?1:line } } } diff --git a/fcli-core/fcli-ssc/src/main/resources/com/fortify/cli/ssc/actions/zip/sarif-sast-report.yaml b/fcli-core/fcli-ssc/src/main/resources/com/fortify/cli/ssc/actions/zip/sarif-sast-report.yaml index 19a957be88..fedffa8757 100644 --- a/fcli-core/fcli-ssc/src/main/resources/com/fortify/cli/ssc/actions/zip/sarif-sast-report.yaml +++ b/fcli-core/fcli-ssc/src/main/resources/com/fortify/cli/ssc/actions/zip/sarif-sast-report.yaml @@ -1,17 +1,3 @@ -# For now, github-sast-report and sarif-report actions are exactly the same, apart from the -# following: -# - Different usage information -# - Different default Optional output file name -# - The sarif-report doesn't impose a limit of 1000 issues -# The reason for having two similar but separate actions is two-fold: -# - We want to explicitly show that fcli supports both GitHub Code Scanning integration (which -# just happens to be based on SARIF) and generic SARIF capabilities. -# - Potentially, outputs for GitHub and generic SARIF may deviate in the future, for example if -# we want to add SARIF properties that are not supported by GitHub. -# Until the latter situation arises, we should make sure though that both actions stay in sync; -# when updating one, the other should also be updated. and ideally we should have functional tests -# that compare the outputs of both actions. - usage: header: Generate SARIF report listing SSC SAST vulnerabilities. description: | diff --git a/fcli-other/fcli-bom/build.gradle b/fcli-other/fcli-bom/build.gradle index 637babf84c..1ecd2d114e 100644 --- a/fcli-other/fcli-bom/build.gradle +++ b/fcli-other/fcli-bom/build.gradle @@ -19,7 +19,7 @@ dependencies { // Annotation-based reflect-config.json generation api('com.formkiq:graalvm-annotations:1.2.0') - api('com.formkiq:graalvm-annotations-processor:1.4.1') + api('com.formkiq:graalvm-annotations-processor:1.4.2') // ANSI support api("org.fusesource.jansi:jansi:2.4.1")