diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/cli/mixin/CommonOptionMixins.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/cli/mixin/CommonOptionMixins.java index 2488882254..4c9bd1c62f 100644 --- a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/cli/mixin/CommonOptionMixins.java +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/cli/mixin/CommonOptionMixins.java @@ -13,13 +13,22 @@ package com.fortify.cli.common.cli.mixin; import java.io.File; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Stack; import com.fortify.cli.common.util.PicocliSpecHelper; import com.fortify.cli.common.util.StringUtils; import lombok.Getter; +import picocli.CommandLine.Command; +import picocli.CommandLine.IParameterPreprocessor; import picocli.CommandLine.Mixin; +import picocli.CommandLine.Model.ArgSpec; import picocli.CommandLine.Model.CommandSpec; +import picocli.CommandLine.Model.UnmatchedArgsBinding; import picocli.CommandLine.Option; import picocli.CommandLine.ParameterException; @@ -78,4 +87,71 @@ private String getPlainPrompt(CommandSpec spec, Object... promptArgs) { return prompt; } } + + /** + * This mixin allows for collecting any unmatched command line options + * and parsing them into an [option name]=[value] map. + */ + @Command(preprocessor = OptionParametersMixinPreprocessor.class) + public static final class OptionParametersMixin { + @Getter private Map options = new LinkedHashMap<>(); + + public final T setUnmatchedArgs(T arg) { + parse((String[])arg); + return arg; + } + + private final void parse(String[] argsArray) { + var args = asDeque(argsArray); + while ( !args.isEmpty() ) { + var opt = args.pop(); + if ( !opt.startsWith("-") ) { + throw new IllegalArgumentException("Unknown command line option: "+opt); + } else { + opt = opt.replaceFirst("-+", ""); + if ( args.peek().startsWith("-") ) { + options.put(opt, ""); + } else { + options.put(opt, args.pop()); + } + } + } + } + + private final Deque asDeque(String[] args) { + Deque result = new ArrayDeque<>(); + // Split --opt=value into separate args on the Deque + for ( var arg: args ) { + if ( arg.startsWith("--") ) { + // Allow for --opt=val, adding option and value as separate args + var elts = arg.split("=", 2); + result.add(elts[0]); + if ( elts.length==2 ) { result.add(elts[1]); } + } else if ( arg.startsWith("-") ) { + result.add(arg.substring(0,2)); + if ( arg.length()>2 ) { + var val = arg.substring(2); + if ( val.startsWith("=") ) { val = val.substring(1); } + result.add(val); + } + } else { + result.add(arg); + } + } + return result; + } + } + + private static final class OptionParametersMixinPreprocessor implements IParameterPreprocessor { + @Override + public final boolean preprocess(Stack args, CommandSpec commandSpec, ArgSpec argSpec, Map info) { + var mixin = commandSpec.mixins().values().stream() + .map(CommandSpec::userObject) + .filter(o->o instanceof OptionParametersMixin) + .map(OptionParametersMixin.class::cast) + .findFirst().get(); + commandSpec.addUnmatchedArgsBinding(UnmatchedArgsBinding.forStringArrayConsumer(mixin::setUnmatchedArgs)); + return false; + } + } } diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/cli/cmd/AbstractDataExtractCreateCommand.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/cli/cmd/AbstractDataExtractCreateCommand.java new file mode 100644 index 0000000000..32d5b0a267 --- /dev/null +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/cli/cmd/AbstractDataExtractCreateCommand.java @@ -0,0 +1,55 @@ +/******************************************************************************* + * Copyright 2021, 2023 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + *******************************************************************************/ +package com.fortify.cli.common.data_extract.cli.cmd; + +import java.util.List; + +import com.fortify.cli.common.cli.cmd.AbstractRunnableCommand; +import com.fortify.cli.common.cli.mixin.CommonOptionMixins.OptionParametersMixin; +import com.fortify.cli.common.data_extract.helper.DataExtractTemplateExecutor; +import com.fortify.cli.common.data_extract.helper.DataExtractTemplateHelper; +import com.fortify.cli.common.progress.cli.mixin.ProgressWriterFactoryMixin; +import com.fortify.cli.common.util.DisableTest; +import com.fortify.cli.common.util.DisableTest.TestType; + +import picocli.CommandLine.Mixin; +import picocli.CommandLine.Option; + +public abstract class AbstractDataExtractCreateCommand extends AbstractRunnableCommand implements Runnable { + @Option(names={"-t", "--template"}, required=true) private String template; + @DisableTest({TestType.MULTI_OPT_SPLIT, TestType.MULTI_OPT_PLURAL_NAME, TestType.OPT_LONG_NAME}) + @Option(names="--", paramLabel="", descriptionKey="fcli.data-extract.create.template-parameter") + private List dummyForSynopsis; + @Mixin private OptionParametersMixin templateParameters; + @Mixin private ProgressWriterFactoryMixin progressWriterFactory; + + @Override + public final void run() { + initMixins(); + try ( var progressWriter = progressWriterFactory.create() ) { + progressWriter.writeProgress("Loading template %s", template); + var templateDescriptor = DataExtractTemplateHelper.load(getType(), template); + progressWriter.writeProgress("Executing template %s", template); + var templateExecutor = DataExtractTemplateExecutor.builder() + .template(templateDescriptor) + .inputParameters(templateParameters.getOptions()) + .progressWriter(progressWriter) + .build(); + configure(templateExecutor); + templateExecutor.execute(); + } + } + + protected abstract String getType(); + protected abstract void configure(DataExtractTemplateExecutor templateExecutor); +} diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/cli/cmd/AbstractDataExtractListTemplatesCommand.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/cli/cmd/AbstractDataExtractListTemplatesCommand.java new file mode 100644 index 0000000000..0cdb7e776e --- /dev/null +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/cli/cmd/AbstractDataExtractListTemplatesCommand.java @@ -0,0 +1,32 @@ +/******************************************************************************* + * Copyright 2021, 2023 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + *******************************************************************************/ +package com.fortify.cli.common.data_extract.cli.cmd; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fortify.cli.common.output.cli.cmd.AbstractOutputCommand; +import com.fortify.cli.common.output.cli.cmd.IJsonNodeSupplier; + +public abstract class AbstractDataExtractListTemplatesCommand extends AbstractOutputCommand implements IJsonNodeSupplier { + @Override + public final JsonNode getJsonNode() { + // TODO Auto-generated method stub + return null; + } + @Override + public final boolean isSingular() { + return false; + } + protected abstract String getType(); + + +} diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/helper/DataExtractRestHelper.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/helper/DataExtractRestHelper.java new file mode 100644 index 0000000000..75e57f6eda --- /dev/null +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/helper/DataExtractRestHelper.java @@ -0,0 +1,55 @@ +/** + * Copyright 2023 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors (“Open Text”) are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.common.data_extract.helper; + +import com.fortify.cli.common.output.product.IProductHelper; +import com.fortify.cli.common.output.transform.IInputTransformer; +import com.fortify.cli.common.output.transform.IRecordTransformer; +import com.fortify.cli.common.rest.paging.INextPageUrlProducer; +import com.fortify.cli.common.rest.paging.INextPageUrlProducerSupplier; +import com.fortify.cli.common.rest.unirest.IUnirestInstanceSupplier; +import com.fortify.cli.common.util.JavaHelper; + +import kong.unirest.UnirestInstance; +import lombok.Data; + +@Data +public final class DataExtractRestHelper implements AutoCloseable { + private final String name; + private final IUnirestInstanceSupplier unirestInstanceSupplier; + private final IProductHelper productHelper; + private UnirestInstance unirestInstance; + public final UnirestInstance getUnirestInstance() { + if ( unirestInstance==null ) { + unirestInstance = unirestInstanceSupplier.getUnirestInstance(); + } + return unirestInstance; + } + public final IInputTransformer getInputTransformer() { + return JavaHelper.as(productHelper, IInputTransformer.class).orElse(i->i); + } + public final IRecordTransformer getRecordTransformer() { + return JavaHelper.as(productHelper, IRecordTransformer.class).orElse(r->r); + } + public final INextPageUrlProducer getNextPageUrlProducer() { + return JavaHelper.as(productHelper, INextPageUrlProducerSupplier.class) + .map(s->s.getNextPageUrlProducer()) + .orElse(null); + } + @Override + public void close() throws Exception { + if ( unirestInstance!=null ) { + unirestInstance.close(); + } + } +} \ No newline at end of file diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/helper/DataExtractTemplateDescriptor.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/helper/DataExtractTemplateDescriptor.java new file mode 100644 index 0000000000..8aa205d44c --- /dev/null +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/helper/DataExtractTemplateDescriptor.java @@ -0,0 +1,319 @@ +/** + * Copyright 2023 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors (“Open Text”) are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.common.data_extract.helper; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +import org.springframework.expression.ParseException; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.TextNode; +import com.fasterxml.jackson.databind.node.ValueNode; +import com.formkiq.graalvm.annotations.Reflectable; +import com.fortify.cli.common.json.JsonHelper.AbstractJsonNodeWalker; +import com.fortify.cli.common.spring.expression.SpelHelper; +import com.fortify.cli.common.spring.expression.wrapper.SimpleExpression; +import com.fortify.cli.common.spring.expression.wrapper.TemplateExpression; +import com.fortify.cli.common.util.StringUtils; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.ToString; + +/** + * This class describes a data extraction template deserialized from a + * data extraction template YAML file, containing elements describing: + *
    + *
  • Template metadata like name and description
    • + *
  • Data to be processed like template parameters and + * responses from REST calls
    • + *
  • Template output
    • + *
+ * + * @author Ruud Senden + */ +@Reflectable @NoArgsConstructor +@Data +public class DataExtractTemplateDescriptor { + /** Template name, set in {@link #postLoad(String)} method */ + private String name; + /** Template description */ + private String description; + /** Template data sources, evaluated in the order as defined in the YAML file */ + private List sources; + private Map> partialOutputs; + private List outputs; + + /** + * This method is invoked by DataExtractTemplateHelper after deserializing + * an instance of this class from a YAML file. It performs some additional + * initialization and validation. + */ + final void postLoad(String name) { + this.name = name; + checkNotNull("template sources", sources, this); + checkNotNull("template outputs", outputs, this); + sources.forEach(d->d.postLoad(null)); + if ( partialOutputs!=null ) { + partialOutputs.entrySet().forEach(e->e.getValue().forEach(DataExtractTemplateOutputDescriptor::postLoad)); + } + outputs.forEach(DataExtractTemplateOutputDescriptor::postLoad); + } + + /** + * Get the list of all (top-level) {@link DataExtractTemplateSourceParameterDescriptor} + * instances, describing all template parameters. + */ + public final List getParameterDescriptors() { + return sources.stream() + .map(DataExtractTemplateSourceDescriptor::getParameters) + .filter(Objects::nonNull) + .flatMap(l->l.stream()) + .collect(Collectors.toList()); + } + + /** + * Utility method for checking whether the given value is not blank, throwing an + * exception otherwise. + * @param property Descriptive name of the YAML property being checked + * @param value Value to be checked for not being blank + * @param entity The object containing the property to be checked + */ + private static final void checkNotBlank(String property, String value, Object entity) { + if ( StringUtils.isBlank(value) ) { + throw new TemplateValidationException(String.format("Template %s property must be specified (entity: %s)", property, entity.toString())); + } + } + + /** + * Utility method for checking whether the given value is not null, throwing an + * exception otherwise. + * @param property Descriptive name of the YAML property being checked + * @param value Value to be checked for not being null + * @param entity The object containing the property to be checked + */ + private static final void checkNotNull(String property, Object value, Object entity) { + if ( value==null ) { + throw new TemplateValidationException(String.format("Template %s property must be specified (entity: %s)", property, entity.toString())); + } + } + + /** + * Interface to be implemented by all template data elements. + */ + public static interface IDataExtractTemplateSource { + public IDataExtractTemplateSource getParent(); + } + + /** + * This class describes a single source element, which may contain zero or more + * parameters or requests. This class is used for both top-level source elements, + * and source elements in forEach elements. Parameters may only be contained in + * top-level source elements, not within forEach elements. Potentially, later + * versions may add support for other sources, like static values or arrays, + * allowing for example iteration over a static array. + * + * @author Ruud Senden + */ + @Reflectable @NoArgsConstructor + @Data + public static final class DataExtractTemplateSourceDescriptor implements IDataExtractTemplateSource { + /** The parent for this source element, or null for top-level element */ + @ToString.Exclude @EqualsAndHashCode.Exclude private IDataExtractTemplateSource parent; + /** Optional parameters for this source element */ + private List parameters; + /** Optional requests for this source element */ + private List requests; + + /** + * This method is invoked by the parent element (which may either be another + * source element, or the top-level {@link DataExtractTemplateDescriptor} instance). + * It stores the given parent, then invokes the postLoad() method on each + * parameter and request. + * @param parent of this element, or null if the parent is the top-level + * {@link DataExtractTemplateDescriptor} + */ + public final void postLoad(IDataExtractTemplateSource parent) { + this.parent = parent; + if ( parameters!=null ) { parameters.forEach(d->d.postLoad(this)); } + if ( requests!=null ) { requests.forEach(d->d.postLoad(this)); } + } + } + + /** + * Abstract base class for all source elements like individual parameter or request descriptors. + * This includes the parent source element (set through {@link #postLoad(IDataExtractTemplateSource)}), + * a required source name, an optional description, and optional forEach block. + */ + @Reflectable @NoArgsConstructor + @Data + public static abstract class AbstractDataExtractTemplateSourceDescriptor implements IDataExtractTemplateSource { + /** The parent for this source element, never null as source elements are always contained in a {@link DataExtractTemplateSourceDescriptor} instance */ + @ToString.Exclude @EqualsAndHashCode.Exclude private IDataExtractTemplateSource parent; + /** Required name for this source element */ + private String name; + /** Optional description for this source element */ + private String description; + /** Optional forEach block, listing sources to be repeated for each entry produced by this data element */ + private DataExtractTemplateSourceForEachDescriptor forEach; + + /** + * This method is invoked by the parent {@link DataExtractTemplateSourceDescriptor} + * instance. It stores the given parent, then invokes the postLoad() method on each + * forEach element, and finally calls the _postLoad() method provided by subclasses. + * @param parent {@link DataExtractTemplateSourceDescriptor} instance + */ + public final void postLoad(IDataExtractTemplateSource parent) { + this.parent = parent; + if ( forEach!=null ) { forEach.postLoad(this); } + _postLoad(); + } + + /** + * Abstract method that allows subclasses to perform post-load processing + */ + protected abstract void _postLoad(); + } + + /** + * This class describes a forEach element, allowing iteration over the output of + * the parent element, like the response of a REST request or the contents of a + * template parameter. + */ + @Reflectable @NoArgsConstructor + @Data @EqualsAndHashCode(callSuper = true) @ToString(callSuper = true) + public static final class DataExtractTemplateSourceForEachDescriptor extends AbstractDataExtractTemplateSourceDescriptor { + /** Optional (sub-)property/expression from which to retrieve the values to be iterated through */ + private SimpleExpression property; + /** Sources to be repeated for each value */ + private List sources; + + /** + * This method is invoked by {@link AbstractDataExtractTemplateSourceDescriptor#postLoad(IDataExtractTemplateSource)} + * method. It checks that required properties are set, then calls the postLoad() method for + * each sub-source. + */ + protected final void _postLoad() { + checkNotBlank("forEach name", getName(), this); + checkNotNull("forEach sources", sources, this); + sources.forEach(d->d.postLoad(this)); + } + } + + /** + * This class describes a template parameter. + */ + @Reflectable @NoArgsConstructor + @Data @EqualsAndHashCode(callSuper = true) @ToString(callSuper = true) + public static final class DataExtractTemplateSourceParameterDescriptor extends AbstractDataExtractTemplateSourceDescriptor { + /** Optional parameter type */ + private String type; + /** Optional type parameters*/ + private Map typeParameters; + /** Optional template expression defining the default parameter value if not provided by user */ + private TemplateExpression defaultValue; + /** Boolean indicating whether this parameter is required */ + private boolean required = true; + + /** + * This method is invoked by {@link AbstractDataExtractTemplateSourceDescriptor#postLoad(IDataExtractTemplateSource)} + * method. It checks that required properties are set, and that this parameter is defined as + * a top-level source. + */ + protected final void _postLoad() { + checkNotBlank("parameter name", getName(), this); + var grandParent = getParent().getParent(); + if ( grandParent!=null ) { + throw new TemplateValidationException("Template parameters must be defined in a top-level data element"); + } + } + } + + /** + * This class describes a REST request. + */ + @Reflectable @NoArgsConstructor + @Data @EqualsAndHashCode(callSuper = true) @ToString(callSuper = true) + public static final class DataExtractTemplateSourceRequestDescriptor extends AbstractDataExtractTemplateSourceDescriptor { + /** Required template expression defining the request URI from which to get the data */ + private TemplateExpression uri; + /** Define from which remote system the data should be retrieved. Required if multiple + * remote systems are available, optional if there's only a single remote system. */ + private String from; + /** Map defining (non-encoded) request query parameters; parameter values are defined as template expressions */ + private Map query; + + /** + * This method is invoked by {@link AbstractDataExtractTemplateSourceDescriptor#postLoad(IDataExtractTemplateSource)} + * method. It checks that required properties are set. + */ + protected final void _postLoad() { + checkNotBlank("request name", getName(), this); + checkNotNull("request uri", uri, this); + } + } + + /** + * This class describes a partial output. + */ + @Reflectable @NoArgsConstructor + @Data + public static final class DataExtractTemplateOutputDescriptor { + private String name; + private JsonNode contents; + private final Map valueExpressions = new LinkedHashMap<>(); + + public final void postLoad() { + checkNotBlank("partialOutput name", name, this); + checkNotNull("partialOutput contents", contents, this); + new ContentsWalker().walk(contents); + } + + private final class ContentsWalker extends AbstractJsonNodeWalker { + @Override + protected Void getResult() { return null; } + @Override + protected void walkValue(Void state, String path, JsonNode parent, ValueNode node) { + if ( node instanceof TextNode ) { + var expr = node.asText(); + try { + valueExpressions.put(path, SpelHelper.parseTemplateExpression(expr)); + } catch (ParseException e) { + throw new TemplateValidationException(String.format("Error parsing template expression '%s' (%s)", expr, DataExtractTemplateOutputDescriptor.this) ,e); + } + } + super.walkValue(state, path, parent, node); + } + } + } + + /** + * Exception class used for template validation errors. + */ + public static final class TemplateValidationException extends IllegalStateException { + private static final long serialVersionUID = 1L; + + public TemplateValidationException(String message, Throwable cause) { + super(message, cause); + } + + public TemplateValidationException(String s) { + super(s); + } + } +} diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/helper/DataExtractTemplateExecutor.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/helper/DataExtractTemplateExecutor.java new file mode 100644 index 0000000000..68d2c2a3a4 --- /dev/null +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/helper/DataExtractTemplateExecutor.java @@ -0,0 +1,317 @@ +/** + * Copyright 2023 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors (“Open Text”) are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.common.data_extract.helper; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.BiFunction; +import java.util.function.Function; + +import org.springframework.expression.spel.SpelEvaluationException; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.BooleanNode; +import com.fasterxml.jackson.databind.node.DoubleNode; +import com.fasterxml.jackson.databind.node.FloatNode; +import com.fasterxml.jackson.databind.node.IntNode; +import com.fasterxml.jackson.databind.node.LongNode; +import com.fasterxml.jackson.databind.node.NullNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +import com.fasterxml.jackson.databind.node.ValueNode; +import com.fortify.cli.common.data_extract.helper.DataExtractTemplateDescriptor.DataExtractTemplateOutputDescriptor; +import com.fortify.cli.common.data_extract.helper.DataExtractTemplateDescriptor.DataExtractTemplateSourceDescriptor; +import com.fortify.cli.common.data_extract.helper.DataExtractTemplateDescriptor.DataExtractTemplateSourceForEachDescriptor; +import com.fortify.cli.common.data_extract.helper.DataExtractTemplateDescriptor.DataExtractTemplateSourceParameterDescriptor; +import com.fortify.cli.common.data_extract.helper.DataExtractTemplateDescriptor.DataExtractTemplateSourceRequestDescriptor; +import com.fortify.cli.common.data_extract.helper.DataExtractTemplateDescriptor.TemplateValidationException; +import com.fortify.cli.common.json.JsonHelper; +import com.fortify.cli.common.json.JsonHelper.JsonNodeDeepCopyWalker; +import com.fortify.cli.common.output.product.IProductHelper; +import com.fortify.cli.common.progress.helper.IProgressWriterI18n; +import com.fortify.cli.common.rest.paging.PagingHelper; +import com.fortify.cli.common.rest.unirest.IUnirestInstanceSupplier; +import com.fortify.cli.common.spring.expression.SpelEvaluator; +import com.fortify.cli.common.spring.expression.SpelHelper; +import com.fortify.cli.common.spring.expression.wrapper.TemplateExpression; +import com.fortify.cli.common.util.StringUtils; + +import kong.unirest.HttpResponse; +import lombok.Builder; +import lombok.Data; +import lombok.RequiredArgsConstructor; + +@Builder +public class DataExtractTemplateExecutor { + private static final ObjectMapper objectMapper = JsonHelper.getObjectMapper(); + private final IProgressWriterI18n progressWriter; + private final DataExtractTemplateDescriptor template; + private final Map inputParameters; + private final ObjectNode data = objectMapper.createObjectNode(); + private final ObjectNode partialOutputs = objectMapper.createObjectNode(); + private final ObjectNode outputs = objectMapper.createObjectNode(); + private final Map> parameterConverters = createDefaultParameterConverters(); + // TODO Instead of a Map of REST helpers, we should have a Map of requests executors, taking a + // list of DataExtractTemplateSourceRequestDescriptors, with a default implementation that + // executes each request one-by one (like the current implementation below) but allowing + // alternative implementations that execute requests in parallel or in bulk (using SSC bulk + // requests for example). The requests executor would handle paging and input/record + // transformations, and pass the raw and transformed response back to this class through a + // lambda function. + private final Map restHelpers = new HashMap<>(); + + public final void execute() { + data.set("partialOutputs", partialOutputs); + processParameters(); + processSources(); + processOutputs(); + outputs.fields().forEachRemaining(e->writeOutput(e.getKey(), e.getValue())); + } + + private final void writeOutput(String name, JsonNode output) { + System.out.println(String.format("\n\n===================== %s ============================", name)); + if ( output instanceof ValueNode ) { + System.out.println(output.asText()); + } else { + System.out.println(output.toPrettyString()); + } + } + + public final DataExtractTemplateExecutor addParameterConverter(String type, BiFunction converter) { + parameterConverters.put(type, converter); + return this; + } + public final DataExtractTemplateExecutor addParameterConverter(String type, Function converter) { + parameterConverters.put(type, (v,a)->converter.apply(v)); + return this; + } + public final DataExtractTemplateExecutor addRestHelper(String name, IUnirestInstanceSupplier unirestInstanceSupplier, IProductHelper productHelper) { + restHelpers.put(name, new DataExtractRestHelper(name, unirestInstanceSupplier, productHelper)); + return this; + } + + private final void processParameters() { + template.getParameterDescriptors().forEach(this::processParameter); + } + + private final void processParameter(DataExtractTemplateSourceParameterDescriptor parameter) { + var name = parameter.getName(); + var value = inputParameters.get(name); + if ( value==null ) { + var defaultValueExpression = parameter.getDefaultValue(); + value = defaultValueExpression==null + ? null + : SpelEvaluator.JSON_DATA_EXTRACT.evaluate(defaultValueExpression, data, String.class); + } + data.set(name, convertParameterValue(value, parameter)); + } + + private JsonNode convertParameterValue(String value, DataExtractTemplateSourceParameterDescriptor parameter) { + var name = parameter.getName(); + var type = StringUtils.isBlank(parameter.getType()) ? "string" : parameter.getType(); + var required = parameter.isRequired(); + if ( StringUtils.isBlank(value) && required ) { + throw new IllegalArgumentException(String.format("Required parameter %s not specified", name)); + } + if ( value==null ) { return NullNode.instance; } + var paramConverter = parameterConverters.get(type); + if ( paramConverter==null ) { + throw new TemplateValidationException(String.format("Unknown parameter type %s for parameter %s", type, name)); + } else { + var args = ParameterTypeConverterArgs.builder() + .progressWriter(progressWriter) + .template(template) + .parameter(parameter) + .data(data) + .build(); + return paramConverter.apply(value, args); + } + } + + private final void processSources() { + template.getSources().forEach(s->processSource(s, data)); + } + + private final void processSource(DataExtractTemplateSourceDescriptor source, ObjectNode data) { + if ( source.getParameters()!=null ) { source.getParameters().forEach(p->processSource(p, data)); } + if ( source.getRequests()!=null ) { source.getRequests().forEach(p->processSource(p, data)); } + } + + private final void processSource(DataExtractTemplateSourceParameterDescriptor source, ObjectNode data) { + // Parameter values have already been inserted into the data object, so just need to + // handle forEach + processSourcePartialOutputs(source.getName(), data); + processSourceForEach(source.getName(), source.getForEach(), data); + } + + private final void processSource(DataExtractTemplateSourceRequestDescriptor source, ObjectNode data) { + executeRequest(source, data); + processSourcePartialOutputs(source.getName(), data); + } + + private final void processSourceForEach(String parentName, DataExtractTemplateSourceForEachDescriptor forEach, ObjectNode data) { + if ( forEach!=null ) { + var property = forEach.getProperty()==null ? SpelHelper.parseSimpleExpression(parentName) : forEach.getProperty(); + var input = SpelEvaluator.JSON_DATA_EXTRACT.evaluate(property, data, JsonNode.class); + if ( input!=null ) { + var childName = forEach.getName(); + if ( input instanceof ArrayNode ) { + processSourceForEach(childName, (ArrayNode)input, data, forEach.getSources()); + } else { + throw new TemplateValidationException("forEach not supported on node type "+input.getNodeType()); + } + } + } + } + + private final void processSourceForEach(String name, ArrayNode source, ObjectNode data, List sources) { + for ( int i = 0 ; i < source.size(); i++ ) { + var currentObject = source.get(i); + var newData = data.deepCopy(); + newData.set(name, currentObject); + processSourcePartialOutputs(name, newData); + sources.forEach(s->processSource(s, newData)); + } + } + + private final void executeRequest(DataExtractTemplateSourceRequestDescriptor requestDescriptor, ObjectNode data) { + var uri = SpelEvaluator.JSON_DATA_EXTRACT.evaluate(requestDescriptor.getUri(), data, String.class); + var query = evaluateQuery(requestDescriptor.getQuery()); + var restHelper = getRestHelper(requestDescriptor); + var unirest = restHelper.getUnirestInstance(); + var nextPageUrlProducer = restHelper.getNextPageUrlProducer(); + var request = unirest.get(uri).queryString(query); + if ( nextPageUrlProducer!=null ) { + PagingHelper.processPages(unirest, request, nextPageUrlProducer, r->processResponse(requestDescriptor, r)); + } else { + processResponse(requestDescriptor, request.asObject(JsonNode.class)); + } + } + + private final void processResponse(DataExtractTemplateSourceRequestDescriptor requestDescriptor, HttpResponse response) { + var name = requestDescriptor.getName(); + var rawBody = response.getBody(); + var body = getRestHelper(requestDescriptor).getInputTransformer().transformInput(rawBody); + data.set(name+"_raw", rawBody); + data.set(name, body); + processSourceForEach(requestDescriptor.getName(), requestDescriptor.getForEach(), data); + //System.out.println(body.toPrettyString()); + } + + // TODO Instead of Map, create separate DataExtractRestHelpers class, + // with add() methods for adding helpers, and this get-method for retrieval. + private DataExtractRestHelper getRestHelper(DataExtractTemplateSourceRequestDescriptor requestDescriptor) { + var name = requestDescriptor.getName(); + var from = requestDescriptor.getFrom(); + if ( StringUtils.isBlank(from) ) { + if ( restHelpers.size()==1 ) { + return restHelpers.values().iterator().next(); + } else { + throw new IllegalStateException(String.format("Required 'from:' property (allowed values: %s) missing for request %s", from, restHelpers.keySet(), name)); + } + } + var result = restHelpers.get(requestDescriptor.getFrom()); + if ( result==null ) { + throw new IllegalStateException(String.format("Invalid 'from: %s' for request %s, allowed values: %s", from, name, restHelpers.keySet())); + } + return result; + } + + private Map evaluateQuery(Map queryExpressions) { + Map result = new LinkedHashMap<>(); + if ( queryExpressions!=null ) { + queryExpressions.entrySet().forEach(e->result.put(e.getKey(), SpelEvaluator.JSON_DATA_EXTRACT.evaluate(e.getValue(), data, String.class))); + } + return result; + } + + private void processSourcePartialOutputs(String name, ObjectNode data) { + var partialOutputDefinitions = template.getPartialOutputs(); + if ( partialOutputDefinitions!=null ) { + var partialOutputs = partialOutputDefinitions.get(name); + if ( partialOutputs!=null ) { + partialOutputs.forEach(d->processPartialOutput(name, d, data)); + } + } + } + + private void processPartialOutput(String sourceName, DataExtractTemplateOutputDescriptor outputDescriptor, ObjectNode data) { + var partialOutputName = outputDescriptor.getName(); + var partialOutput = getOutput(outputDescriptor, data); + var partialOutputArray = (ArrayNode)partialOutputs.get(partialOutputName); + if ( partialOutputArray==null ) { + partialOutputArray = objectMapper.createArrayNode(); + partialOutputs.set(partialOutputName, partialOutputArray); + } + partialOutputArray.add(partialOutput); + } + + private JsonNode getOutput(DataExtractTemplateOutputDescriptor outputDescriptor, ObjectNode data) { + var outputRawContents = outputDescriptor.getContents(); + var output = new JsonNodeOutputWalker(outputDescriptor, data).walk(outputRawContents); + return output; + } + + private final void processOutputs() { + template.getOutputs().forEach(this::processOutput); + } + + private final void processOutput(DataExtractTemplateOutputDescriptor outputDescriptor) { + outputs.set(outputDescriptor.getName(), getOutput(outputDescriptor, data)); + } + + @Builder @Data + public static final class ParameterTypeConverterArgs { + private final IProgressWriterI18n progressWriter; + private final DataExtractTemplateDescriptor template; + private final DataExtractTemplateSourceParameterDescriptor parameter; + private final ObjectNode data; + } + + private static final Map> createDefaultParameterConverters() { + Map> result = new HashMap<>(); + result.put("string", (v,a)->new TextNode(v)); + result.put("boolean", (v,a)->BooleanNode.valueOf(Boolean.parseBoolean(v))); + result.put("int", (v,a)->IntNode.valueOf(Integer.parseInt(v))); + result.put("long", (v,a)->LongNode.valueOf(Long.parseLong(v))); + result.put("double", (v,a)->DoubleNode.valueOf(Double.parseDouble(v))); + result.put("float", (v,a)->FloatNode.valueOf(Float.parseFloat(v))); + // TODO Add BigIntegerNode/DecimalNode/ShortNode support? + return result; + } + + @RequiredArgsConstructor + private static final class JsonNodeOutputWalker extends JsonNodeDeepCopyWalker { + private final DataExtractTemplateOutputDescriptor outputDescriptor; + private final ObjectNode data; + @Override + protected JsonNode copyValue(JsonNode state, String path, JsonNode parent, ValueNode node) { + if ( !(node instanceof TextNode) ) { + return super.copyValue(state, path, parent, node); + } else { + TemplateExpression expression = outputDescriptor.getValueExpressions().get(path); + if ( expression==null ) { throw new RuntimeException("No expression for "+path); } + try { + var rawResult = SpelEvaluator.JSON_DATA_EXTRACT.evaluate(expression, data, Object.class); + return objectMapper.valueToTree(rawResult); + } catch ( SpelEvaluationException e ) { + throw new RuntimeException("Error evaluating template expression "+expression.getExpressionString(), e); + } + } + } + } +} diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/helper/DataExtractTemplateHelper.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/helper/DataExtractTemplateHelper.java new file mode 100644 index 0000000000..450bc007c5 --- /dev/null +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/helper/DataExtractTemplateHelper.java @@ -0,0 +1,86 @@ +/** + * Copyright 2023 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors (“Open Text”) are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.common.data_extract.helper; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; +import java.util.stream.Stream; + +import org.springframework.core.io.Resource; +import org.springframework.core.io.support.PathMatchingResourcePatternResolver; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fortify.cli.common.util.FileUtils; + +import lombok.SneakyThrows; + +public class DataExtractTemplateHelper { + private static final ObjectMapper yamlObjectMapper = new ObjectMapper(new YAMLFactory()); + private DataExtractTemplateHelper() {} + + public static final DataExtractTemplateDescriptor load(String type, String name) { + var resourceFile = getTemplateResourceFile(type, name); + try ( var is = FileUtils.getResourceInputStream(resourceFile) ) { + return load(resourceFile, is); + } catch ( IOException e ) { + throw new RuntimeException("Error loading template "+name, e); + } + } + + public static final Stream list(String type) { + return Stream.of(getTemplateResources(type)) + .map(DataExtractTemplateHelper::load); + } + + private static final String getTemplatesResourceDir(String type) { + return String.format("com/fortify/cli/%s/data_extract/templates", type.toLowerCase().replace('-', '_')); + } + + private static final String getTemplateResourceFile(String type, String name) { + return String.format("%s/%s.yaml", getTemplatesResourceDir(type), name); + } + + @SneakyThrows + private static final Resource[] getTemplateResources(String type) { + var matchPattern = String.format("%s/*.yaml", getTemplatesResourceDir(type)); + return new PathMatchingResourcePatternResolver().getResources(matchPattern); + } + + private static final DataExtractTemplateDescriptor load(Resource resource) { + try ( var is = resource.getInputStream() ) { + return load(resource.getFilename(), is); + } catch (IOException e) { + throw new RuntimeException("Error loading template "+resource.getFilename(), e); + } + } + + private static final DataExtractTemplateDescriptor load(String fileName, InputStream is) { + if ( is==null ) { + // TODO Use more descriptive exception message + throw new IllegalStateException("Can't read "+fileName); + } + try { + var result = yamlObjectMapper.readValue(is, DataExtractTemplateDescriptor.class); + result.postLoad(getTemplateName(fileName)); + return result; + } catch (IOException e) { + throw new RuntimeException("Error loading template "+fileName, e); + } + } + + private static final String getTemplateName(String fileName) { + return Path.of(fileName).getFileName().toString().replace(".yaml", ""); + } +} diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/json/AbstractJsonWalker.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/json/AbstractJsonWalker.java new file mode 100644 index 0000000000..cdc7dbd0b6 --- /dev/null +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/json/AbstractJsonWalker.java @@ -0,0 +1,75 @@ +/** + * Copyright 2023 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors (“Open Text”) are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.common.json; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ContainerNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.ValueNode; + +public abstract class AbstractJsonWalker { + public R walk(JsonNode node) { + walk("", null, node); + return getResult(); + } + protected abstract R getResult(); + + protected void walk(String path, JsonNode parent, JsonNode node) { + if ( !skipNode(path, parent, node) ) { + if ( node instanceof ContainerNode ) { + walkContainer(path, parent, (ContainerNode)node); + } else if ( node instanceof ValueNode ) { + walkValue(path, parent, (ValueNode)node); + } + } + } + + protected boolean skipNode(String path, JsonNode parent, JsonNode node) { + return false; + } + + protected void walkContainer(String path, JsonNode parent, ContainerNode node) { + if ( node instanceof ArrayNode ) { + walkArray(path, parent, (ArrayNode)node); + } else if ( node instanceof ObjectNode ) { + walkObject(path, parent, (ObjectNode)node); + } + } + + protected void walkObject(String path, JsonNode parent, ObjectNode node) { + node.fields().forEachRemaining(e->walkObjectProperty(appendPath(path, e.getKey()), node, e.getKey(), e.getValue())); + } + + protected void walkObjectProperty(String path, ObjectNode parent, String property, JsonNode value) { + walk(path, parent, value); + } + + protected void walkArray(String path, JsonNode parent, ArrayNode node) { + for ( int i = 0 ; i < node.size() ; i++ ) { + walkArrayElement(appendPath(path, i+""), node, i, node.get(i)); + } + } + + protected void walkArrayElement(String path, ArrayNode parent, int index, JsonNode value) { + walk(path, parent, value); + } + + protected void walkValue(String path, JsonNode parent, ValueNode node) { + + } + + protected final String appendPath(String parent, String entry) { + return String.format("%s[%s]", parent, entry); + } +} diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/json/JsonHelper.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/json/JsonHelper.java index 5495f20bc4..fa196743bd 100644 --- a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/json/JsonHelper.java +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/json/JsonHelper.java @@ -30,8 +30,11 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ContainerNode; +import com.fasterxml.jackson.databind.node.JsonNodeType; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; +import com.fasterxml.jackson.databind.node.ValueNode; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.fortify.cli.common.spring.expression.SpelEvaluator; import com.fortify.cli.common.util.StringUtils; @@ -149,5 +152,110 @@ public Function finisher() { public Set characteristics() { return EnumSet.of(Characteristics.UNORDERED); } - } + } + + public static abstract class AbstractJsonNodeWalker { + public final R walk(JsonNode node) { + if ( node!=null ) { + walk(null, "", null, node); + } + return getResult(); + } + protected abstract R getResult(); + + protected void walk(S state, String path, JsonNode parent, JsonNode node) { + if ( !skipNode(state, path, parent, node) ) { + if ( node instanceof ContainerNode ) { + walkContainer(state, path, parent, (ContainerNode)node); + } else if ( node instanceof ValueNode ) { + walkValue(state, path, parent, (ValueNode)node); + } + } + } + + protected boolean skipNode(S state, String path, JsonNode parent, JsonNode node) { + return false; + } + + protected void walkContainer(S state, String path, JsonNode parent, ContainerNode node) { + if ( node instanceof ArrayNode ) { + walkArray(state, path, parent, (ArrayNode)node); + } else if ( node instanceof ObjectNode ) { + walkObject(state, path, parent, (ObjectNode)node); + } + } + + protected void walkObject(S state, String path, JsonNode parent, ObjectNode node) { + node.fields().forEachRemaining(e->walkObjectProperty(state, appendPath(path, e.getKey()), node, e.getKey(), e.getValue())); + } + + protected void walkObjectProperty(S state, String path, ObjectNode parent, String property, JsonNode value) { + walk(state, path, parent, value); + } + + protected void walkArray(S state, String path, JsonNode parent, ArrayNode node) { + for ( int i = 0 ; i < node.size() ; i++ ) { + walkArrayElement(state, appendPath(path, i+""), node, i, node.get(i)); + } + } + + protected void walkArrayElement(S state, String path, ArrayNode parent, int index, JsonNode value) { + walk(state, path, parent, value); + } + + protected void walkValue(S state, String path, JsonNode parent, ValueNode node) {} + + protected final String appendPath(String parent, String entry) { + return String.format("%s[%s]", parent, entry); + } + } + + public static class JsonNodeDeepCopyWalker extends AbstractJsonNodeWalker { + @Getter JsonNode result; + @Override + protected void walkObject(JsonNode state, String path, JsonNode parent, ObjectNode node) { + if ( state==null ) { state = objectMapper.createObjectNode(); } + if ( result==null ) { result = state; } + super.walkObject(state, path, parent, node); + } + @Override + protected void walkObjectProperty(JsonNode state, String path, ObjectNode parent, String property, JsonNode value) { + if ( value instanceof ContainerNode ) { + var newState = createContainerNode(value.getNodeType()); + ((ObjectNode)state).set(property, newState); + super.walkObjectProperty(newState, path, parent, property, value); + } else { + ((ObjectNode)state).set(property, copyValue(state, path, parent, (ValueNode)value)); + } + } + @Override + protected void walkArray(JsonNode state, String path, JsonNode parent, ArrayNode node) { + if ( state==null ) { state = objectMapper.createArrayNode(); } + if ( result==null ) { result = state; } + super.walkArray(state, path, parent, node); + } + @Override + protected void walkArrayElement(JsonNode state, String path, ArrayNode parent, int index, JsonNode value) { + if ( value instanceof ContainerNode ) { + var newState = createContainerNode(value.getNodeType()); + ((ArrayNode)state).insert(index, newState); + super.walkArrayElement(newState, path, parent, index, value); + } else { + ((ArrayNode)state).insert(index, copyValue(state, path, parent, (ValueNode)value)); + } + } + @Override + protected void walkValue(JsonNode state, String path, JsonNode parent, ValueNode node) { + if ( result == null ) { result = copyValue(state, path, parent, node); } + } + protected final JsonNode createContainerNode(JsonNodeType jsonNodeType) { + return jsonNodeType==JsonNodeType.ARRAY + ? objectMapper.createArrayNode() + : objectMapper.createObjectNode(); + } + // We return JsonNode to allow subclasses to return other node types + protected JsonNode copyValue(JsonNode state, String path, JsonNode parent, ValueNode node) { + return node.deepCopy(); + } + } } diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/output/writer/output/standard/StandardOutputWriter.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/output/writer/output/standard/StandardOutputWriter.java index 247a4564c2..a1a8f7cde9 100644 --- a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/output/writer/output/standard/StandardOutputWriter.java +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/output/writer/output/standard/StandardOutputWriter.java @@ -168,13 +168,8 @@ private final void writeRecords(IRecordWriter recordWriter, HttpRequest httpR * @param httpRequest * @param nextPageRequestProducer */ - private final void writeRecords(IRecordWriter recordWriter, HttpRequest originalRequest, INextPageRequestProducer nextPageRequestProducer) { - var currentRequest = originalRequest; - while ( currentRequest!=null ) { - HttpResponse response = currentRequest.asObject(JsonNode.class); - writeRecords(recordWriter, response); - currentRequest = nextPageRequestProducer.getNextPageRequest(originalRequest, response); - } + private final void writeRecords(IRecordWriter recordWriter, HttpRequest initialRequest, INextPageRequestProducer nextPageRequestProducer) { + PagingHelper.processPages(initialRequest, nextPageRequestProducer, r->writeRecords(recordWriter, r)); } /** diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/rest/paging/PagingHelper.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/rest/paging/PagingHelper.java index 4b47dda9ab..eb388f7a0b 100644 --- a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/rest/paging/PagingHelper.java +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/rest/paging/PagingHelper.java @@ -12,6 +12,8 @@ *******************************************************************************/ package com.fortify.cli.common.rest.paging; +import java.util.function.Consumer; + import com.fasterxml.jackson.databind.JsonNode; import kong.unirest.Header; @@ -61,6 +63,23 @@ public static final INextPageRequestProducer asNextPageRequestProducer(UnirestIn return unirest==null || nextPageUrlProducer==null ? null : new NextPageRequestProducer(unirest, nextPageUrlProducer); } + public static final void processPages(UnirestInstance unirest, HttpRequest initialRequest, INextPageUrlProducer nextPageUrlProducer, Consumer> consumer) { + var nextPageRequestProducer = asNextPageRequestProducer(unirest, nextPageUrlProducer); + if ( nextPageRequestProducer==null ) { + throw new IllegalStateException("Cannot process pages without a valid NextPageRequestProducer"); + } + processPages(initialRequest, nextPageRequestProducer, consumer); + } + + public static final void processPages(HttpRequest initialRequest, INextPageRequestProducer nextPageRequestProducer, Consumer> consumer) { + var currentRequest = initialRequest; + while ( currentRequest!=null ) { + HttpResponse response = currentRequest.asObject(JsonNode.class); + consumer.accept(response); + currentRequest = nextPageRequestProducer.getNextPageRequest(initialRequest, response); + } + } + @RequiredArgsConstructor private static final class NextPageRequestProducer implements INextPageRequestProducer { private final UnirestInstance unirest; diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/SpelEvaluator.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/SpelEvaluator.java index ddbb1cbead..d9af7d0ddc 100644 --- a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/SpelEvaluator.java +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/SpelEvaluator.java @@ -25,6 +25,7 @@ import org.springframework.format.support.DefaultFormattingConversionService; import org.springframework.integration.json.JsonNodeWrapperToJsonNodeConverter; import org.springframework.integration.json.JsonPropertyAccessor; +import org.springframework.integration.json.JsonPropertyAccessor.JsonNodeWrapper; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; @@ -37,36 +38,51 @@ @RequiredArgsConstructor public enum SpelEvaluator { JSON_GENERIC(createJsonGenericContext()), - JSON_QUERY(createJsonQueryContext()); + JSON_QUERY(createJsonQueryContext()), + JSON_DATA_EXTRACT(createJsonDataExtractContext()); private static final SpelExpressionParser SPEL_PARSER = new SpelExpressionParser(); @Getter private final EvaluationContext context; public final R evaluate(Expression expression, Object input, Class returnClass) { - return expression.getValue(context, input, returnClass); + return unwrapSpelExpressionResult(expression.getValue(context, input, returnClass), returnClass); } public final R evaluate(String expression, Object input, Class returnClass) { - return evaluate(SPEL_PARSER.parseExpression(expression), input, returnClass); + return unwrapSpelExpressionResult(evaluate(SPEL_PARSER.parseExpression(expression), input, returnClass), returnClass); } - private static final EvaluationContext createJsonGenericContext() { + @SuppressWarnings("unchecked") + private static final R unwrapSpelExpressionResult(R result, Class returnClass) { + if ( result instanceof JsonNodeWrapper && returnClass.isAssignableFrom(JsonNode.class) ) { + result = (R)((JsonNodeWrapper)result).getRealNode(); + } + return result; + } + + private static final SimpleEvaluationContext createJsonGenericContext() { SimpleEvaluationContext context = SimpleEvaluationContext .forPropertyAccessors(new JsonPropertyAccessor()) .withConversionService(createJsonConversionService()) .withInstanceMethods() .build(); - SpelHelper.registerFunctions(context, StandardSpelFunctions.class); + SpelHelper.registerFunctions(context, SpelFunctionsStandard.class); return context; } - private static final EvaluationContext createJsonQueryContext() { + private static final SimpleEvaluationContext createJsonQueryContext() { SimpleEvaluationContext context = SimpleEvaluationContext .forPropertyAccessors(new ExistingJsonPropertyAccessor()) .withConversionService(createJsonConversionService()) .withInstanceMethods() .build(); - SpelHelper.registerFunctions(context, StandardSpelFunctions.class); + SpelHelper.registerFunctions(context, SpelFunctionsStandard.class); + return context; + } + + private static final EvaluationContext createJsonDataExtractContext() { + SimpleEvaluationContext context = createJsonGenericContext(); + SpelHelper.registerFunctions(context, SpelFunctionsDataExtract.class); return context; } diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/SpelFunctionsDataExtract.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/SpelFunctionsDataExtract.java new file mode 100644 index 0000000000..77ecfd7393 --- /dev/null +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/SpelFunctionsDataExtract.java @@ -0,0 +1,83 @@ +/******************************************************************************* + * Copyright 2021, 2023 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + *******************************************************************************/ +package com.fortify.cli.common.spring.expression; + +import java.util.List; + +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.safety.Safelist; + +import com.formkiq.graalvm.annotations.Reflectable; +import com.fortify.cli.common.util.StringUtils; + +import lombok.NoArgsConstructor; + +@Reflectable @NoArgsConstructor +public class SpelFunctionsDataExtract { + public static final String join(String separator, List elts) { + switch (separator) { + case "\\n": separator="\n"; break; + case "\\t": separator="\t"; break; + } + return String.join(separator, elts.stream().map(Object::toString).toList()); + } + + /** + * Convenience method to throw an exception if an expression evaluates to false + * @param throwError true if error should be thrown, false otherwise + * @param msg Message for exception to be thrown + * @return true if throwError is false + * @throws IllegalStateException with the given message if throwError is true + */ + public static final boolean check(boolean throwError, String msg) { + if ( throwError ) { + throw new IllegalStateException(msg); + } else { + return true; + } + } + + /** + * Abbreviate the given text to the given maximum width + * @param text to abbreviate + * @param maxWidth Maximum width + * @return Abbreviated text + */ + public static final String abbreviate(String text, int maxWidth) { + return StringUtils.abbreviate(text, maxWidth); + } + + /** + * @param html to be converted to plain text + * @return Formatted plain-text string for the given HTML contents + */ + public static final String htmlToText(String html) { + if( html==null ) { return null; } + Document document = Jsoup.parse(html); + document.outputSettings(new Document.OutputSettings().prettyPrint(false));//makes html() preserve linebreaks and spacing + document.select("br").append("\\n"); + document.select("p").prepend("\\n\\n"); + String s = document.html().replaceAll("\\\\n", "\n"); + return Jsoup.clean(s, "", Safelist.none(), new Document.OutputSettings().prettyPrint(false)); + } + + /** + * @param html to be converted to plain text + * @return Single line of plain text for the given HTML contents + */ + public static final String htmlToSingleLineText(String html) { + if( html==null ) { return null; } + return Jsoup.clean(html, "", Safelist.none()); + } +} diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/StandardSpelFunctions.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/SpelFunctionsStandard.java similarity index 98% rename from fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/StandardSpelFunctions.java rename to fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/SpelFunctionsStandard.java index 8952489a88..afe231b553 100644 --- a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/StandardSpelFunctions.java +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/SpelFunctionsStandard.java @@ -28,7 +28,7 @@ import lombok.NoArgsConstructor; @Reflectable @NoArgsConstructor -public class StandardSpelFunctions { +public class SpelFunctionsStandard { private static final DateTimePeriodHelper PeriodHelper = DateTimePeriodHelper.all(); public static final OffsetDateTime date(String s) { diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/SpelHelper.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/SpelHelper.java index eb41c96e63..69ebbcaa05 100644 --- a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/SpelHelper.java +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/SpelHelper.java @@ -14,9 +14,23 @@ import java.lang.reflect.Method; +import org.springframework.expression.common.TemplateParserContext; +import org.springframework.expression.spel.standard.SpelExpressionParser; import org.springframework.expression.spel.support.SimpleEvaluationContext; +import com.fortify.cli.common.spring.expression.wrapper.SimpleExpression; +import com.fortify.cli.common.spring.expression.wrapper.TemplateExpression; + public final class SpelHelper { + private static final SpelExpressionParser parser = new SpelExpressionParser(); + private static final TemplateParserContext templateContext = new TemplateParserContext("${","}"); + + public static final SimpleExpression parseSimpleExpression(String s) { + return new SimpleExpression(parser.parseExpression(s)); + } + public static final TemplateExpression parseTemplateExpression(String s) { + return new TemplateExpression(parser.parseExpression(s, templateContext)); + } public static final void registerFunctions(SimpleEvaluationContext context, Class clazz) { for ( Method m : clazz.getDeclaredMethods() ) { context.setVariable(m.getName(), m); diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/wrapper/SimpleExpression.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/wrapper/SimpleExpression.java new file mode 100644 index 0000000000..23625e90f1 --- /dev/null +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/wrapper/SimpleExpression.java @@ -0,0 +1,47 @@ +/******************************************************************************* + * (c) Copyright 2020 Micro Focus or one of its affiliates, a Micro Focus company + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including without + * limitation the rights to use, copy, modify, merge, publish, distribute, + * sublicense, and/or sell copies of the Software, and to permit persons to + * whom the Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY + * KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + ******************************************************************************/ +package com.fortify.cli.common.spring.expression.wrapper; + +import org.springframework.expression.Expression; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + +/** + *

This is a simple wrapper class for a Spring {@link Expression} + * instance. It's main use is in combination with + * {@link SimpleExpressionDeserializer} to allow automatic + * conversion from String values to simple {@link Expression} + * instances in JSON/YAML documents.

+ * + *

The reason for needing this wrapper class is to differentiate + * with templated {@link Expression} instances that are handled + * by {@link TemplateExpressionDeserializer}.

+ */ +@JsonDeserialize(using = SimpleExpressionDeserializer.class) +public class SimpleExpression extends WrappedExpression { + public SimpleExpression(Expression target) { + super(target); + } +} diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/wrapper/SimpleExpressionDeserializer.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/wrapper/SimpleExpressionDeserializer.java new file mode 100644 index 0000000000..790d39d63f --- /dev/null +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/wrapper/SimpleExpressionDeserializer.java @@ -0,0 +1,55 @@ +/******************************************************************************* + * (c) Copyright 2020 Micro Focus or one of its affiliates, a Micro Focus company + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including without + * limitation the rights to use, copy, modify, merge, publish, distribute, + * sublicense, and/or sell copies of the Software, and to permit persons to + * whom the Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY + * KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + ******************************************************************************/ +package com.fortify.cli.common.spring.expression.wrapper; + +import java.io.IOException; + +import org.springframework.expression.spel.standard.SpelExpressionParser; +import org.springframework.stereotype.Component; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; + +/** + * This Jackson deserializer allows parsing String values into an + * SpEL Expression object. + */ +@Component +public final class SimpleExpressionDeserializer extends StdDeserializer { + private static final long serialVersionUID = 1L; + private static final SpelExpressionParser parser = new SpelExpressionParser(); + public SimpleExpressionDeserializer() { this(null); } + public SimpleExpressionDeserializer(Class vc) { super(vc); } + + @Override + public SimpleExpression deserialize(JsonParser jp, DeserializationContext ctxt) + throws IOException, JsonProcessingException { + JsonNode node = jp.getCodec().readTree(jp); + return node==null || node.isNull() ? null : new SimpleExpression(parser.parseExpression(node.asText())); + } +} diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/wrapper/TemplateExpression.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/wrapper/TemplateExpression.java new file mode 100644 index 0000000000..ec37450f8d --- /dev/null +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/wrapper/TemplateExpression.java @@ -0,0 +1,47 @@ +/******************************************************************************* + * (c) Copyright 2020 Micro Focus or one of its affiliates, a Micro Focus company + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including without + * limitation the rights to use, copy, modify, merge, publish, distribute, + * sublicense, and/or sell copies of the Software, and to permit persons to + * whom the Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY + * KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + ******************************************************************************/ +package com.fortify.cli.common.spring.expression.wrapper; + +import org.springframework.expression.Expression; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + +/** + *

This is a simple wrapper class for a Spring {@link Expression} + * instance. It's main use is in combination with + * {@link TemplateExpressionDeserializer} to allow automatic + * conversion from String values to templated {@link Expression} + * instances.

+ * + *

The reason for needing this wrapper class is to differentiate + * with non-templated {@link Expression} instances that are + * handled by {@link SimpleExpressionDeserializer}.

+ */ +@JsonDeserialize(using = TemplateExpressionDeserializer.class) +public class TemplateExpression extends WrappedExpression { + public TemplateExpression(Expression target) { + super(target); + } +} diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/wrapper/TemplateExpressionDeserializer.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/wrapper/TemplateExpressionDeserializer.java new file mode 100644 index 0000000000..ca577c1564 --- /dev/null +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/wrapper/TemplateExpressionDeserializer.java @@ -0,0 +1,55 @@ +/******************************************************************************* + * (c) Copyright 2020 Micro Focus or one of its affiliates, a Micro Focus company + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including without + * limitation the rights to use, copy, modify, merge, publish, distribute, + * sublicense, and/or sell copies of the Software, and to permit persons to + * whom the Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY + * KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + ******************************************************************************/ +package com.fortify.cli.common.spring.expression.wrapper; + +import java.io.IOException; + +import org.springframework.stereotype.Component; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import com.fortify.cli.common.spring.expression.SpelHelper; + +/** + * This {@link PropertyEditor} allows parsing String values into a + * TemplateExpression object. + */ +@Component +public final class TemplateExpressionDeserializer extends StdDeserializer { + private static final long serialVersionUID = 1L; + + public TemplateExpressionDeserializer() { this(null); } + public TemplateExpressionDeserializer(Class vc) { super(vc); } + + @Override + public TemplateExpression deserialize(JsonParser jp, DeserializationContext ctxt) + throws IOException, JsonProcessingException { + JsonNode node = jp.getCodec().readTree(jp); + return node==null || node.isNull() ? null : SpelHelper.parseTemplateExpression(node.asText()); + } +} diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/wrapper/WrappedExpression.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/wrapper/WrappedExpression.java new file mode 100644 index 0000000000..c06bbe1132 --- /dev/null +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/wrapper/WrappedExpression.java @@ -0,0 +1,292 @@ +/******************************************************************************* + * (c) Copyright 2020 Micro Focus or one of its affiliates, a Micro Focus company + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including without + * limitation the rights to use, copy, modify, merge, publish, distribute, + * sublicense, and/or sell copies of the Software, and to permit persons to + * whom the Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY + * KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + ******************************************************************************/ +package com.fortify.cli.common.spring.expression.wrapper; + +import org.springframework.core.convert.TypeDescriptor; +import org.springframework.expression.EvaluationContext; +import org.springframework.expression.EvaluationException; +import org.springframework.expression.Expression; + +/** + *

This is a simple wrapper class for a Spring {@link Expression} + * instance. This class is used as a based class for both + * {@link SimpleExpression} and {@link TemplateExpression}.

+ */ +public class WrappedExpression implements Expression { + private final Expression target; + + /** + * Constructor for configuring the expression to be wrapped + * @param target {@link Expression} to be wrapped + */ + public WrappedExpression(Expression target) { + this.target = target; + } + + /** + * @see org.springframework.expression.Expression#getValue() + * @return The evaluation result + * @throws EvaluationException if there is a problem during evaluation + */ + public Object getValue() throws EvaluationException { + return target.getValue(); + } + + /** + * @see org.springframework.expression.Expression#getValue(Object) + * @param rootObject the root object against which to evaluate the expression + * @return the evaluation result + * @throws EvaluationException if there is a problem during evaluation + */ + public Object getValue(Object rootObject) throws EvaluationException { + return target.getValue(rootObject); + } + + /** + * @see org.springframework.expression.Expression#getValue(java.lang.Class) + * @param desiredResultType the class the caller would like the result to be + * @return the evaluation result + * @throws EvaluationException if there is a problem during evaluation + */ + public T getValue(Class desiredResultType) throws EvaluationException { + return target.getValue(desiredResultType); + } + + /** + * @see org.springframework.expression.Expression#getValue(Object, java.lang.Class) + * @param rootObject the root object against which to evaluate the expression + * @param desiredResultType the class the caller would like the result to be + * @return the evaluation result + * @throws EvaluationException if there is a problem during evaluation + */ + public T getValue(Object rootObject,Class desiredResultType) throws EvaluationException { + return target.getValue(rootObject, desiredResultType); + } + + /** + * @see org.springframework.expression.Expression#getValue(EvaluationContext) + * @param context the context in which to evaluate the expression + * @return the evaluation result + * @throws EvaluationException if there is a problem during evaluation + * + */ + public Object getValue(EvaluationContext context) throws EvaluationException { + return target.getValue(context); + } + + /** + * @see org.springframework.expression.Expression#getValue(EvaluationContext, Object) + * @param context the context in which to evaluate the expression + * @param rootObject the root object against which to evaluate the expression + * @return the evaluation result + * @throws EvaluationException if there is a problem during evaluation + * + */ + public Object getValue(EvaluationContext context, Object rootObject) throws EvaluationException { + return target.getValue(context, rootObject); + } + + /** + * @see org.springframework.expression.Expression#getValue(EvaluationContext, java.lang.Class) + * @param context the context in which to evaluate the expression + * @param desiredResultType the class the caller would like the result to be + * @return the evaluation result + * @throws EvaluationException if there is a problem during evaluation + */ + public T getValue(EvaluationContext context, Class desiredResultType) throws EvaluationException { + return target.getValue(context, desiredResultType); + } + + /** + * @see org.springframework.expression.Expression#getValue(EvaluationContext, Object, java.lang.Class) + * @param context the context in which to evaluate the expression + * @param rootObject the root object against which to evaluate the expression + * @param desiredResultType the class the caller would like the result to be + * @return the evaluation result + * @throws EvaluationException if there is a problem during evaluation + */ + public T getValue(EvaluationContext context, Object rootObject, Class desiredResultType) throws EvaluationException { + return target.getValue(context, rootObject, desiredResultType); + } + + /** + * @see org.springframework.expression.Expression#getValueType() + * @return the most general type of value that can be set on this context + * @throws EvaluationException if there is a problem determining the type + */ + public Class getValueType() throws EvaluationException { + return target.getValueType(); + } + + /** + * @see org.springframework.expression.Expression#getValueType(Object) + * @param rootObject the root object against which to evaluate the expression + * @return the most general type of value that can be set on this context + * @throws EvaluationException if there is a problem determining the type + */ + public Class getValueType(Object rootObject) throws EvaluationException { + return target.getValueType(rootObject); + } + + /** + * @see org.springframework.expression.Expression#getValueType(EvaluationContext) + * @param context the context in which to evaluate the expression + * @return the most general type of value that can be set on this context + * @throws EvaluationException if there is a problem determining the type + */ + public Class getValueType(EvaluationContext context) throws EvaluationException { + return target.getValueType(context); + } + + /** + * @see org.springframework.expression.Expression#getValueType(EvaluationContext, Object) + * @param context the context in which to evaluate the expression + * @param rootObject the root object against which to evaluate the expression + * @return the most general type of value that can be set on this context + * @throws EvaluationException if there is a problem determining the type + */ + public Class getValueType(EvaluationContext context, Object rootObject) throws EvaluationException { + return target.getValueType(context, rootObject); + } + + /** + * @see org.springframework.expression.Expression#getValueTypeDescriptor() + * @return a type descriptor for values that can be set on this context + * @throws EvaluationException if there is a problem determining the type + */ + public TypeDescriptor getValueTypeDescriptor() throws EvaluationException { + return target.getValueTypeDescriptor(); + } + + /** + * @see org.springframework.expression.Expression#getValueTypeDescriptor(Object) + * @param rootObject the root object against which to evaluate the expression + * @return a type descriptor for values that can be set on this context + * @throws EvaluationException if there is a problem determining the type + */ + public TypeDescriptor getValueTypeDescriptor(Object rootObject) throws EvaluationException { + return target.getValueTypeDescriptor(rootObject); + } + + /** + * @see org.springframework.expression.Expression#getValueTypeDescriptor(EvaluationContext) + * @param context the context in which to evaluate the expression + * @return a type descriptor for values that can be set on this context + * @throws EvaluationException if there is a problem determining the type + */ + public TypeDescriptor getValueTypeDescriptor(EvaluationContext context) throws EvaluationException { + return target.getValueTypeDescriptor(context); + } + + /** + * @see org.springframework.expression.Expression#getValueTypeDescriptor(EvaluationContext, Object) + * @param context the context in which to evaluate the expression + * @param rootObject the root object against which to evaluate the expression + * @return a type descriptor for values that can be set on this context + * @throws EvaluationException if there is a problem determining the type + */ + public TypeDescriptor getValueTypeDescriptor(EvaluationContext context, Object rootObject) throws EvaluationException { + return target.getValueTypeDescriptor(context, rootObject); + } + + /** + * @see org.springframework.expression.Expression#isWritable(EvaluationContext) + * @param context the context in which the expression should be checked + * @return {@code true} if the expression is writable; {@code false} otherwise + * @throws EvaluationException if there is a problem determining if it is writable + */ + public boolean isWritable(EvaluationContext context) throws EvaluationException { + return target.isWritable(context); + } + + /** + * @see org.springframework.expression.Expression#isWritable(EvaluationContext, Object) + * @param context the context in which the expression should be checked + * @param rootObject the root object against which to evaluate the expression + * @return {@code true} if the expression is writable; {@code false} otherwise + * @throws EvaluationException if there is a problem determining if it is writable + */ + public boolean isWritable(EvaluationContext context, Object rootObject) throws EvaluationException { + return target.isWritable(context, rootObject); + } + + /** + * @see org.springframework.expression.Expression#isWritable(Object) + * @param rootObject the root object against which to evaluate the expression + * @return {@code true} if the expression is writable; {@code false} otherwise + * @throws EvaluationException if there is a problem determining if it is writable + */ + public boolean isWritable(Object rootObject) throws EvaluationException { + return target.isWritable(rootObject); + } + + /** + * @see org.springframework.expression.Expression#setValue(EvaluationContext, Object) + * @param context the context in which to set the value of the expression + * @param value the new value + * @throws EvaluationException if there is a problem during evaluation + */ + public void setValue(EvaluationContext context, Object value) throws EvaluationException { + target.setValue(context, value); + } + + /** + * @see org.springframework.expression.Expression#setValue(Object, Object) + * @param rootObject the root object against which to evaluate the expression + * @param value the new value + * @throws EvaluationException if there is a problem during evaluation + */ + public void setValue(Object rootObject, Object value) throws EvaluationException { + target.setValue(rootObject, value); + } + + /** + * @see org.springframework.expression.Expression#setValue(EvaluationContext, Object, Object) + * @param context the context in which to set the value of the expression + * @param rootObject the root object against which to evaluate the expression + * @param value the new value + * @throws EvaluationException if there is a problem during evaluation + */ + public void setValue(EvaluationContext context, Object rootObject, Object value) throws EvaluationException { + target.setValue(context, rootObject, value); + } + + /** + * @see org.springframework.expression.Expression#getExpressionString() + * @return the original expression string + */ + public String getExpressionString() { + return target.getExpressionString(); + } + + /** + * @return String representation for this {@link WrappedExpression} + */ + @Override + public String toString() { + return this.getClass().getSimpleName()+"("+getExpressionString()+")"; + } + +} diff --git a/fcli-core/fcli-common/src/main/java/org/springframework/integration/json/JsonPropertyAccessor.java b/fcli-core/fcli-common/src/main/java/org/springframework/integration/json/JsonPropertyAccessor.java index 966e3cc719..0e21fe571d 100644 --- a/fcli-core/fcli-common/src/main/java/org/springframework/integration/json/JsonPropertyAccessor.java +++ b/fcli-core/fcli-common/src/main/java/org/springframework/integration/json/JsonPropertyAccessor.java @@ -210,7 +210,10 @@ else if (json.isValueNode()) { } } - interface JsonNodeWrapper extends Comparable { + // CHANGED: Compared to original version, this interface was changed to public + // to allow access by JsonHelper::evaluateSpelExpression (through + // JsonHelper::unwrapSpelExpressionResult). + public interface JsonNodeWrapper extends Comparable { String toString(); diff --git a/fcli-core/fcli-common/src/main/java/org/springframework/integration/json/package-info.java b/fcli-core/fcli-common/src/main/java/org/springframework/integration/json/package-info.java index 6bf933274f..136f18dda5 100644 --- a/fcli-core/fcli-common/src/main/java/org/springframework/integration/json/package-info.java +++ b/fcli-core/fcli-common/src/main/java/org/springframework/integration/json/package-info.java @@ -1,7 +1,11 @@ /** * This package contains a copy of two classes from spring-integration that * fcli depends on, to avoid having to pull in the full spring-integration - * dependency and transitive dependencies. + * dependency and transitive dependencies. Compared to the original version, + * there's one minor change; the JsonNodeWrapper interface has been changed + * to public to allow access to the original JsonNode instance if an SpEL + * expression returns a JsonNode, as used by JsonHelper::evaluateSpelExpression + * (through JsonHelper::unwrapSpelExpressionResult). */ package org.springframework.integration.json; diff --git a/fcli-core/fcli-common/src/main/resources/com/fortify/cli/common/i18n/FortifyCLIMessages.properties b/fcli-core/fcli-common/src/main/resources/com/fortify/cli/common/i18n/FortifyCLIMessages.properties index 9bf8c02f6a..1cba3b7034 100644 --- a/fcli-core/fcli-common/src/main/resources/com/fortify/cli/common/i18n/FortifyCLIMessages.properties +++ b/fcli-core/fcli-common/src/main/resources/com/fortify/cli/common/i18n/FortifyCLIMessages.properties @@ -27,6 +27,7 @@ log-level = Set logging level. Note that DEBUG and TRACE levels may result in se being written to the log file. Allowed values: ${COMPLETION-CANDIDATES}. log-file = File where logging data will be written. Defaults to fcli.log in current directory \ if --log-level is specified. +fcli.data-extract.create.template-parameter = TODO # Generic, non command-specific output and query options arggroup.output.heading = Output options:%n diff --git a/fcli-core/fcli-common/src/test/java/com/fortify/cli/common/json/JsonNodeDeepCopyWalkerTest.java b/fcli-core/fcli-common/src/test/java/com/fortify/cli/common/json/JsonNodeDeepCopyWalkerTest.java new file mode 100644 index 0000000000..5c5a070b2e --- /dev/null +++ b/fcli-core/fcli-common/src/test/java/com/fortify/cli/common/json/JsonNodeDeepCopyWalkerTest.java @@ -0,0 +1,105 @@ +/******************************************************************************* + * Copyright 2021, 2023 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + *******************************************************************************/ +package com.fortify.cli.common.json; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + +import java.util.UUID; +import java.util.stream.Stream; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +import com.fasterxml.jackson.databind.node.ValueNode; +import com.fortify.cli.common.json.JsonHelper.JsonNodeDeepCopyWalker; + +public class JsonNodeDeepCopyWalkerTest { + private static final ObjectMapper objectMapper = JsonHelper.getObjectMapper(); + + @ParameterizedTest + @MethodSource("getSampleNodesStream") + public void testDeepCopy(JsonNode node) throws Exception { + var copy = new JsonNodeDeepCopyWalker().walk(node); + assertEquals(node, copy); + if ( node!=null && !(node instanceof ValueNode) ) { + System.out.println(copy.toPrettyString()); + assertFalse(node==copy); + } + } + + private static Stream getSampleNodesStream() { + return Stream.of( + Arguments.of(new Object[] {null}), + Arguments.of(createEmptyObjectNode()), + Arguments.of(createSampleValueObjectNode()), + Arguments.of(createSampleMultiLevelObjectNode()), + Arguments.of(createEmptyArrayNode()), + Arguments.of(createSampleValueArrayNode()), + Arguments.of(createSampleMultiLevelArrayNode()), + Arguments.of(createSampleValueNode()) + ); + } + + private static ObjectNode createEmptyObjectNode() { + return objectMapper.createObjectNode(); + } + + private static ArrayNode createEmptyArrayNode() { + return objectMapper.createArrayNode(); + } + + private static ObjectNode createSampleMultiLevelObjectNode() { + var result = createSampleValueObjectNode(); + result.set("array", createSampleValueArrayNode()); + result.set("obj", createSampleValueObjectNode()); + return result; + } + + private static ObjectNode createSampleValueObjectNode() { + var result = objectMapper.createObjectNode(); + result.put("int", 1); + result.put("bool", true); + result.put("float", 2.2f); + result.put("str1", "someString"); + result.set("val", createSampleValueNode()); + return result; + } + + private static ArrayNode createSampleMultiLevelArrayNode() { + var result = objectMapper.createArrayNode(); + for ( int i = 0 ; i < 20 ; i++ ) { + result.add(createSampleMultiLevelObjectNode()); + } + return result; + } + + private static ArrayNode createSampleValueArrayNode() { + var result = objectMapper.createArrayNode(); + for ( int i = 0 ; i < 10 ; i++ ) { + result.add(100+i); + } + return result; + } + + private static ValueNode createSampleValueNode() { + return new TextNode(UUID.randomUUID().toString()); + } + +} diff --git a/fcli-core/fcli-sc-sast/src/main/java/com/fortify/cli/sc_sast/_main/cli/cmd/SCSastCommands.java b/fcli-core/fcli-sc-sast/src/main/java/com/fortify/cli/sc_sast/_main/cli/cmd/SCSastCommands.java index 299f8bf401..e659ca0f66 100644 --- a/fcli-core/fcli-sc-sast/src/main/java/com/fortify/cli/sc_sast/_main/cli/cmd/SCSastCommands.java +++ b/fcli-core/fcli-sc-sast/src/main/java/com/fortify/cli/sc_sast/_main/cli/cmd/SCSastCommands.java @@ -17,6 +17,7 @@ import com.fortify.cli.sc_sast.rest.cli.cmd.SCSastControllerRestCommands; import com.fortify.cli.sc_sast.scan.cli.cmd.SCSastScanCommands; import com.fortify.cli.sc_sast.sensor.cli.cmd.SCSastSensorCommands; +import com.fortify.cli.ssc.data_extract.cli.cmd.SSCDataExtractCommands; import picocli.CommandLine.Command; @@ -33,6 +34,7 @@ // 'rest' has a different header ('Interact with' compared to most // other commands ('Manage'). SCSastSessionCommands.class, + SSCDataExtractCommands.class, SCSastScanCommands.class, SCSastSensorCommands.class, SCSastControllerRestCommands.class, diff --git a/fcli-core/fcli-sc-sast/src/main/java/com/fortify/cli/sc_sast/data_extract/cli/cmd/SCSastDataExtractCommands.java b/fcli-core/fcli-sc-sast/src/main/java/com/fortify/cli/sc_sast/data_extract/cli/cmd/SCSastDataExtractCommands.java new file mode 100644 index 0000000000..438e276e6f --- /dev/null +++ b/fcli-core/fcli-sc-sast/src/main/java/com/fortify/cli/sc_sast/data_extract/cli/cmd/SCSastDataExtractCommands.java @@ -0,0 +1,27 @@ +/******************************************************************************* + * Copyright 2021, 2023 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + *******************************************************************************/ +package com.fortify.cli.sc_sast.data_extract.cli.cmd; + +import com.fortify.cli.common.cli.cmd.AbstractContainerCommand; +import com.fortify.cli.ssc.data_extract.cli.cmd.SSCDataExtractCreateCommand; + +import picocli.CommandLine.Command; + +@Command( + name = "data-extract", + subcommands = { + SSCDataExtractCreateCommand.class, + } +) +public class SCSastDataExtractCommands extends AbstractContainerCommand { +} diff --git a/fcli-core/fcli-sc-sast/src/main/java/com/fortify/cli/sc_sast/data_extract/cli/cmd/SCSastDataExtractCreateCommand.java b/fcli-core/fcli-sc-sast/src/main/java/com/fortify/cli/sc_sast/data_extract/cli/cmd/SCSastDataExtractCreateCommand.java new file mode 100644 index 0000000000..4fad9fa564 --- /dev/null +++ b/fcli-core/fcli-sc-sast/src/main/java/com/fortify/cli/sc_sast/data_extract/cli/cmd/SCSastDataExtractCreateCommand.java @@ -0,0 +1,89 @@ +/******************************************************************************* + * Copyright 2021, 2023 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + *******************************************************************************/ +package com.fortify.cli.sc_sast.data_extract.cli.cmd; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fortify.cli.common.data_extract.cli.cmd.AbstractDataExtractCreateCommand; +import com.fortify.cli.common.data_extract.helper.DataExtractTemplateDescriptor.TemplateValidationException; +import com.fortify.cli.common.data_extract.helper.DataExtractTemplateExecutor; +import com.fortify.cli.common.data_extract.helper.DataExtractTemplateExecutor.ParameterTypeConverterArgs; +import com.fortify.cli.common.output.cli.mixin.OutputHelperMixins; +import com.fortify.cli.common.spring.expression.SpelEvaluator; +import com.fortify.cli.common.spring.expression.SpelHelper; +import com.fortify.cli.common.util.StringUtils; +import com.fortify.cli.sc_sast._common.rest.helper.SCSastControllerProductHelper; +import com.fortify.cli.sc_sast._common.rest.helper.SCSastSSCProductHelper; +import com.fortify.cli.sc_sast._common.session.cli.mixin.SCSastUnirestInstanceSupplierMixin; +import com.fortify.cli.ssc.appversion.helper.SSCAppVersionHelper; +import com.fortify.cli.ssc.issue.helper.SSCIssueFilterSetHelper; + +import lombok.Getter; +import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; + +@Command(name = OutputHelperMixins.Create.CMD_NAME) +public class SCSastDataExtractCreateCommand extends AbstractDataExtractCreateCommand { + @Getter @Mixin private SCSastUnirestInstanceSupplierMixin unirestInstanceSupplier; + + @Override + protected final String getType() { + return "SC-SAST"; + } + + @Override + protected void configure(DataExtractTemplateExecutor templateExecutor) { + templateExecutor + .addParameterConverter("appversion_single", this::loadAppVersion) + .addParameterConverter("filterset", this::loadFilterSet) + .addRestHelper("ssc", unirestInstanceSupplier::getSscUnirestInstance, SCSastSSCProductHelper.INSTANCE) + .addRestHelper("sc-sast", unirestInstanceSupplier::getControllerUnirestInstance, SCSastControllerProductHelper.INSTANCE); + } + + private final JsonNode loadAppVersion(String nameOrId, ParameterTypeConverterArgs args) { + args.getProgressWriter().writeProgress("Loading application version %s", nameOrId); + return SSCAppVersionHelper.getRequiredAppVersion(unirestInstanceSupplier.getSscUnirestInstance(), nameOrId, ":").asJsonNode(); + } + + private final JsonNode loadFilterSet(String titleOrId, ParameterTypeConverterArgs args) { + args.getProgressWriter().writeProgress("Loading filter set %s", titleOrId); + var parameter = args.getParameter(); + var typeParameters = parameter.getTypeParameters(); + var appVersionIdExpression = typeParameters==null ? null : typeParameters.get("appversion.id"); + if ( appVersionIdExpression==null ) { + appVersionIdExpression = SpelHelper.parseTemplateExpression("${appversion?.id}"); + } + var appVersionId = SpelEvaluator.JSON_DATA_EXTRACT.evaluate(appVersionIdExpression, args.getData(), String.class); + if ( StringUtils.isBlank(appVersionId) ) { + throw new TemplateValidationException(String.format("Template parameter %s requires ${%s} to be available", parameter.getName(), appVersionIdExpression.getExpressionString())); + } + return new SSCIssueFilterSetHelper(unirestInstanceSupplier.getSscUnirestInstance(), appVersionId).getDescriptorByTitleOrId(titleOrId, false).asJsonNode(); + } + + /* + @Override + protected final List getRestHelpers() { + return Arrays.asList( + new DataExtractRestHelper("ssc", unirestInstanceSupplier::getSscUnirestInstance, SCSastSSCProductHelper.INSTANCE), + new DataExtractRestHelper("sc-sast", unirestInstanceSupplier::getControllerUnirestInstance, SCSastControllerProductHelper.INSTANCE) + ); + } + + @Override + protected final List getParamHelpers() { + return Arrays.asList( + new DataExtractParameterHelper("appversion_single", v-> + SSCAppVersionHelper.getRequiredAppVersion(unirestInstanceSupplier.getSscUnirestInstance(), v, ":").asJsonNode()) + ); + } + */ +} diff --git a/fcli-core/fcli-sc-sast/src/main/resources/com/fortify/cli/data_extract/templates/SC-SAST.zip b/fcli-core/fcli-sc-sast/src/main/resources/com/fortify/cli/data_extract/templates/SC-SAST.zip deleted file mode 100644 index 1b82ba32a2..0000000000 Binary files a/fcli-core/fcli-sc-sast/src/main/resources/com/fortify/cli/data_extract/templates/SC-SAST.zip and /dev/null differ diff --git a/fcli-core/fcli-sc-sast/src/main/resources/com/fortify/cli/sc_sast/data_extract/templates/github-code-scanning.yaml b/fcli-core/fcli-sc-sast/src/main/resources/com/fortify/cli/sc_sast/data_extract/templates/github-code-scanning.yaml new file mode 100644 index 0000000000..4259e820e6 --- /dev/null +++ b/fcli-core/fcli-sc-sast/src/main/resources/com/fortify/cli/sc_sast/data_extract/templates/github-code-scanning.yaml @@ -0,0 +1,104 @@ +description: | + bla + +sources: + - parameters: + - name: appversion + type: appversion_single + - name: filterset + type: filterset + + - requests: + - name: issues + uri: /api/v1/projectVersions/${appversion.id}/issues?limit=200 + query: + filter: ISSUE[11111111-1111-1111-1111-111111111151]:SCA + filterset: ${filterset.guid} + from: ssc + forEach: + name: issue + sources: + - requests: + - name: issue_details + uri: /api/v1/issueDetails/${issue.id} + from: ssc + +outputs: + - name: markdownTest + contents: |- + ## ${appversion.project.name} - ${appversion.name} + bla bla + + ${#join('\n', partialOutputs.mdTest)} + - name: github-code-scanning + contents: + "[$schema]": https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json + version: '2.1.0' + runs: + - tool: + driver: + name: 'Fortify SCA' + version: ${appversion.currentStaticScan?.engineVersion?:'unknown'} + rules: ${partialOutputs.rules} + results: ${#check(partialOutputs.results.size()>1000 , "GitHub does not support importing more than 1000 vulnerabilities. Please clean the scan results or update vulnerability search criteria.")?partialOutputs.results:{}} + +partialOutputs: + issue_details: + - name: mdTest + contents: |- + ### ${issue.id} + ${issue_details?.brief} + - name: rules + contents: + id: ${issue.id+''} + shortDescription: + text: ${issue.issueName} + fullDescription: + text: ${issue_details?.brief} + help: + text: ${#htmlToText(issue_details?.detail)+'\n\n'+#htmlToText(issue_details?.recommendation)+"\n\nFor more information, see "+issue.deepLink} + properties: + tags: ${{issue.friority}} + precision: ${(issue.friority matches "(Critical|Medium)") ? "high":"low" } + security-severity: ${{Critical:10.0,High:8.9,Medium:6.9,Low:3.9}.get(issue.friority)+''} + - name: results + contents: + ruleId: ${issue.id+''} + message: + text: ${issue_details?.brief} + level: ${(issue.friority matches "(Critical|High)") ? "warning":"note" } + partialFingerprints: + issueInstanceId: ${issue.issueInstanceId} + locations: + - physicalLocation: + artifactLocation: + uri: ${issue.fullFileName} + region: + startLine: ${issue.lineNumber==0?1:issue.lineNumber} + endLine: ${issue.lineNumber==0?1:issue.lineNumber} + startColumn: ${1} # Needs to be specified as an expression in order to end up as integer instead of string in JSON + endColumn: ${80} + codeFlows: |- + ${ + issue_details?.traceNodes==null ? {} + : + {{ + threadFlows: issue_details?.traceNodes.![{ + locations: #this.![{ + location: { + message: { + text: text + }, + physicalLocation: { + artifactLocation: { + uri: fullPath + }, + region: { + startLine: line==0?1:line + } + } + } + }] + }] + }} + } \ No newline at end of file diff --git a/fcli-core/fcli-sc-sast/src/main/resources/com/fortify/cli/sc_sast/i18n/SCSastMessages.properties b/fcli-core/fcli-sc-sast/src/main/resources/com/fortify/cli/sc_sast/i18n/SCSastMessages.properties index 9c79d702ee..174c2a32f7 100644 --- a/fcli-core/fcli-sc-sast/src/main/resources/com/fortify/cli/sc_sast/i18n/SCSastMessages.properties +++ b/fcli-core/fcli-sc-sast/src/main/resources/com/fortify/cli/sc_sast/i18n/SCSastMessages.properties @@ -101,6 +101,11 @@ fcli.sc-sast.session.list.usage.description.1 = For sessions created using user fcli.sc-sast.session.list.usage.description.2 = For sessions created using a pre-generated token, fcli cannot \ display session expiration date or status, as SSC doesn't allow for obtaining this information. +# fcli sc-sast data-extract +fcli.sc-sast.data-extract.usage.header = TODO +fcli.sc-sast.data-extract.create.usage.header = TODO +fcli.sc-sast.data-extract.create.template = TODO + # fcli sc-sast scan fcli.sc-sast.scan.usage.header = Manage ScanCentral SAST scans. fcli.sc-sast.scan.cancel.usage.header = Cancel a previously submitted scan request. diff --git a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/_main/cli/cmd/SSCCommands.java b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/_main/cli/cmd/SSCCommands.java index e9c36420a4..449615bee6 100644 --- a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/_main/cli/cmd/SSCCommands.java +++ b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/_main/cli/cmd/SSCCommands.java @@ -20,6 +20,7 @@ import com.fortify.cli.ssc.appversion.cli.cmd.SSCAppVersionCommands; import com.fortify.cli.ssc.artifact.cli.cmd.SSCArtifactCommands; import com.fortify.cli.ssc.attribute.cli.cmd.SSCAttributeCommands; +import com.fortify.cli.ssc.data_extract.cli.cmd.SSCDataExtractCommands; import com.fortify.cli.ssc.issue.cli.cmd.SSCIssueCommands; import com.fortify.cli.ssc.performance_indicator.cli.cmd.SSCPerformanceIndicatorCommands; import com.fortify.cli.ssc.plugin.cli.cmd.SSCPluginCommands; @@ -49,6 +50,7 @@ SSCAppVersionCommands.class, SSCArtifactCommands.class, SSCAttributeCommands.class, + SSCDataExtractCommands.class, SSCIssueCommands.class, SSCPerformanceIndicatorCommands.class, SSCVariableCommands.class, diff --git a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/data_extract/cli/cmd/SSCDataExtractCommands.java b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/data_extract/cli/cmd/SSCDataExtractCommands.java new file mode 100644 index 0000000000..63d9522b3d --- /dev/null +++ b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/data_extract/cli/cmd/SSCDataExtractCommands.java @@ -0,0 +1,26 @@ +/******************************************************************************* + * Copyright 2021, 2023 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + *******************************************************************************/ +package com.fortify.cli.ssc.data_extract.cli.cmd; + +import com.fortify.cli.common.cli.cmd.AbstractContainerCommand; + +import picocli.CommandLine.Command; + +@Command( + name = "data-extract", + subcommands = { + SSCDataExtractCreateCommand.class, + } +) +public class SSCDataExtractCommands extends AbstractContainerCommand { +} diff --git a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/data_extract/cli/cmd/SSCDataExtractCreateCommand.java b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/data_extract/cli/cmd/SSCDataExtractCreateCommand.java new file mode 100644 index 0000000000..c171ff2233 --- /dev/null +++ b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/data_extract/cli/cmd/SSCDataExtractCreateCommand.java @@ -0,0 +1,87 @@ +/******************************************************************************* + * Copyright 2021, 2023 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + *******************************************************************************/ +package com.fortify.cli.ssc.data_extract.cli.cmd; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fortify.cli.common.data_extract.cli.cmd.AbstractDataExtractCreateCommand; +import com.fortify.cli.common.data_extract.helper.DataExtractTemplateDescriptor.TemplateValidationException; +import com.fortify.cli.common.data_extract.helper.DataExtractTemplateExecutor; +import com.fortify.cli.common.data_extract.helper.DataExtractTemplateExecutor.ParameterTypeConverterArgs; +import com.fortify.cli.common.output.cli.mixin.OutputHelperMixins; +import com.fortify.cli.common.spring.expression.SpelEvaluator; +import com.fortify.cli.common.spring.expression.SpelHelper; +import com.fortify.cli.common.util.StringUtils; +import com.fortify.cli.ssc._common.rest.helper.SSCProductHelper; +import com.fortify.cli.ssc._common.session.cli.mixin.SSCUnirestInstanceSupplierMixin; +import com.fortify.cli.ssc.appversion.helper.SSCAppVersionHelper; +import com.fortify.cli.ssc.issue.helper.SSCIssueFilterSetHelper; + +import lombok.Getter; +import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; + +@Command(name = OutputHelperMixins.Create.CMD_NAME) +public class SSCDataExtractCreateCommand extends AbstractDataExtractCreateCommand { + @Getter @Mixin private SSCUnirestInstanceSupplierMixin unirestInstanceSupplier; + + @Override + protected final String getType() { + return "SSC"; + } + + @Override + protected void configure(DataExtractTemplateExecutor templateExecutor) { + templateExecutor + .addParameterConverter("appversion_single", this::loadAppVersion) + .addParameterConverter("filterset", this::loadFilterSet) + .addRestHelper("ssc", unirestInstanceSupplier::getUnirestInstance, SSCProductHelper.INSTANCE); + } + + private final JsonNode loadAppVersion(String nameOrId, ParameterTypeConverterArgs args) { + args.getProgressWriter().writeProgress("Loading application version %s", nameOrId); + return SSCAppVersionHelper.getRequiredAppVersion(unirestInstanceSupplier.getUnirestInstance(), nameOrId, ":").asJsonNode(); + } + + private final JsonNode loadFilterSet(String titleOrId, ParameterTypeConverterArgs args) { + args.getProgressWriter().writeProgress("Loading filter set %s", titleOrId); + var parameter = args.getParameter(); + var typeParameters = parameter.getTypeParameters(); + var appVersionIdExpression = typeParameters==null ? null : typeParameters.get("appversion.id"); + if ( appVersionIdExpression==null ) { + appVersionIdExpression = SpelHelper.parseTemplateExpression("${appversion?.id}"); + } + var appVersionId = SpelEvaluator.JSON_DATA_EXTRACT.evaluate(appVersionIdExpression, args.getData(), String.class); + if ( StringUtils.isBlank(appVersionId) ) { + throw new TemplateValidationException(String.format("Template parameter %s requires ${%s} to be available", parameter.getName(), appVersionIdExpression.getExpressionString())); + } + return new SSCIssueFilterSetHelper(unirestInstanceSupplier.getUnirestInstance(), appVersionId).getDescriptorByTitleOrId(titleOrId, false).asJsonNode(); + } + + /* + @Override + protected final List getRestHelpers() { + return Arrays.asList( + new DataExtractRestHelper("ssc", unirestInstanceSupplier::getSscUnirestInstance, SCSastSSCProductHelper.INSTANCE), + new DataExtractRestHelper("sc-sast", unirestInstanceSupplier::getControllerUnirestInstance, SCSastControllerProductHelper.INSTANCE) + ); + } + + @Override + protected final List getParamHelpers() { + return Arrays.asList( + new DataExtractParameterHelper("appversion_single", v-> + SSCAppVersionHelper.getRequiredAppVersion(unirestInstanceSupplier.getSscUnirestInstance(), v, ":").asJsonNode()) + ); + } + */ +} diff --git a/fcli-core/fcli-ssc/src/main/resources/com/fortify/cli/ssc/data_extract/templates/github-code-scanning.yaml b/fcli-core/fcli-ssc/src/main/resources/com/fortify/cli/ssc/data_extract/templates/github-code-scanning.yaml new file mode 100644 index 0000000000..4259e820e6 --- /dev/null +++ b/fcli-core/fcli-ssc/src/main/resources/com/fortify/cli/ssc/data_extract/templates/github-code-scanning.yaml @@ -0,0 +1,104 @@ +description: | + bla + +sources: + - parameters: + - name: appversion + type: appversion_single + - name: filterset + type: filterset + + - requests: + - name: issues + uri: /api/v1/projectVersions/${appversion.id}/issues?limit=200 + query: + filter: ISSUE[11111111-1111-1111-1111-111111111151]:SCA + filterset: ${filterset.guid} + from: ssc + forEach: + name: issue + sources: + - requests: + - name: issue_details + uri: /api/v1/issueDetails/${issue.id} + from: ssc + +outputs: + - name: markdownTest + contents: |- + ## ${appversion.project.name} - ${appversion.name} + bla bla + + ${#join('\n', partialOutputs.mdTest)} + - name: github-code-scanning + contents: + "[$schema]": https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json + version: '2.1.0' + runs: + - tool: + driver: + name: 'Fortify SCA' + version: ${appversion.currentStaticScan?.engineVersion?:'unknown'} + rules: ${partialOutputs.rules} + results: ${#check(partialOutputs.results.size()>1000 , "GitHub does not support importing more than 1000 vulnerabilities. Please clean the scan results or update vulnerability search criteria.")?partialOutputs.results:{}} + +partialOutputs: + issue_details: + - name: mdTest + contents: |- + ### ${issue.id} + ${issue_details?.brief} + - name: rules + contents: + id: ${issue.id+''} + shortDescription: + text: ${issue.issueName} + fullDescription: + text: ${issue_details?.brief} + help: + text: ${#htmlToText(issue_details?.detail)+'\n\n'+#htmlToText(issue_details?.recommendation)+"\n\nFor more information, see "+issue.deepLink} + properties: + tags: ${{issue.friority}} + precision: ${(issue.friority matches "(Critical|Medium)") ? "high":"low" } + security-severity: ${{Critical:10.0,High:8.9,Medium:6.9,Low:3.9}.get(issue.friority)+''} + - name: results + contents: + ruleId: ${issue.id+''} + message: + text: ${issue_details?.brief} + level: ${(issue.friority matches "(Critical|High)") ? "warning":"note" } + partialFingerprints: + issueInstanceId: ${issue.issueInstanceId} + locations: + - physicalLocation: + artifactLocation: + uri: ${issue.fullFileName} + region: + startLine: ${issue.lineNumber==0?1:issue.lineNumber} + endLine: ${issue.lineNumber==0?1:issue.lineNumber} + startColumn: ${1} # Needs to be specified as an expression in order to end up as integer instead of string in JSON + endColumn: ${80} + codeFlows: |- + ${ + issue_details?.traceNodes==null ? {} + : + {{ + threadFlows: issue_details?.traceNodes.![{ + locations: #this.![{ + location: { + message: { + text: text + }, + physicalLocation: { + artifactLocation: { + uri: fullPath + }, + region: { + startLine: line==0?1:line + } + } + } + }] + }] + }} + } \ No newline at end of file diff --git a/fcli-core/fcli-ssc/src/main/resources/com/fortify/cli/ssc/i18n/SSCMessages.properties b/fcli-core/fcli-ssc/src/main/resources/com/fortify/cli/ssc/i18n/SSCMessages.properties index 247df255de..503d6b3a36 100644 --- a/fcli-core/fcli-ssc/src/main/resources/com/fortify/cli/ssc/i18n/SSCMessages.properties +++ b/fcli-core/fcli-ssc/src/main/resources/com/fortify/cli/ssc/i18n/SSCMessages.properties @@ -333,6 +333,11 @@ fcli.ssc.attribute.get-definition.usage.header = Get attribute definition detail fcli.ssc.attribute.list-definitions.usage.header = List attribute definitions. fcli.ssc.attribute-definition.resolver.nameOrId = Attribute definition name or id. +# fcli ssc data-extract +fcli.ssc.data-extract.usage.header = TODO +fcli.ssc.data-extract.create.usage.header = TODO +fcli.ssc.data-extract.create.template = TODO + # fcli ssc issue fcli.ssc.issue.usage.header = Manage SSC issues (vulnerabilities) and related entities like issue templates, filters and groups. fcli.ssc.issue.count.usage.header = Count application version vulnerabilities by grouping. diff --git a/fcli-other/fcli-bom/build.gradle b/fcli-other/fcli-bom/build.gradle index 236a6cc0ed..b6dcdb95e7 100644 --- a/fcli-other/fcli-bom/build.gradle +++ b/fcli-other/fcli-bom/build.gradle @@ -53,7 +53,11 @@ dependencies { api 'org.junit.jupiter:junit-jupiter-params:5.9.3' api 'org.junit.jupiter:junit-jupiter-engine:5.9.3' - //required for unpacking tar.gz (debricked cli) + // Required for unpacking tar.gz (debricked cli) api('org.apache.commons:commons-compress:1.25.0') + + // Used for processing HTML text returned by SSC/FoD endpoints like issue summaries/details/... + api('org.jsoup:jsoup:1.17.2') + } } \ No newline at end of file diff --git a/fcli-other/fcli-gradle/fcli-java.gradle b/fcli-other/fcli-gradle/fcli-java.gradle index ee1eb57ca8..76ac6766c7 100644 --- a/fcli-other/fcli-gradle/fcli-java.gradle +++ b/fcli-other/fcli-gradle/fcli-java.gradle @@ -60,8 +60,11 @@ dependencies { testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine' testRuntimeOnly 'org.junit.platform:junit-platform-launcher' - //required for unpacking tar.gz (debricked cli) - implementation('org.apache.commons:commons-compress') + // Required for unpacking tar.gz (debricked cli) + implementation('org.apache.commons:commons-compress') + + // Used for processing HTML text returned by SSC/FoD endpoints like issue summaries/details/... + implementation('org.jsoup:jsoup') } compileJava {