diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/cli/mixin/CommonOptionMixins.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/cli/mixin/CommonOptionMixins.java index 2488882254..fda1e8eeb5 100644 --- a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/cli/mixin/CommonOptionMixins.java +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/cli/mixin/CommonOptionMixins.java @@ -13,13 +13,33 @@ package com.fortify.cli.common.cli.mixin; import java.io.File; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Stack; +import java.util.function.Consumer; +import java.util.function.Function; +import com.formkiq.graalvm.annotations.Reflectable; import com.fortify.cli.common.util.PicocliSpecHelper; import com.fortify.cli.common.util.StringUtils; +import com.github.freva.asciitable.AsciiTable; +import com.github.freva.asciitable.Column; +import com.github.freva.asciitable.HorizontalAlign; +import lombok.Data; import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import picocli.CommandLine.Command; +import picocli.CommandLine.IParameterPreprocessor; import picocli.CommandLine.Mixin; +import picocli.CommandLine.Model.ArgSpec; import picocli.CommandLine.Model.CommandSpec; +import picocli.CommandLine.Model.UnmatchedArgsBinding; import picocli.CommandLine.Option; import picocli.CommandLine.ParameterException; @@ -78,4 +98,157 @@ private String getPlainPrompt(CommandSpec spec, Object... promptArgs) { return prompt; } } + + /** + * This mixin allows for collecting any unmatched command line options + * and parsing them into an [option name]=[value] map. + */ + @Command(preprocessor = OptionParametersMixinPreprocessor.class) + public static final class OptionParametersMixin { + @Getter private Map options = new LinkedHashMap<>(); + @Getter private Map optionNameToCurrentArgMap = new LinkedHashMap<>(); + @Mixin private CommandHelperMixin commandHelper; + + public final T setUnmatchedArgs(T arg) { + parse((String[])arg); + return arg; + } + + public final void validate(Consumer validatorConfigurer) { + var validator = new OptionParameterValidator(); + validatorConfigurer.accept(validator); + validator.validate(); + } + + public final class OptionParameterValidator { + private final Map optionsWithDescriptions = new LinkedHashMap<>(); + private final Map> optionValidators = new LinkedHashMap<>(); + private final List validationErrors = new ArrayList<>(); + @Setter private String unknownOptionFormat = "Unknown option: %s"; + @Setter private String validationExceptionFormat = "Invalid options:\n %s\nSupported options:\n%s\n"; + + public final void addSupportedOption(String name, String description) { + optionsWithDescriptions.put(name, description); + } + + public final void addOptionValidator(String name, Function validator) { + optionValidators.put(name, validator); + } + + public final void requiredOption(String name, String validationErrorFormat) { + addOptionValidator(name, args->StringUtils.isBlank(args.getValue()) + ? String.format(validationErrorFormat, args.getArg()) + : null); + } + + public final void validate() { + for ( var e : optionNameToCurrentArgMap.entrySet() ) { + if ( !optionsWithDescriptions.containsKey(e.getKey()) ) { + validationErrors.add(String.format(unknownOptionFormat, e.getValue())); + } + } + optionValidators.entrySet().forEach(this::validate); + if ( validationErrors.size()>0 ) { + var errorsString = String.join("\n ", validationErrors); + var supportedOptionsString = getSupportedOptionsTable(); + var msg = String.format(validationExceptionFormat, errorsString, supportedOptionsString); + throw new ParameterException(commandHelper.getCommandSpec().commandLine(), msg); + } + } + + private final String getSupportedOptionsTable() { + return AsciiTable.builder() + .border(AsciiTable.NO_BORDERS) + .data(new Column[] { + new Column().dataAlign(HorizontalAlign.LEFT), + new Column().dataAlign(HorizontalAlign.LEFT), + }, + optionsWithDescriptions.entrySet().stream() + .map(e->new String[] {asArg(e.getKey()), e.getValue()}) + .toList().toArray(String[][]::new)) + .asString(); + } + + private final String asArg(String name) { + var arg = optionNameToCurrentArgMap.get(name); + if ( arg==null ) { + arg = name.length()==1 ? "-"+name : "--"+name; + } + return arg; + } + + private final void validate(Map.Entry> entry) { + var name = entry.getKey(); + var arg = asArg(name); + var value = options.get(name); + var validationError = entry.getValue().apply(new OptionParameterValidatorArgs(name, arg, value)); + if ( StringUtils.isNotBlank(validationError) ) { + validationErrors.add(validationError); + } + } + } + + @Data + public static final class OptionParameterValidatorArgs { + private final String name; + private final String arg; + private final String value; + } + + private final void parse(String[] argsArray) { + var args = asDeque(argsArray); + while ( !args.isEmpty() ) { + var arg = args.pop(); + if ( !arg.startsWith("-") ) { + throw new IllegalArgumentException("Unknown command line option: "+arg); + } else { + var optName = arg.replaceFirst("-+", ""); + getOptionNameToCurrentArgMap().put(optName, arg); + var nextArg = args.peek(); + if ( nextArg==null || nextArg.startsWith("-") ) { + options.put(optName, ""); + } else { + options.put(optName, args.pop()); + } + } + } + } + + private final Deque asDeque(String[] args) { + Deque result = new ArrayDeque<>(); + // Split --opt=value into separate args on the Deque + for ( var arg: args ) { + if ( arg.startsWith("--") ) { + // Allow for --opt=val, adding option and value as separate args + var elts = arg.split("=", 2); + result.add(elts[0]); + if ( elts.length==2 ) { result.add(elts[1]); } + } else if ( arg.startsWith("-") ) { + result.add(arg.substring(0,2)); + if ( arg.length()>2 ) { + var val = arg.substring(2); + if ( val.startsWith("=") ) { val = val.substring(1); } + result.add(val); + } + } else { + result.add(arg); + } + } + return result; + } + } + + @Reflectable @NoArgsConstructor + static final class OptionParametersMixinPreprocessor implements IParameterPreprocessor { + @Override + public final boolean preprocess(Stack args, CommandSpec commandSpec, ArgSpec argSpec, Map info) { + var mixin = commandSpec.mixins().values().stream() + .map(CommandSpec::userObject) + .filter(o->o instanceof OptionParametersMixin) + .map(OptionParametersMixin.class::cast) + .findFirst().get(); + commandSpec.addUnmatchedArgsBinding(UnmatchedArgsBinding.forStringArrayConsumer(mixin::setUnmatchedArgs)); + return false; + } + } } diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/cli/cmd/AbstractDataExtractCreateCommand.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/cli/cmd/AbstractDataExtractCreateCommand.java new file mode 100644 index 0000000000..0a7e4432b3 --- /dev/null +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/cli/cmd/AbstractDataExtractCreateCommand.java @@ -0,0 +1,57 @@ +/******************************************************************************* + * Copyright 2021, 2023 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + *******************************************************************************/ +package com.fortify.cli.common.data_extract.cli.cmd; + +import java.util.List; + +import com.fortify.cli.common.cli.cmd.AbstractRunnableCommand; +import com.fortify.cli.common.cli.mixin.CommonOptionMixins.OptionParametersMixin; +import com.fortify.cli.common.data_extract.helper.DataExtractTemplateRunner; +import com.fortify.cli.common.data_extract.helper.DataExtractTemplateHelper; +import com.fortify.cli.common.progress.cli.mixin.ProgressWriterFactoryMixin; +import com.fortify.cli.common.util.DisableTest; +import com.fortify.cli.common.util.DisableTest.TestType; + +import picocli.CommandLine.Mixin; +import picocli.CommandLine.Option; + +public abstract class AbstractDataExtractCreateCommand extends AbstractRunnableCommand implements Runnable { + @Option(names={"-t", "--template"}, required=true) private String template; + @DisableTest({TestType.MULTI_OPT_SPLIT, TestType.MULTI_OPT_PLURAL_NAME, TestType.OPT_LONG_NAME}) + @Option(names="--", paramLabel="", descriptionKey="fcli.data-extract.create.template-parameter") + private List dummyForSynopsis; + @Mixin private OptionParametersMixin templateParameters; + @Mixin private ProgressWriterFactoryMixin progressWriterFactory; + + @Override + public final void run() { + initMixins(); + try ( var progressWriter = progressWriterFactory.create() ) { + progressWriter.writeProgress("Loading template %s", template); + var templateDescriptor = DataExtractTemplateHelper.load(getType(), template); + progressWriter.writeProgress("Executing template %s", template); + try ( var templateExecutor = DataExtractTemplateRunner.builder() + .template(templateDescriptor) + .inputParameters(templateParameters.getOptions()) + .progressWriter(progressWriter).build() ) + { + configure(templateExecutor); + templateParameters.validate(templateExecutor::configureOptionParameterValidator); + templateExecutor.execute(); + } + } + } + + protected abstract String getType(); + protected abstract void configure(DataExtractTemplateRunner templateExecutor); +} diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/cli/cmd/AbstractDataExtractListTemplatesCommand.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/cli/cmd/AbstractDataExtractListTemplatesCommand.java new file mode 100644 index 0000000000..0cdb7e776e --- /dev/null +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/cli/cmd/AbstractDataExtractListTemplatesCommand.java @@ -0,0 +1,32 @@ +/******************************************************************************* + * Copyright 2021, 2023 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + *******************************************************************************/ +package com.fortify.cli.common.data_extract.cli.cmd; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fortify.cli.common.output.cli.cmd.AbstractOutputCommand; +import com.fortify.cli.common.output.cli.cmd.IJsonNodeSupplier; + +public abstract class AbstractDataExtractListTemplatesCommand extends AbstractOutputCommand implements IJsonNodeSupplier { + @Override + public final JsonNode getJsonNode() { + // TODO Auto-generated method stub + return null; + } + @Override + public final boolean isSingular() { + return false; + } + protected abstract String getType(); + + +} diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/helper/DataExtractSpelFunctions.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/helper/DataExtractSpelFunctions.java new file mode 100644 index 0000000000..503caffaf4 --- /dev/null +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/helper/DataExtractSpelFunctions.java @@ -0,0 +1,83 @@ +/******************************************************************************* + * Copyright 2021, 2023 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + *******************************************************************************/ +package com.fortify.cli.common.data_extract.helper; + +import java.util.List; + +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.safety.Safelist; + +import com.formkiq.graalvm.annotations.Reflectable; +import com.fortify.cli.common.util.StringUtils; + +import lombok.NoArgsConstructor; + +@Reflectable @NoArgsConstructor +public class DataExtractSpelFunctions { + public static final String join(String separator, List elts) { + switch (separator) { + case "\\n": separator="\n"; break; + case "\\t": separator="\t"; break; + } + return String.join(separator, elts.stream().map(Object::toString).toList()); + } + + /** + * Convenience method to throw an exception if an expression evaluates to false + * @param throwError true if error should be thrown, false otherwise + * @param msg Message for exception to be thrown + * @return true if throwError is false + * @throws IllegalStateException with the given message if throwError is true + */ + public static final boolean check(boolean throwError, String msg) { + if ( throwError ) { + throw new IllegalStateException(msg); + } else { + return true; + } + } + + /** + * Abbreviate the given text to the given maximum width + * @param text to abbreviate + * @param maxWidth Maximum width + * @return Abbreviated text + */ + public static final String abbreviate(String text, int maxWidth) { + return StringUtils.abbreviate(text, maxWidth); + } + + /** + * @param html to be converted to plain text + * @return Formatted plain-text string for the given HTML contents + */ + public static final String htmlToText(String html) { + if( html==null ) { return null; } + Document document = Jsoup.parse(html); + document.outputSettings(new Document.OutputSettings().prettyPrint(false));//makes html() preserve linebreaks and spacing + document.select("br").append("\\n"); + document.select("p").prepend("\\n\\n"); + String s = document.html().replaceAll("\\\\n", "\n"); + return Jsoup.clean(s, "", Safelist.none(), new Document.OutputSettings().prettyPrint(false)); + } + + /** + * @param html to be converted to plain text + * @return Single line of plain text for the given HTML contents + */ + public static final String htmlToSingleLineText(String html) { + if( html==null ) { return null; } + return Jsoup.clean(html, "", Safelist.none()); + } +} diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/helper/DataExtractTemplateDescriptor.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/helper/DataExtractTemplateDescriptor.java new file mode 100644 index 0000000000..5c76f8b5a2 --- /dev/null +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/helper/DataExtractTemplateDescriptor.java @@ -0,0 +1,332 @@ +/** + * Copyright 2023 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors (“Open Text”) are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.common.data_extract.helper; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.springframework.expression.ParseException; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.TextNode; +import com.fasterxml.jackson.databind.node.ValueNode; +import com.formkiq.graalvm.annotations.Reflectable; +import com.fortify.cli.common.json.JsonHelper.AbstractJsonNodeWalker; +import com.fortify.cli.common.spring.expression.SpelHelper; +import com.fortify.cli.common.spring.expression.wrapper.SimpleExpression; +import com.fortify.cli.common.spring.expression.wrapper.TemplateExpression; +import com.fortify.cli.common.util.StringUtils; + +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * This class describes a data extraction template deserialized from a + * data extraction template YAML file, containing elements describing: + *
    + *
  • Template metadata like name and description
    • + *
  • Template parameters
    • + *
  • Steps to be executed, like executing REST requests and writing output
    • + *
  • (Partial) output formats
    • + *
+ * + * @author Ruud Senden + */ +@Reflectable @NoArgsConstructor +@Data +public class DataExtractTemplateDescriptor { + /** Template name, set in {@link #postLoad(String)} method */ + private String name; + /** Template description */ + private String description; + /** Template parameters */ + private List parameters; + /** Template steps, evaluated in the order as defined in the YAML file */ + private List steps; + /** Partial outputs */ + private Map> partialOutputs; + /** Full outputs */ + private List outputs; + /** Output writers */ + private List outputWriters; + + /** + * This method is invoked by DataExtractTemplateHelper after deserializing + * an instance of this class from a YAML file. It performs some additional + * initialization and validation. + */ + final void postLoad(String name) { + this.name = name; + checkNotNull("template steps", steps, this); + checkNotNull("template outputs", outputs, this); + if ( parameters!=null ) { parameters.forEach(DataExtractTemplateParameterDescriptor::postLoad); } + steps.forEach(DataExtractTemplateStepDescriptor::postLoad); + if ( partialOutputs!=null ) { + partialOutputs.entrySet().forEach(e->e.getValue().forEach(DataExtractTemplateOutputDescriptor::postLoad)); + } + outputs.forEach(DataExtractTemplateOutputDescriptor::postLoad); + outputWriters.forEach(w->w.postLoad(outputs)); + } + + /** + * Utility method for checking whether the given value is not blank, throwing an + * exception otherwise. + * @param property Descriptive name of the YAML property being checked + * @param value Value to be checked for not being blank + * @param entity The object containing the property to be checked + */ + private static final void checkNotBlank(String property, String value, Object entity) { + if ( StringUtils.isBlank(value) ) { + throw new TemplateValidationException(String.format("Template %s property must be specified", property), entity); + } + } + + /** + * Utility method for checking whether the given value is not null, throwing an + * exception otherwise. + * @param property Descriptive name of the YAML property being checked + * @param value Value to be checked for not being null + * @param entity The object containing the property to be checked + */ + private static final void checkNotNull(String property, Object value, Object entity) { + if ( value==null ) { + throw new TemplateValidationException(String.format("Template %s property must be specified", property), entity); + } + } + + /** + * This class describes a template parameter. + */ + @Reflectable @NoArgsConstructor + @Data + public static final class DataExtractTemplateParameterDescriptor { + /** Required parameter name */ + private String name; + /** Required parameter description */ + private String description; + /** Optional parameter type */ + private String type; + /** Optional type parameters*/ + private Map typeParameters; + /** Optional template expression defining the default parameter value if not provided by user */ + private TemplateExpression defaultValue; + /** Boolean indicating whether this parameter is required */ + private boolean required = true; + + public final void postLoad() { + checkNotBlank("parameter name", getName(), this); + checkNotNull("parameter description", getDescription(), this); + } + } + + /** + * This class describes a single template step element, which may contain + * requests, progress message, and/or write instructions. This class is + * used for both top-level step elements, and step elements in forEach elements. + * Potentially, later versions may add support for other steps, like static + * values or arrays, allowing for example iteration over a static array or + * array retrieved from a template parameter. + * + * @author Ruud Senden + */ + @Reflectable @NoArgsConstructor + @Data + public static final class DataExtractTemplateStepDescriptor { + /** Optional requests for this step element */ + private List requests; + /** Optional progress message template expression for this step element */ + private TemplateExpression progress; + + /** + * This method is invoked by the parent element (which may either be another + * step element, or the top-level {@link DataExtractTemplateDescriptor} instance). + * It invokes the postLoad() method on each requests and write descriptor. + */ + public final void postLoad() { + if ( requests!=null ) { requests.forEach(DataExtractTemplateStepRequestDescriptor::postLoad); } + } + } + + public static interface IDataExtractTemplateForEachSupplier { + DataExtractTemplateStepForEachDescriptor getForEach(); + } + + /** + * This class describes a forEach element, allowing iteration over the output of + * the parent element, like the response of a REST request or the contents of a + * template parameter. + */ + @Reflectable @NoArgsConstructor + @Data + public static final class DataExtractTemplateStepForEachDescriptor { + /** Required name for this step element */ + private String name; + /** Optional (sub-)property/expression from which to retrieve the values to be iterated through */ + private SimpleExpression property; + /** Steps to be repeated for each value */ + private List steps; + + /** + * This method is invoked by the {@link DataExtractTemplateStepDescriptor#postLoad()} + * method. It checks that required properties are set, then calls the postLoad() method for + * each sub-step. + */ + public final void postLoad() { + checkNotBlank("forEach name", getName(), this); + checkNotNull("forEach steps", steps, this); + steps.forEach(DataExtractTemplateStepDescriptor::postLoad); + } + } + + /** + * This class describes a REST request. + */ + @Reflectable @NoArgsConstructor + @Data + public static final class DataExtractTemplateStepRequestDescriptor implements IDataExtractTemplateForEachSupplier { + /** Required name for this step element */ + private String name; + /** Required template expression defining the request URI from which to get the data */ + private TemplateExpression uri; + /** Define from which remote system the data should be retrieved. Required if multiple + * remote systems are available, optional if there's only a single remote system. */ + private String from; + /** Map defining (non-encoded) request query parameters; parameter values are defined as template expressions */ + private Map query; + /** Type of request; either 'simple' or 'paged' for now */ + private DataExtractTemplateStepRequestType type = DataExtractTemplateStepRequestType.simple; + /** Optional if-expression, executing this request only if condition evaluates to true */ + @JsonProperty("if") private SimpleExpression _if; + /** Optional progress messages for various stages of request processing */ + private DataExtractTemplateStepRequestPagingProgress pagingProgress; + /** Optional forEach block, listing steps to be repeated for each entry produced by this step element */ + private DataExtractTemplateStepForEachDescriptor forEach; + + /** + * This method is invoked by {@link DataExtractTemplateStepDescriptor#postLoad()} + * method. It checks that required properties are set. + */ + protected final void postLoad() { + checkNotBlank("request name", getName(), this); + checkNotNull("request uri", uri, this); + if ( pagingProgress!=null ) { + type = DataExtractTemplateStepRequestType.paged; + } + if ( forEach!=null ) { + forEach.postLoad(); + } + } + + public static enum DataExtractTemplateStepRequestType { + simple, paged + } + + @Data + public static final class DataExtractTemplateStepRequestPagingProgress { + private TemplateExpression prePageLoad; + private TemplateExpression postPageLoad; + private TemplateExpression postPageProcess; + } + } + + /** + * This class describes an output, which can be either a top-level output + * or partial output. + */ + @Reflectable @NoArgsConstructor + @Data + public static final class DataExtractTemplateOutputDescriptor { + /** Required name for this output */ + private String name; + /** Output contents in JSON format, where each text node is assumed to be a template expression */ + private JsonNode contents; + /** Cached mapping from text node property path to corresponding TemplateExpression instance */ + private final Map valueExpressions = new LinkedHashMap<>(); + + /** + * This method checks whether required name and contents are not blank or null, then + * walks the given contents to parse each text node as a {@link TemplateExpression}, + * caching the resulting {@link TemplateExpression} instance in the {@link #valueExpressions} + * map, throwing an exception if the text node cannot be parsed as a {@link TemplateExpression}. + */ + public final void postLoad() { + checkNotBlank("(partial) output name", name, this); + checkNotNull("(partial) output contents", contents, this); + new ContentsWalker().walk(contents); + } + + private final class ContentsWalker extends AbstractJsonNodeWalker { + @Override + protected Void getResult() { return null; } + @Override + protected void walkValue(Void state, String path, JsonNode parent, ValueNode node) { + if ( node instanceof TextNode ) { + var expr = node.asText(); + try { + valueExpressions.put(path, SpelHelper.parseTemplateExpression(expr)); + } catch (ParseException e) { + throw new TemplateValidationException(String.format("Error parsing template expression '%s'", expr), DataExtractTemplateOutputDescriptor.this, e); + } + } + super.walkValue(state, path, parent, node); + } + } + } + + /** + * This class describes an output writer. + */ + @Reflectable @NoArgsConstructor + @Data + public static final class DataExtractTemplateOutputWriterDescriptor { + /** Required output name for this step element */ + private String outputName; + /** Required template expression defining where to write the data, either stdout, stderr or filename */ + private TemplateExpression to; + + public void postLoad(List outputDescriptors) { + checkNotBlank("output writer outputName", outputName, this); + outputDescriptors.stream().filter(d->outputName.equals(d.getName())).findFirst() + .orElseThrow(()->new TemplateValidationException(String.format("No output with name %s available", outputName), this)); + } + } + + /** + * Exception class used for template validation errors. + */ + public static final class TemplateValidationException extends IllegalStateException { + private static final long serialVersionUID = 1L; + + public TemplateValidationException(String message, Throwable cause) { + super(message, cause); + } + + public TemplateValidationException(String message, Object templateElement, Throwable cause) { + this(getMessageWithEntity(message, templateElement), cause); + } + + public TemplateValidationException(String message) { + super(message); + } + + public TemplateValidationException(String message, Object templateElement) { + this(getMessageWithEntity(message, templateElement)); + } + + private static final String getMessageWithEntity(String message, Object templateElement) { + return String.format("%s (entity: %s)", message, templateElement.toString()); + } + } +} diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/helper/DataExtractTemplateHelper.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/helper/DataExtractTemplateHelper.java new file mode 100644 index 0000000000..450bc007c5 --- /dev/null +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/helper/DataExtractTemplateHelper.java @@ -0,0 +1,86 @@ +/** + * Copyright 2023 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors (“Open Text”) are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.common.data_extract.helper; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; +import java.util.stream.Stream; + +import org.springframework.core.io.Resource; +import org.springframework.core.io.support.PathMatchingResourcePatternResolver; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fortify.cli.common.util.FileUtils; + +import lombok.SneakyThrows; + +public class DataExtractTemplateHelper { + private static final ObjectMapper yamlObjectMapper = new ObjectMapper(new YAMLFactory()); + private DataExtractTemplateHelper() {} + + public static final DataExtractTemplateDescriptor load(String type, String name) { + var resourceFile = getTemplateResourceFile(type, name); + try ( var is = FileUtils.getResourceInputStream(resourceFile) ) { + return load(resourceFile, is); + } catch ( IOException e ) { + throw new RuntimeException("Error loading template "+name, e); + } + } + + public static final Stream list(String type) { + return Stream.of(getTemplateResources(type)) + .map(DataExtractTemplateHelper::load); + } + + private static final String getTemplatesResourceDir(String type) { + return String.format("com/fortify/cli/%s/data_extract/templates", type.toLowerCase().replace('-', '_')); + } + + private static final String getTemplateResourceFile(String type, String name) { + return String.format("%s/%s.yaml", getTemplatesResourceDir(type), name); + } + + @SneakyThrows + private static final Resource[] getTemplateResources(String type) { + var matchPattern = String.format("%s/*.yaml", getTemplatesResourceDir(type)); + return new PathMatchingResourcePatternResolver().getResources(matchPattern); + } + + private static final DataExtractTemplateDescriptor load(Resource resource) { + try ( var is = resource.getInputStream() ) { + return load(resource.getFilename(), is); + } catch (IOException e) { + throw new RuntimeException("Error loading template "+resource.getFilename(), e); + } + } + + private static final DataExtractTemplateDescriptor load(String fileName, InputStream is) { + if ( is==null ) { + // TODO Use more descriptive exception message + throw new IllegalStateException("Can't read "+fileName); + } + try { + var result = yamlObjectMapper.readValue(is, DataExtractTemplateDescriptor.class); + result.postLoad(getTemplateName(fileName)); + return result; + } catch (IOException e) { + throw new RuntimeException("Error loading template "+fileName, e); + } + } + + private static final String getTemplateName(String fileName) { + return Path.of(fileName).getFileName().toString().replace(".yaml", ""); + } +} diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/helper/DataExtractTemplateRunner.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/helper/DataExtractTemplateRunner.java new file mode 100644 index 0000000000..1977ffb216 --- /dev/null +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/helper/DataExtractTemplateRunner.java @@ -0,0 +1,551 @@ +/** + * Copyright 2023 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors (“Open Text”) are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.common.data_extract.helper; + +import java.io.File; +import java.io.IOException; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.function.Function; + +import org.springframework.expression.spel.SpelEvaluationException; +import org.springframework.expression.spel.support.SimpleEvaluationContext; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.BooleanNode; +import com.fasterxml.jackson.databind.node.DoubleNode; +import com.fasterxml.jackson.databind.node.FloatNode; +import com.fasterxml.jackson.databind.node.IntNode; +import com.fasterxml.jackson.databind.node.LongNode; +import com.fasterxml.jackson.databind.node.NullNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +import com.fasterxml.jackson.databind.node.ValueNode; +import com.fortify.cli.common.cli.mixin.CommonOptionMixins.OptionParametersMixin.OptionParameterValidator; +import com.fortify.cli.common.data_extract.helper.DataExtractTemplateDescriptor.DataExtractTemplateOutputDescriptor; +import com.fortify.cli.common.data_extract.helper.DataExtractTemplateDescriptor.DataExtractTemplateOutputWriterDescriptor; +import com.fortify.cli.common.data_extract.helper.DataExtractTemplateDescriptor.DataExtractTemplateParameterDescriptor; +import com.fortify.cli.common.data_extract.helper.DataExtractTemplateDescriptor.DataExtractTemplateStepDescriptor; +import com.fortify.cli.common.data_extract.helper.DataExtractTemplateDescriptor.DataExtractTemplateStepForEachDescriptor; +import com.fortify.cli.common.data_extract.helper.DataExtractTemplateDescriptor.DataExtractTemplateStepRequestDescriptor; +import com.fortify.cli.common.data_extract.helper.DataExtractTemplateDescriptor.DataExtractTemplateStepRequestDescriptor.DataExtractTemplateStepRequestPagingProgress; +import com.fortify.cli.common.data_extract.helper.DataExtractTemplateDescriptor.DataExtractTemplateStepRequestDescriptor.DataExtractTemplateStepRequestType; +import com.fortify.cli.common.data_extract.helper.DataExtractTemplateDescriptor.TemplateValidationException; +import com.fortify.cli.common.data_extract.helper.DataExtractTemplateRunner.IDataExtractRequestHelper.DataExtractRequestDescriptor; +import com.fortify.cli.common.json.JsonHelper; +import com.fortify.cli.common.json.JsonHelper.JsonNodeDeepCopyWalker; +import com.fortify.cli.common.output.product.IProductHelper; +import com.fortify.cli.common.output.transform.IInputTransformer; +import com.fortify.cli.common.progress.helper.IProgressWriterI18n; +import com.fortify.cli.common.rest.paging.INextPageUrlProducer; +import com.fortify.cli.common.rest.paging.INextPageUrlProducerSupplier; +import com.fortify.cli.common.rest.paging.PagingHelper; +import com.fortify.cli.common.rest.unirest.IUnirestInstanceSupplier; +import com.fortify.cli.common.spring.expression.IConfigurableSpelEvaluator; +import com.fortify.cli.common.spring.expression.ISpelEvaluator; +import com.fortify.cli.common.spring.expression.SpelEvaluator; +import com.fortify.cli.common.spring.expression.SpelHelper; +import com.fortify.cli.common.spring.expression.wrapper.TemplateExpression; +import com.fortify.cli.common.util.JavaHelper; +import com.fortify.cli.common.util.StringUtils; + +import kong.unirest.GetRequest; +import kong.unirest.UnirestInstance; +import lombok.Builder; +import lombok.Data; +import lombok.RequiredArgsConstructor; + +@Builder +public class DataExtractTemplateRunner implements AutoCloseable { + /** Jackson {@link ObjectMapper} used for various JSON-related operations */ + private static final ObjectMapper objectMapper = JsonHelper.getObjectMapper(); + /** Progress writer, provided through builder method */ + private final IProgressWriterI18n progressWriter; + /** Data extract template, provided through builder method */ + private final DataExtractTemplateDescriptor template; + /** Template input parameters, provided through builder method */ + private final Map inputParameters; + /** ObjectNode holding parameter values as generated by DataExtractTemplateParameterProcessor */ + private final ObjectNode parameters = objectMapper.createObjectNode(); + /** ObjectNode holding partial outputs as generated by DataExtractTemplateStepsProcessor */ + private final ObjectNode partialOutputs = objectMapper.createObjectNode(); + /** ObjectNode holding final outputs as generated by DataExtractTemplateOutputProcessor */ + private final ObjectNode outputs = objectMapper.createObjectNode(); + /** SpEL evaluator configured with {@link DataExtractSpelFunctions} and variables for + * parameters, partialOutputs and outputs as defined above */ + private final IConfigurableSpelEvaluator spelEvaluator = SpelEvaluator.JSON_GENERIC.copy().configure(this::configureSpelEvaluator); + /** Parameter converters as generated by {@link #createDefaultParameterConverters()} amended by + * custom converters as added through the {@link #addParameterConverter(String, BiFunction)} and + * {@link #addParameterConverter(String, Function)} methods. */ + private final Map> parameterConverters = createDefaultParameterConverters(); + /** Request helpers as configured through the {@link #addRequestHelper(String, IDataExtractRequestHelper)} method */ + private final Map requestHelpers = new HashMap<>(); + + public final void execute() { + new DataExtractTemplateParameterProcessor().processParameters(); + var data = new DataExtractTemplateStepsProcessor().processSteps(); + new DataExtractTemplateOutputProcessor(data).processOutputs(); + new DataExtractTemplateOutputWriterProcessor(data).processOutputWriters(); + } + + public final void close() { + requestHelpers.values().forEach(IDataExtractRequestHelper::close); + } + + public final void configureOptionParameterValidator(OptionParameterValidator validator) { + validator.setUnknownOptionFormat("Unknown template parameter: %s"); + validator.setValidationExceptionFormat("Invalid template parameter:\n %s\nSupported template parameters:\n%s\n"); + template.getParameters().forEach(p->configureOptionParameterValidator(validator, p)); + } + + private final void configureOptionParameterValidator(OptionParameterValidator validator, DataExtractTemplateParameterDescriptor parameter) { + validator.addSupportedOption(parameter.getName(), parameter.getDescription()); + if ( parameter.isRequired() ) { + validator.requiredOption(parameter.getName(), "Missing template parameter: %s"); + } + } + + private final void configureSpelEvaluator(SimpleEvaluationContext context) { + SpelHelper.registerFunctions(context, DataExtractSpelFunctions.class); + context.setVariable("parameters", parameters); + context.setVariable("partialOutputs", partialOutputs); + context.setVariable("outputs", outputs); + } + + public final DataExtractTemplateRunner addParameterConverter(String type, BiFunction converter) { + parameterConverters.put(type, converter); + return this; + } + public final DataExtractTemplateRunner addParameterConverter(String type, Function converter) { + parameterConverters.put(type, (v,a)->converter.apply(v)); + return this; + } + public final DataExtractTemplateRunner addRequestHelper(String name, IDataExtractRequestHelper requestHelper) { + requestHelpers.put(name, requestHelper); + return this; + } + + private IDataExtractRequestHelper getRequestHelper(String name) { + if ( StringUtils.isBlank(name) ) { + if ( requestHelpers.size()==1 ) { + return requestHelpers.values().iterator().next(); + } else { + throw new IllegalStateException(String.format("Required 'from:' property (allowed values: %s) missing", requestHelpers.keySet())); + } + } + var result = requestHelpers.get(name); + if ( result==null ) { + throw new IllegalStateException(String.format("Invalid 'from: %s', allowed values: %s", name, requestHelpers.keySet())); + } + return result; + } + + private static final JsonNode getOutput(ISpelEvaluator spelEvaluator, DataExtractTemplateOutputDescriptor outputDescriptor, ObjectNode data) { + var outputRawContents = outputDescriptor.getContents(); + var output = new JsonNodeOutputWalker(spelEvaluator, outputDescriptor, data).walk(outputRawContents); + return output; + } + + private final class DataExtractTemplateParameterProcessor { + private final void processParameters() { + template.getParameters().forEach(this::processParameter); + } + + private final void processParameter(DataExtractTemplateParameterDescriptor parameter) { + var name = parameter.getName(); + var value = inputParameters.get(name); + if ( value==null ) { + var defaultValueExpression = parameter.getDefaultValue(); + value = defaultValueExpression==null + ? null + : spelEvaluator.evaluate(defaultValueExpression, parameters, String.class); + } + parameters.set(name, convertParameterValue(value, parameter)); + } + + private JsonNode convertParameterValue(String value, DataExtractTemplateParameterDescriptor parameter) { + var name = parameter.getName(); + var type = StringUtils.isBlank(parameter.getType()) ? "string" : parameter.getType(); + var paramConverter = parameterConverters.get(type); + if ( paramConverter==null ) { + throw new TemplateValidationException(String.format("Unknown parameter type %s for parameter %s", type, name)); + } else { + var args = ParameterTypeConverterArgs.builder() + .progressWriter(progressWriter) + .spelEvaluator(spelEvaluator) + .template(template) + .parameter(parameter) + .parameters(parameters) + .build(); + var result = paramConverter.apply(value, args); + return result==null ? NullNode.instance : result; + } + } + } + + @RequiredArgsConstructor + private final class DataExtractTemplateStepsProcessor { + private final ObjectNode data; + + public DataExtractTemplateStepsProcessor() { + this(objectMapper.createObjectNode()); + } + + private final ObjectNode processSteps() { + template.getSteps().forEach(this::processStep); + return data; + } + + private final void processStep(DataExtractTemplateStepDescriptor step) { + processNonRequestSteps(step); + processRequests(step); + } + + private void processNonRequestSteps(DataExtractTemplateStepDescriptor step) { + processProgress(step); + } + + private void processProgress(DataExtractTemplateStepDescriptor step) { + var progress = step.getProgress(); + if ( progress!=null ) { + progressWriter.writeProgress(spelEvaluator.evaluate(progress, data, String.class)); + } + } + + private void processRequests(DataExtractTemplateStepDescriptor step) { + var requests = step.getRequests(); + if ( requests!=null ) { + var requestsExecutor = new DataExtractTemplateStepRequestsProcessor(); + requestsExecutor.addRequests(requests, this::processResponse, data); + requestsExecutor.executeRequests(); + } + } + + private final void processStepForEach(String parentName, DataExtractTemplateStepForEachDescriptor forEach) { + if ( forEach!=null ) { + var property = forEach.getProperty()==null ? SpelHelper.parseSimpleExpression(parentName) : forEach.getProperty(); + var input = spelEvaluator.evaluate(property, data, JsonNode.class); + if ( input!=null ) { + var childName = forEach.getName(); + if ( input instanceof ArrayNode ) { + processStepsForEach(childName, (ArrayNode)input, forEach.getSteps()); + } else { + throw new TemplateValidationException("forEach not supported on node type "+input.getNodeType()); + } + } + } + } + + private final void processStepsForEach(String name, ArrayNode source, List steps) { + steps.forEach(s->processSourceForEach(name, source, s)); + } + + private final void processSourceForEach(String name, ArrayNode source, DataExtractTemplateStepDescriptor stepDescriptor) { + var requestExecutor = new DataExtractTemplateStepRequestsProcessor(); + for ( int i = 0 ; i < source.size(); i++ ) { + var currentObject = source.get(i); + var newData = data.deepCopy(); + newData.set(name, currentObject); + var processor = new DataExtractTemplateStepsProcessor(newData); + processor.processNonRequestSteps(stepDescriptor); + requestExecutor.addRequests(stepDescriptor.getRequests(), processor::processResponse, newData); + } + requestExecutor.executeRequests(); + } + + private final void processResponse(DataExtractTemplateStepRequestDescriptor requestDescriptor, JsonNode rawBody) { + var name = requestDescriptor.getName(); + var body = getRequestHelper(requestDescriptor.getFrom()).transformInput(rawBody); + data.set(name+"_raw", rawBody); + data.set(name, body); + processStepPartialOutputs(name); + processStepForEach(name, requestDescriptor.getForEach()); + } + + private void processStepPartialOutputs(String name) { + var partialOutputDefinitions = template.getPartialOutputs(); + if ( partialOutputDefinitions!=null ) { + var partialOutputs = partialOutputDefinitions.get(name); + if ( partialOutputs!=null ) { + partialOutputs.forEach(d->processPartialOutput(name, d)); + } + } + } + + private void processPartialOutput(String stepName, DataExtractTemplateOutputDescriptor outputDescriptor) { + var partialOutputName = outputDescriptor.getName(); + var partialOutput = getOutput(spelEvaluator, outputDescriptor, data); + var partialOutputArray = (ArrayNode)partialOutputs.get(partialOutputName); + if ( partialOutputArray==null ) { + partialOutputArray = objectMapper.createArrayNode(); + partialOutputs.set(partialOutputName, partialOutputArray); + } + partialOutputArray.add(partialOutput); + } + } + + private final class DataExtractTemplateStepRequestsProcessor { + private final Map> simpleRequests = new LinkedHashMap<>(); + private final Map> pagedRequests = new LinkedHashMap<>(); + + private final void addRequests(List requestDescriptors, BiConsumer responseConsumer, ObjectNode data) { + if ( requestDescriptors!=null ) { + requestDescriptors.forEach(r->addRequest(r, responseConsumer, data)); + } + } + + private final void addRequest(DataExtractTemplateStepRequestDescriptor requestDescriptor, BiConsumer responseConsumer, ObjectNode data) { + var _if = requestDescriptor.get_if(); + if ( _if==null || spelEvaluator.evaluate(_if, data, Boolean.class) ) { + var uri = spelEvaluator.evaluate(requestDescriptor.getUri(), data, String.class); + var query = evaluateQuery(requestDescriptor.getQuery(), data); + var requestData = new IDataExtractRequestHelper.DataExtractRequestDescriptor(uri, query, r->responseConsumer.accept(requestDescriptor, r)); + addPagingProgress(requestData, requestDescriptor.getPagingProgress(), data); + if ( requestDescriptor.getType()==DataExtractTemplateStepRequestType.paged ) { + pagedRequests.computeIfAbsent(requestDescriptor.getFrom(), s->new ArrayList()).add(requestData); + } else { + simpleRequests.computeIfAbsent(requestDescriptor.getFrom(), s->new ArrayList()).add(requestData); + } + } + } + + private void addPagingProgress(DataExtractRequestDescriptor requestData, DataExtractTemplateStepRequestPagingProgress pagingProgress, ObjectNode data) { + if ( pagingProgress!=null ) { + addPagingProgress(pagingProgress.getPrePageLoad(), requestData::setPrePageLoad, data); + addPagingProgress(pagingProgress.getPostPageLoad(), requestData::setPostPageLoad, data); + addPagingProgress(pagingProgress.getPostPageProcess(), requestData::setPostPageProcess, data); + } + } + + private void addPagingProgress(TemplateExpression expr, Consumer consumer, ObjectNode data) { + if ( expr!=null ) { + consumer.accept(()->progressWriter.writeProgress(spelEvaluator.evaluate(expr, data, String.class))); + } + } + + private Map evaluateQuery(Map queryExpressions, ObjectNode data) { + Map result = new LinkedHashMap<>(); + if ( queryExpressions!=null ) { + queryExpressions.entrySet().forEach(e->result.put(e.getKey(), spelEvaluator.evaluate(e.getValue(), data, String.class))); + } + return result; + } + + private final void executeRequests() { + simpleRequests.entrySet().forEach(e->executeRequest(e.getKey(), e.getValue(), false)); + pagedRequests.entrySet().forEach(e->executeRequest(e.getKey(), e.getValue(), true)); + } + + private void executeRequest(String from, List requests, boolean isPaged) { + var requestHelper = getRequestHelper(from); + if ( isPaged ) { + requests.forEach(r->requestHelper.executePagedRequest(r)); + } else { + requestHelper.executeSimpleRequests(requests); + } + } + } + + @RequiredArgsConstructor + private final class DataExtractTemplateOutputProcessor { + private final ObjectNode data; + + private final void processOutputs() { + template.getOutputs().forEach(this::processOutput); + } + + private final void processOutput(DataExtractTemplateOutputDescriptor outputDescriptor) { + outputs.set(outputDescriptor.getName(), getOutput(spelEvaluator, outputDescriptor, data)); + } + } + + @RequiredArgsConstructor + private final class DataExtractTemplateOutputWriterProcessor { + private final ObjectNode data; + + private final void processOutputWriters() { + template.getOutputWriters().forEach(this::processOutputWriter); + } + + private final void processOutputWriter(DataExtractTemplateOutputWriterDescriptor outputWriterDescriptor) { + var outputName = outputWriterDescriptor.getOutputName(); + var to = spelEvaluator.evaluate(outputWriterDescriptor.getTo(), data, String.class); + var output = outputs.get(outputName); + try { + switch (to.toLowerCase()) { + case "stdout": write(System.out, output); break; + case "stderr": write(System.err, output); break; + default: write(new File(to), output); + } + } catch (IOException e) { + throw new RuntimeException("Error writing template output to "+to); + } + } + + private void write(File file, JsonNode output) throws IOException { + try ( var out = new PrintStream(file) ) { + write(out, output); + } + } + + private void write(PrintStream out, JsonNode output) throws IOException { + if ( output instanceof TextNode ) { + out.print(output.asText()); + } else { + out.print(output.toPrettyString()); + } + } + } + + public static interface IDataExtractRequestHelper extends AutoCloseable { + public JsonNode transformInput(JsonNode input); + public void executePagedRequest(DataExtractRequestDescriptor requestDescriptor); + public void executeSimpleRequests(List requestDescriptor); + public void close(); + + @Data + public static final class DataExtractRequestDescriptor { + private final String uri; + private final Map queryParams; + private final Consumer responseConsumer; + private Runnable prePageLoad; + private Runnable postPageLoad; + private Runnable postPageProcess; + + public void prePageLoad() { + run(prePageLoad); + } + public void postPageLoad() { + run(postPageLoad); + } + public void postPageProcess() { + run(postPageProcess); + } + private void run(Runnable runnable) { + if ( runnable!=null ) { runnable.run(); } + } + } + + @RequiredArgsConstructor + public static class BasicDataExtractRequestHelper implements IDataExtractRequestHelper { + private final IUnirestInstanceSupplier unirestInstanceSupplier; + private final IProductHelper productHelper; + private UnirestInstance unirestInstance; + public final UnirestInstance getUnirestInstance() { + if ( unirestInstance==null ) { + unirestInstance = unirestInstanceSupplier.getUnirestInstance(); + } + return unirestInstance; + } + + @Override + public JsonNode transformInput(JsonNode input) { + return JavaHelper.as(productHelper, IInputTransformer.class).orElse(i->i).transformInput(input); + } + @Override + public void executePagedRequest(DataExtractRequestDescriptor requestDescriptor) { + var unirest = getUnirestInstance(); + INextPageUrlProducer nextPageUrlProducer = (req, resp)->{ + var nextPageUrl = JavaHelper.as(productHelper, INextPageUrlProducerSupplier.class).get() + .getNextPageUrlProducer().getNextPageUrl(req, resp); + if ( nextPageUrl!=null ) { + requestDescriptor.prePageLoad(); + } + return nextPageUrl; + }; + GetRequest request = createRequest(unirest, requestDescriptor); + requestDescriptor.prePageLoad(); + PagingHelper.processPages(unirest, request, nextPageUrlProducer, r->{ + requestDescriptor.postPageLoad(); + requestDescriptor.getResponseConsumer().accept(r.getBody()); + requestDescriptor.postPageProcess(); + }); + } + @Override + public void executeSimpleRequests(List requestDescriptors) { + var unirest = getUnirestInstance(); + requestDescriptors.forEach(r->executeSimpleRequest(unirest, r)); + } + private void executeSimpleRequest(UnirestInstance unirest, DataExtractRequestDescriptor requestDescriptor) { + createRequest(unirest, requestDescriptor) + .asObject(JsonNode.class) + .ifSuccess(r->requestDescriptor.getResponseConsumer().accept(r.getBody())); + } + + private GetRequest createRequest(UnirestInstance unirest, DataExtractRequestDescriptor r) { + return unirest.get(r.getUri()) + .queryString(r.getQueryParams()); + } + + @Override + public void close() { + if ( unirestInstance!=null ) { + unirestInstance.close(); + } + } + } + } + + @Builder @Data + public static final class ParameterTypeConverterArgs { + private final IProgressWriterI18n progressWriter; + private final ISpelEvaluator spelEvaluator; + private final DataExtractTemplateDescriptor template; + private final DataExtractTemplateParameterDescriptor parameter; + private final ObjectNode parameters; + } + + private static final Map> createDefaultParameterConverters() { + Map> result = new HashMap<>(); + result.put("string", (v,a)->new TextNode(v)); + result.put("boolean", (v,a)->BooleanNode.valueOf(Boolean.parseBoolean(v))); + result.put("int", (v,a)->IntNode.valueOf(Integer.parseInt(v))); + result.put("long", (v,a)->LongNode.valueOf(Long.parseLong(v))); + result.put("double", (v,a)->DoubleNode.valueOf(Double.parseDouble(v))); + result.put("float", (v,a)->FloatNode.valueOf(Float.parseFloat(v))); + // TODO Add BigIntegerNode/DecimalNode/ShortNode support? + // TODO Add array support? + return result; + } + + @RequiredArgsConstructor + private static final class JsonNodeOutputWalker extends JsonNodeDeepCopyWalker { + private final ISpelEvaluator spelEvaluator; + private final DataExtractTemplateOutputDescriptor outputDescriptor; + private final ObjectNode data; + @Override + protected JsonNode copyValue(JsonNode state, String path, JsonNode parent, ValueNode node) { + if ( !(node instanceof TextNode) ) { + return super.copyValue(state, path, parent, node); + } else { + TemplateExpression expression = outputDescriptor.getValueExpressions().get(path); + if ( expression==null ) { throw new RuntimeException("No expression for "+path); } + try { + var rawResult = spelEvaluator.evaluate(expression, data, Object.class); + return objectMapper.valueToTree(rawResult); + } catch ( SpelEvaluationException e ) { + throw new RuntimeException("Error evaluating template expression "+expression.getExpressionString(), e); + } + } + } + } +} diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/json/AbstractJsonWalker.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/json/AbstractJsonWalker.java new file mode 100644 index 0000000000..cdc7dbd0b6 --- /dev/null +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/json/AbstractJsonWalker.java @@ -0,0 +1,75 @@ +/** + * Copyright 2023 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors (“Open Text”) are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.common.json; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ContainerNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.ValueNode; + +public abstract class AbstractJsonWalker { + public R walk(JsonNode node) { + walk("", null, node); + return getResult(); + } + protected abstract R getResult(); + + protected void walk(String path, JsonNode parent, JsonNode node) { + if ( !skipNode(path, parent, node) ) { + if ( node instanceof ContainerNode ) { + walkContainer(path, parent, (ContainerNode)node); + } else if ( node instanceof ValueNode ) { + walkValue(path, parent, (ValueNode)node); + } + } + } + + protected boolean skipNode(String path, JsonNode parent, JsonNode node) { + return false; + } + + protected void walkContainer(String path, JsonNode parent, ContainerNode node) { + if ( node instanceof ArrayNode ) { + walkArray(path, parent, (ArrayNode)node); + } else if ( node instanceof ObjectNode ) { + walkObject(path, parent, (ObjectNode)node); + } + } + + protected void walkObject(String path, JsonNode parent, ObjectNode node) { + node.fields().forEachRemaining(e->walkObjectProperty(appendPath(path, e.getKey()), node, e.getKey(), e.getValue())); + } + + protected void walkObjectProperty(String path, ObjectNode parent, String property, JsonNode value) { + walk(path, parent, value); + } + + protected void walkArray(String path, JsonNode parent, ArrayNode node) { + for ( int i = 0 ; i < node.size() ; i++ ) { + walkArrayElement(appendPath(path, i+""), node, i, node.get(i)); + } + } + + protected void walkArrayElement(String path, ArrayNode parent, int index, JsonNode value) { + walk(path, parent, value); + } + + protected void walkValue(String path, JsonNode parent, ValueNode node) { + + } + + protected final String appendPath(String parent, String entry) { + return String.format("%s[%s]", parent, entry); + } +} diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/json/JsonHelper.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/json/JsonHelper.java index 5495f20bc4..fa196743bd 100644 --- a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/json/JsonHelper.java +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/json/JsonHelper.java @@ -30,8 +30,11 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ContainerNode; +import com.fasterxml.jackson.databind.node.JsonNodeType; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; +import com.fasterxml.jackson.databind.node.ValueNode; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.fortify.cli.common.spring.expression.SpelEvaluator; import com.fortify.cli.common.util.StringUtils; @@ -149,5 +152,110 @@ public Function finisher() { public Set characteristics() { return EnumSet.of(Characteristics.UNORDERED); } - } + } + + public static abstract class AbstractJsonNodeWalker { + public final R walk(JsonNode node) { + if ( node!=null ) { + walk(null, "", null, node); + } + return getResult(); + } + protected abstract R getResult(); + + protected void walk(S state, String path, JsonNode parent, JsonNode node) { + if ( !skipNode(state, path, parent, node) ) { + if ( node instanceof ContainerNode ) { + walkContainer(state, path, parent, (ContainerNode)node); + } else if ( node instanceof ValueNode ) { + walkValue(state, path, parent, (ValueNode)node); + } + } + } + + protected boolean skipNode(S state, String path, JsonNode parent, JsonNode node) { + return false; + } + + protected void walkContainer(S state, String path, JsonNode parent, ContainerNode node) { + if ( node instanceof ArrayNode ) { + walkArray(state, path, parent, (ArrayNode)node); + } else if ( node instanceof ObjectNode ) { + walkObject(state, path, parent, (ObjectNode)node); + } + } + + protected void walkObject(S state, String path, JsonNode parent, ObjectNode node) { + node.fields().forEachRemaining(e->walkObjectProperty(state, appendPath(path, e.getKey()), node, e.getKey(), e.getValue())); + } + + protected void walkObjectProperty(S state, String path, ObjectNode parent, String property, JsonNode value) { + walk(state, path, parent, value); + } + + protected void walkArray(S state, String path, JsonNode parent, ArrayNode node) { + for ( int i = 0 ; i < node.size() ; i++ ) { + walkArrayElement(state, appendPath(path, i+""), node, i, node.get(i)); + } + } + + protected void walkArrayElement(S state, String path, ArrayNode parent, int index, JsonNode value) { + walk(state, path, parent, value); + } + + protected void walkValue(S state, String path, JsonNode parent, ValueNode node) {} + + protected final String appendPath(String parent, String entry) { + return String.format("%s[%s]", parent, entry); + } + } + + public static class JsonNodeDeepCopyWalker extends AbstractJsonNodeWalker { + @Getter JsonNode result; + @Override + protected void walkObject(JsonNode state, String path, JsonNode parent, ObjectNode node) { + if ( state==null ) { state = objectMapper.createObjectNode(); } + if ( result==null ) { result = state; } + super.walkObject(state, path, parent, node); + } + @Override + protected void walkObjectProperty(JsonNode state, String path, ObjectNode parent, String property, JsonNode value) { + if ( value instanceof ContainerNode ) { + var newState = createContainerNode(value.getNodeType()); + ((ObjectNode)state).set(property, newState); + super.walkObjectProperty(newState, path, parent, property, value); + } else { + ((ObjectNode)state).set(property, copyValue(state, path, parent, (ValueNode)value)); + } + } + @Override + protected void walkArray(JsonNode state, String path, JsonNode parent, ArrayNode node) { + if ( state==null ) { state = objectMapper.createArrayNode(); } + if ( result==null ) { result = state; } + super.walkArray(state, path, parent, node); + } + @Override + protected void walkArrayElement(JsonNode state, String path, ArrayNode parent, int index, JsonNode value) { + if ( value instanceof ContainerNode ) { + var newState = createContainerNode(value.getNodeType()); + ((ArrayNode)state).insert(index, newState); + super.walkArrayElement(newState, path, parent, index, value); + } else { + ((ArrayNode)state).insert(index, copyValue(state, path, parent, (ValueNode)value)); + } + } + @Override + protected void walkValue(JsonNode state, String path, JsonNode parent, ValueNode node) { + if ( result == null ) { result = copyValue(state, path, parent, node); } + } + protected final JsonNode createContainerNode(JsonNodeType jsonNodeType) { + return jsonNodeType==JsonNodeType.ARRAY + ? objectMapper.createArrayNode() + : objectMapper.createObjectNode(); + } + // We return JsonNode to allow subclasses to return other node types + protected JsonNode copyValue(JsonNode state, String path, JsonNode parent, ValueNode node) { + return node.deepCopy(); + } + } } diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/output/writer/output/standard/StandardOutputWriter.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/output/writer/output/standard/StandardOutputWriter.java index 247a4564c2..a1a8f7cde9 100644 --- a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/output/writer/output/standard/StandardOutputWriter.java +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/output/writer/output/standard/StandardOutputWriter.java @@ -168,13 +168,8 @@ private final void writeRecords(IRecordWriter recordWriter, HttpRequest httpR * @param httpRequest * @param nextPageRequestProducer */ - private final void writeRecords(IRecordWriter recordWriter, HttpRequest originalRequest, INextPageRequestProducer nextPageRequestProducer) { - var currentRequest = originalRequest; - while ( currentRequest!=null ) { - HttpResponse response = currentRequest.asObject(JsonNode.class); - writeRecords(recordWriter, response); - currentRequest = nextPageRequestProducer.getNextPageRequest(originalRequest, response); - } + private final void writeRecords(IRecordWriter recordWriter, HttpRequest initialRequest, INextPageRequestProducer nextPageRequestProducer) { + PagingHelper.processPages(initialRequest, nextPageRequestProducer, r->writeRecords(recordWriter, r)); } /** diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/progress/helper/ProgressWriterType.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/progress/helper/ProgressWriterType.java index ddf41ca54a..a33dee701a 100644 --- a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/progress/helper/ProgressWriterType.java +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/progress/helper/ProgressWriterType.java @@ -55,6 +55,14 @@ public void close() { clearProgress(); warnings.forEach(System.err::println); } + + protected final String format(String message, Object... args) { + if ( args==null || args.length==0 ) { + return message; + } else { + return String.format(message, args); + } + } } private static final class NoProgressWriter extends AbstractProgressWriter { @@ -78,7 +86,7 @@ public boolean isMultiLineSupported() { @Override public void writeProgress(String message, Object... args) { - String formattedMessage = String.format(message, args); + String formattedMessage = format(message, args); if ( formattedMessage.indexOf('\n') > 0 ) { // Add extra newline to separate multi-line blocks formattedMessage += "\n"; @@ -98,7 +106,7 @@ public boolean isMultiLineSupported() { @Override public void writeProgress(String message, Object... args) { - String formattedMessage = String.format(message, args); + String formattedMessage = format(message, args); if ( formattedMessage.indexOf('\n') > 0 ) { // Add extra newline to separate multi-line blocks formattedMessage += "\n"; @@ -121,9 +129,9 @@ public boolean isMultiLineSupported() { @Override public void writeProgress(String message, Object... args) { - if ( message.contains("\n") ) { throw new RuntimeException("Multiline status updates are not supported; please file a bug"); } + String formattedMessage = format(message, args); + if ( formattedMessage.contains("\n") ) { throw new RuntimeException("Multiline status updates are not supported; please file a bug"); } clearProgress(); - String formattedMessage = String.format(message, args); System.out.print(formattedMessage); this.lastNumberOfChars = formattedMessage.length(); } @@ -148,8 +156,8 @@ public boolean isMultiLineSupported() { @Override public void writeProgress(String message, Object... args) { + String formattedMessage = format(message, args); clearProgress(); - String formattedMessage = String.format(message, args); System.out.print(formattedMessage); this.lastNumberOfLines = (int)formattedMessage.chars().filter(ch -> ch == '\n').count(); } diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/rest/paging/PagingHelper.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/rest/paging/PagingHelper.java index 4b47dda9ab..eb388f7a0b 100644 --- a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/rest/paging/PagingHelper.java +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/rest/paging/PagingHelper.java @@ -12,6 +12,8 @@ *******************************************************************************/ package com.fortify.cli.common.rest.paging; +import java.util.function.Consumer; + import com.fasterxml.jackson.databind.JsonNode; import kong.unirest.Header; @@ -61,6 +63,23 @@ public static final INextPageRequestProducer asNextPageRequestProducer(UnirestIn return unirest==null || nextPageUrlProducer==null ? null : new NextPageRequestProducer(unirest, nextPageUrlProducer); } + public static final void processPages(UnirestInstance unirest, HttpRequest initialRequest, INextPageUrlProducer nextPageUrlProducer, Consumer> consumer) { + var nextPageRequestProducer = asNextPageRequestProducer(unirest, nextPageUrlProducer); + if ( nextPageRequestProducer==null ) { + throw new IllegalStateException("Cannot process pages without a valid NextPageRequestProducer"); + } + processPages(initialRequest, nextPageRequestProducer, consumer); + } + + public static final void processPages(HttpRequest initialRequest, INextPageRequestProducer nextPageRequestProducer, Consumer> consumer) { + var currentRequest = initialRequest; + while ( currentRequest!=null ) { + HttpResponse response = currentRequest.asObject(JsonNode.class); + consumer.accept(response); + currentRequest = nextPageRequestProducer.getNextPageRequest(initialRequest, response); + } + } + @RequiredArgsConstructor private static final class NextPageRequestProducer implements INextPageRequestProducer { private final UnirestInstance unirest; diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/IConfigurableSpelEvaluator.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/IConfigurableSpelEvaluator.java new file mode 100644 index 0000000000..edf2a0075d --- /dev/null +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/IConfigurableSpelEvaluator.java @@ -0,0 +1,21 @@ +/** + * Copyright 2023 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors (“Open Text”) are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.common.spring.expression; + +import java.util.function.Consumer; + +import org.springframework.expression.spel.support.SimpleEvaluationContext; + +public interface IConfigurableSpelEvaluator extends ISpelEvaluator { + IConfigurableSpelEvaluator configure(Consumer contextConfigurer); +} diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/ISpelEvaluator.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/ISpelEvaluator.java new file mode 100644 index 0000000000..41281ad1ff --- /dev/null +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/ISpelEvaluator.java @@ -0,0 +1,20 @@ +/** + * Copyright 2023 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors (“Open Text”) are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.common.spring.expression; + +import org.springframework.expression.Expression; + +public interface ISpelEvaluator { + R evaluate(Expression expression, Object input, Class returnClass); + R evaluate(String expression, Object input, Class returnClass); +} \ No newline at end of file diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/SpelEvaluator.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/SpelEvaluator.java index ddbb1cbead..c8be72c2ce 100644 --- a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/SpelEvaluator.java +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/SpelEvaluator.java @@ -14,6 +14,8 @@ import java.time.format.DateTimeFormatter; import java.util.List; +import java.util.function.Consumer; +import java.util.function.Supplier; import org.springframework.core.convert.converter.Converter; import org.springframework.expression.AccessException; @@ -25,48 +27,88 @@ import org.springframework.format.support.DefaultFormattingConversionService; import org.springframework.integration.json.JsonNodeWrapperToJsonNodeConverter; import org.springframework.integration.json.JsonPropertyAccessor; +import org.springframework.integration.json.JsonPropertyAccessor.JsonNodeWrapper; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fortify.cli.common.json.JsonHelper; -import lombok.Getter; import lombok.RequiredArgsConstructor; -@RequiredArgsConstructor -public enum SpelEvaluator { - JSON_GENERIC(createJsonGenericContext()), - JSON_QUERY(createJsonQueryContext()); +public enum SpelEvaluator implements ISpelEvaluator { + JSON_GENERIC(SpelEvaluator::createJsonGenericContext), + JSON_QUERY(SpelEvaluator::createJsonQueryContext); private static final SpelExpressionParser SPEL_PARSER = new SpelExpressionParser(); - @Getter private final EvaluationContext context; + private EvaluationContext context; + private final Supplier contextSupplier; + + private SpelEvaluator(Supplier contextSupplier) { + this.context = contextSupplier.get(); + this.contextSupplier = contextSupplier; + } public final R evaluate(Expression expression, Object input, Class returnClass) { - return expression.getValue(context, input, returnClass); + return evaluate(context, expression, input, returnClass); } public final R evaluate(String expression, Object input, Class returnClass) { return evaluate(SPEL_PARSER.parseExpression(expression), input, returnClass); } - private static final EvaluationContext createJsonGenericContext() { + public final IConfigurableSpelEvaluator copy() { + return new ConfigurableSpelEvaluator(contextSupplier.get()); + } + + @RequiredArgsConstructor + private static final class ConfigurableSpelEvaluator implements IConfigurableSpelEvaluator { + private final SimpleEvaluationContext context; + + public final R evaluate(Expression expression, Object input, Class returnClass) { + return SpelEvaluator.evaluate(context, expression, input, returnClass); + } + + public final R evaluate(String expression, Object input, Class returnClass) { + return evaluate(SPEL_PARSER.parseExpression(expression), input, returnClass); + } + + @Override + public IConfigurableSpelEvaluator configure(Consumer contextConfigurer) { + contextConfigurer.accept(context); + return this; + } + } + + private static final R evaluate(EvaluationContext context, Expression expression, Object input, Class returnClass) { + return unwrapSpelExpressionResult(expression.getValue(context, input, returnClass), returnClass); + } + + @SuppressWarnings("unchecked") + private static final R unwrapSpelExpressionResult(R result, Class returnClass) { + if ( result instanceof JsonNodeWrapper && returnClass.isAssignableFrom(JsonNode.class) ) { + result = (R)((JsonNodeWrapper)result).getRealNode(); + } + return result; + } + + private static final SimpleEvaluationContext createJsonGenericContext() { SimpleEvaluationContext context = SimpleEvaluationContext .forPropertyAccessors(new JsonPropertyAccessor()) .withConversionService(createJsonConversionService()) .withInstanceMethods() .build(); - SpelHelper.registerFunctions(context, StandardSpelFunctions.class); + SpelHelper.registerFunctions(context, SpelFunctionsStandard.class); return context; } - private static final EvaluationContext createJsonQueryContext() { + private static final SimpleEvaluationContext createJsonQueryContext() { SimpleEvaluationContext context = SimpleEvaluationContext .forPropertyAccessors(new ExistingJsonPropertyAccessor()) .withConversionService(createJsonConversionService()) .withInstanceMethods() .build(); - SpelHelper.registerFunctions(context, StandardSpelFunctions.class); + SpelHelper.registerFunctions(context, SpelFunctionsStandard.class); return context; } diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/StandardSpelFunctions.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/SpelFunctionsStandard.java similarity index 98% rename from fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/StandardSpelFunctions.java rename to fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/SpelFunctionsStandard.java index 8952489a88..afe231b553 100644 --- a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/StandardSpelFunctions.java +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/SpelFunctionsStandard.java @@ -28,7 +28,7 @@ import lombok.NoArgsConstructor; @Reflectable @NoArgsConstructor -public class StandardSpelFunctions { +public class SpelFunctionsStandard { private static final DateTimePeriodHelper PeriodHelper = DateTimePeriodHelper.all(); public static final OffsetDateTime date(String s) { diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/SpelHelper.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/SpelHelper.java index eb41c96e63..69ebbcaa05 100644 --- a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/SpelHelper.java +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/SpelHelper.java @@ -14,9 +14,23 @@ import java.lang.reflect.Method; +import org.springframework.expression.common.TemplateParserContext; +import org.springframework.expression.spel.standard.SpelExpressionParser; import org.springframework.expression.spel.support.SimpleEvaluationContext; +import com.fortify.cli.common.spring.expression.wrapper.SimpleExpression; +import com.fortify.cli.common.spring.expression.wrapper.TemplateExpression; + public final class SpelHelper { + private static final SpelExpressionParser parser = new SpelExpressionParser(); + private static final TemplateParserContext templateContext = new TemplateParserContext("${","}"); + + public static final SimpleExpression parseSimpleExpression(String s) { + return new SimpleExpression(parser.parseExpression(s)); + } + public static final TemplateExpression parseTemplateExpression(String s) { + return new TemplateExpression(parser.parseExpression(s, templateContext)); + } public static final void registerFunctions(SimpleEvaluationContext context, Class clazz) { for ( Method m : clazz.getDeclaredMethods() ) { context.setVariable(m.getName(), m); diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/wrapper/SimpleExpression.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/wrapper/SimpleExpression.java new file mode 100644 index 0000000000..23625e90f1 --- /dev/null +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/wrapper/SimpleExpression.java @@ -0,0 +1,47 @@ +/******************************************************************************* + * (c) Copyright 2020 Micro Focus or one of its affiliates, a Micro Focus company + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including without + * limitation the rights to use, copy, modify, merge, publish, distribute, + * sublicense, and/or sell copies of the Software, and to permit persons to + * whom the Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY + * KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + ******************************************************************************/ +package com.fortify.cli.common.spring.expression.wrapper; + +import org.springframework.expression.Expression; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + +/** + *

This is a simple wrapper class for a Spring {@link Expression} + * instance. It's main use is in combination with + * {@link SimpleExpressionDeserializer} to allow automatic + * conversion from String values to simple {@link Expression} + * instances in JSON/YAML documents.

+ * + *

The reason for needing this wrapper class is to differentiate + * with templated {@link Expression} instances that are handled + * by {@link TemplateExpressionDeserializer}.

+ */ +@JsonDeserialize(using = SimpleExpressionDeserializer.class) +public class SimpleExpression extends WrappedExpression { + public SimpleExpression(Expression target) { + super(target); + } +} diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/wrapper/SimpleExpressionDeserializer.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/wrapper/SimpleExpressionDeserializer.java new file mode 100644 index 0000000000..84f7d59594 --- /dev/null +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/wrapper/SimpleExpressionDeserializer.java @@ -0,0 +1,55 @@ +/******************************************************************************* + * (c) Copyright 2020 Micro Focus or one of its affiliates, a Micro Focus company + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including without + * limitation the rights to use, copy, modify, merge, publish, distribute, + * sublicense, and/or sell copies of the Software, and to permit persons to + * whom the Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY + * KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + ******************************************************************************/ +package com.fortify.cli.common.spring.expression.wrapper; + +import java.io.IOException; + +import org.springframework.expression.spel.standard.SpelExpressionParser; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import com.formkiq.graalvm.annotations.Reflectable; + +/** + * This Jackson deserializer allows parsing String values into an + * SpEL Expression object. + */ +@Reflectable +public final class SimpleExpressionDeserializer extends StdDeserializer { + private static final long serialVersionUID = 1L; + private static final SpelExpressionParser parser = new SpelExpressionParser(); + public SimpleExpressionDeserializer() { this(null); } + public SimpleExpressionDeserializer(Class vc) { super(vc); } + + @Override + public SimpleExpression deserialize(JsonParser jp, DeserializationContext ctxt) + throws IOException, JsonProcessingException { + JsonNode node = jp.getCodec().readTree(jp); + return node==null || node.isNull() ? null : new SimpleExpression(parser.parseExpression(node.asText())); + } +} diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/wrapper/TemplateExpression.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/wrapper/TemplateExpression.java new file mode 100644 index 0000000000..ec37450f8d --- /dev/null +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/wrapper/TemplateExpression.java @@ -0,0 +1,47 @@ +/******************************************************************************* + * (c) Copyright 2020 Micro Focus or one of its affiliates, a Micro Focus company + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including without + * limitation the rights to use, copy, modify, merge, publish, distribute, + * sublicense, and/or sell copies of the Software, and to permit persons to + * whom the Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY + * KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + ******************************************************************************/ +package com.fortify.cli.common.spring.expression.wrapper; + +import org.springframework.expression.Expression; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + +/** + *

This is a simple wrapper class for a Spring {@link Expression} + * instance. It's main use is in combination with + * {@link TemplateExpressionDeserializer} to allow automatic + * conversion from String values to templated {@link Expression} + * instances.

+ * + *

The reason for needing this wrapper class is to differentiate + * with non-templated {@link Expression} instances that are + * handled by {@link SimpleExpressionDeserializer}.

+ */ +@JsonDeserialize(using = TemplateExpressionDeserializer.class) +public class TemplateExpression extends WrappedExpression { + public TemplateExpression(Expression target) { + super(target); + } +} diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/wrapper/TemplateExpressionDeserializer.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/wrapper/TemplateExpressionDeserializer.java new file mode 100644 index 0000000000..8831a324a7 --- /dev/null +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/wrapper/TemplateExpressionDeserializer.java @@ -0,0 +1,55 @@ +/******************************************************************************* + * (c) Copyright 2020 Micro Focus or one of its affiliates, a Micro Focus company + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including without + * limitation the rights to use, copy, modify, merge, publish, distribute, + * sublicense, and/or sell copies of the Software, and to permit persons to + * whom the Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY + * KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + ******************************************************************************/ +package com.fortify.cli.common.spring.expression.wrapper; + +import java.beans.PropertyEditor; +import java.io.IOException; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import com.formkiq.graalvm.annotations.Reflectable; +import com.fortify.cli.common.spring.expression.SpelHelper; + +/** + * This {@link PropertyEditor} allows parsing String values into a + * TemplateExpression object. + */ +@Reflectable +public final class TemplateExpressionDeserializer extends StdDeserializer { + private static final long serialVersionUID = 1L; + + public TemplateExpressionDeserializer() { this(null); } + public TemplateExpressionDeserializer(Class vc) { super(vc); } + + @Override + public TemplateExpression deserialize(JsonParser jp, DeserializationContext ctxt) + throws IOException, JsonProcessingException { + JsonNode node = jp.getCodec().readTree(jp); + return node==null || node.isNull() ? null : SpelHelper.parseTemplateExpression(node.asText()); + } +} diff --git a/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/wrapper/WrappedExpression.java b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/wrapper/WrappedExpression.java new file mode 100644 index 0000000000..c06bbe1132 --- /dev/null +++ b/fcli-core/fcli-common/src/main/java/com/fortify/cli/common/spring/expression/wrapper/WrappedExpression.java @@ -0,0 +1,292 @@ +/******************************************************************************* + * (c) Copyright 2020 Micro Focus or one of its affiliates, a Micro Focus company + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including without + * limitation the rights to use, copy, modify, merge, publish, distribute, + * sublicense, and/or sell copies of the Software, and to permit persons to + * whom the Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY + * KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + ******************************************************************************/ +package com.fortify.cli.common.spring.expression.wrapper; + +import org.springframework.core.convert.TypeDescriptor; +import org.springframework.expression.EvaluationContext; +import org.springframework.expression.EvaluationException; +import org.springframework.expression.Expression; + +/** + *

This is a simple wrapper class for a Spring {@link Expression} + * instance. This class is used as a based class for both + * {@link SimpleExpression} and {@link TemplateExpression}.

+ */ +public class WrappedExpression implements Expression { + private final Expression target; + + /** + * Constructor for configuring the expression to be wrapped + * @param target {@link Expression} to be wrapped + */ + public WrappedExpression(Expression target) { + this.target = target; + } + + /** + * @see org.springframework.expression.Expression#getValue() + * @return The evaluation result + * @throws EvaluationException if there is a problem during evaluation + */ + public Object getValue() throws EvaluationException { + return target.getValue(); + } + + /** + * @see org.springframework.expression.Expression#getValue(Object) + * @param rootObject the root object against which to evaluate the expression + * @return the evaluation result + * @throws EvaluationException if there is a problem during evaluation + */ + public Object getValue(Object rootObject) throws EvaluationException { + return target.getValue(rootObject); + } + + /** + * @see org.springframework.expression.Expression#getValue(java.lang.Class) + * @param desiredResultType the class the caller would like the result to be + * @return the evaluation result + * @throws EvaluationException if there is a problem during evaluation + */ + public T getValue(Class desiredResultType) throws EvaluationException { + return target.getValue(desiredResultType); + } + + /** + * @see org.springframework.expression.Expression#getValue(Object, java.lang.Class) + * @param rootObject the root object against which to evaluate the expression + * @param desiredResultType the class the caller would like the result to be + * @return the evaluation result + * @throws EvaluationException if there is a problem during evaluation + */ + public T getValue(Object rootObject,Class desiredResultType) throws EvaluationException { + return target.getValue(rootObject, desiredResultType); + } + + /** + * @see org.springframework.expression.Expression#getValue(EvaluationContext) + * @param context the context in which to evaluate the expression + * @return the evaluation result + * @throws EvaluationException if there is a problem during evaluation + * + */ + public Object getValue(EvaluationContext context) throws EvaluationException { + return target.getValue(context); + } + + /** + * @see org.springframework.expression.Expression#getValue(EvaluationContext, Object) + * @param context the context in which to evaluate the expression + * @param rootObject the root object against which to evaluate the expression + * @return the evaluation result + * @throws EvaluationException if there is a problem during evaluation + * + */ + public Object getValue(EvaluationContext context, Object rootObject) throws EvaluationException { + return target.getValue(context, rootObject); + } + + /** + * @see org.springframework.expression.Expression#getValue(EvaluationContext, java.lang.Class) + * @param context the context in which to evaluate the expression + * @param desiredResultType the class the caller would like the result to be + * @return the evaluation result + * @throws EvaluationException if there is a problem during evaluation + */ + public T getValue(EvaluationContext context, Class desiredResultType) throws EvaluationException { + return target.getValue(context, desiredResultType); + } + + /** + * @see org.springframework.expression.Expression#getValue(EvaluationContext, Object, java.lang.Class) + * @param context the context in which to evaluate the expression + * @param rootObject the root object against which to evaluate the expression + * @param desiredResultType the class the caller would like the result to be + * @return the evaluation result + * @throws EvaluationException if there is a problem during evaluation + */ + public T getValue(EvaluationContext context, Object rootObject, Class desiredResultType) throws EvaluationException { + return target.getValue(context, rootObject, desiredResultType); + } + + /** + * @see org.springframework.expression.Expression#getValueType() + * @return the most general type of value that can be set on this context + * @throws EvaluationException if there is a problem determining the type + */ + public Class getValueType() throws EvaluationException { + return target.getValueType(); + } + + /** + * @see org.springframework.expression.Expression#getValueType(Object) + * @param rootObject the root object against which to evaluate the expression + * @return the most general type of value that can be set on this context + * @throws EvaluationException if there is a problem determining the type + */ + public Class getValueType(Object rootObject) throws EvaluationException { + return target.getValueType(rootObject); + } + + /** + * @see org.springframework.expression.Expression#getValueType(EvaluationContext) + * @param context the context in which to evaluate the expression + * @return the most general type of value that can be set on this context + * @throws EvaluationException if there is a problem determining the type + */ + public Class getValueType(EvaluationContext context) throws EvaluationException { + return target.getValueType(context); + } + + /** + * @see org.springframework.expression.Expression#getValueType(EvaluationContext, Object) + * @param context the context in which to evaluate the expression + * @param rootObject the root object against which to evaluate the expression + * @return the most general type of value that can be set on this context + * @throws EvaluationException if there is a problem determining the type + */ + public Class getValueType(EvaluationContext context, Object rootObject) throws EvaluationException { + return target.getValueType(context, rootObject); + } + + /** + * @see org.springframework.expression.Expression#getValueTypeDescriptor() + * @return a type descriptor for values that can be set on this context + * @throws EvaluationException if there is a problem determining the type + */ + public TypeDescriptor getValueTypeDescriptor() throws EvaluationException { + return target.getValueTypeDescriptor(); + } + + /** + * @see org.springframework.expression.Expression#getValueTypeDescriptor(Object) + * @param rootObject the root object against which to evaluate the expression + * @return a type descriptor for values that can be set on this context + * @throws EvaluationException if there is a problem determining the type + */ + public TypeDescriptor getValueTypeDescriptor(Object rootObject) throws EvaluationException { + return target.getValueTypeDescriptor(rootObject); + } + + /** + * @see org.springframework.expression.Expression#getValueTypeDescriptor(EvaluationContext) + * @param context the context in which to evaluate the expression + * @return a type descriptor for values that can be set on this context + * @throws EvaluationException if there is a problem determining the type + */ + public TypeDescriptor getValueTypeDescriptor(EvaluationContext context) throws EvaluationException { + return target.getValueTypeDescriptor(context); + } + + /** + * @see org.springframework.expression.Expression#getValueTypeDescriptor(EvaluationContext, Object) + * @param context the context in which to evaluate the expression + * @param rootObject the root object against which to evaluate the expression + * @return a type descriptor for values that can be set on this context + * @throws EvaluationException if there is a problem determining the type + */ + public TypeDescriptor getValueTypeDescriptor(EvaluationContext context, Object rootObject) throws EvaluationException { + return target.getValueTypeDescriptor(context, rootObject); + } + + /** + * @see org.springframework.expression.Expression#isWritable(EvaluationContext) + * @param context the context in which the expression should be checked + * @return {@code true} if the expression is writable; {@code false} otherwise + * @throws EvaluationException if there is a problem determining if it is writable + */ + public boolean isWritable(EvaluationContext context) throws EvaluationException { + return target.isWritable(context); + } + + /** + * @see org.springframework.expression.Expression#isWritable(EvaluationContext, Object) + * @param context the context in which the expression should be checked + * @param rootObject the root object against which to evaluate the expression + * @return {@code true} if the expression is writable; {@code false} otherwise + * @throws EvaluationException if there is a problem determining if it is writable + */ + public boolean isWritable(EvaluationContext context, Object rootObject) throws EvaluationException { + return target.isWritable(context, rootObject); + } + + /** + * @see org.springframework.expression.Expression#isWritable(Object) + * @param rootObject the root object against which to evaluate the expression + * @return {@code true} if the expression is writable; {@code false} otherwise + * @throws EvaluationException if there is a problem determining if it is writable + */ + public boolean isWritable(Object rootObject) throws EvaluationException { + return target.isWritable(rootObject); + } + + /** + * @see org.springframework.expression.Expression#setValue(EvaluationContext, Object) + * @param context the context in which to set the value of the expression + * @param value the new value + * @throws EvaluationException if there is a problem during evaluation + */ + public void setValue(EvaluationContext context, Object value) throws EvaluationException { + target.setValue(context, value); + } + + /** + * @see org.springframework.expression.Expression#setValue(Object, Object) + * @param rootObject the root object against which to evaluate the expression + * @param value the new value + * @throws EvaluationException if there is a problem during evaluation + */ + public void setValue(Object rootObject, Object value) throws EvaluationException { + target.setValue(rootObject, value); + } + + /** + * @see org.springframework.expression.Expression#setValue(EvaluationContext, Object, Object) + * @param context the context in which to set the value of the expression + * @param rootObject the root object against which to evaluate the expression + * @param value the new value + * @throws EvaluationException if there is a problem during evaluation + */ + public void setValue(EvaluationContext context, Object rootObject, Object value) throws EvaluationException { + target.setValue(context, rootObject, value); + } + + /** + * @see org.springframework.expression.Expression#getExpressionString() + * @return the original expression string + */ + public String getExpressionString() { + return target.getExpressionString(); + } + + /** + * @return String representation for this {@link WrappedExpression} + */ + @Override + public String toString() { + return this.getClass().getSimpleName()+"("+getExpressionString()+")"; + } + +} diff --git a/fcli-core/fcli-common/src/main/java/org/springframework/integration/json/JsonPropertyAccessor.java b/fcli-core/fcli-common/src/main/java/org/springframework/integration/json/JsonPropertyAccessor.java index 966e3cc719..1c925b71ae 100644 --- a/fcli-core/fcli-common/src/main/java/org/springframework/integration/json/JsonPropertyAccessor.java +++ b/fcli-core/fcli-common/src/main/java/org/springframework/integration/json/JsonPropertyAccessor.java @@ -20,11 +20,6 @@ import java.util.AbstractList; import java.util.Iterator; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; - import org.springframework.expression.AccessException; import org.springframework.expression.EvaluationContext; import org.springframework.expression.PropertyAccessor; @@ -33,6 +28,12 @@ import org.springframework.util.Assert; import org.springframework.util.StringUtils; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.POJONode; + /** * A SpEL {@link PropertyAccessor} that knows how to read properties from JSON objects. * Uses Jackson {@link JsonNode} API for nested properties access. @@ -192,6 +193,10 @@ else if (json.isBinary()) { "Can not get content of binary value: " + json, e); } } + // CHANGED: Compared to original version, this code was added to allow access to POJO node values. + else if (json.isPojo() ) { + return ((POJONode)json).getPojo(); + } throw new IllegalArgumentException("Json is not ValueNode."); } @@ -210,7 +215,10 @@ else if (json.isValueNode()) { } } - interface JsonNodeWrapper extends Comparable { + // CHANGED: Compared to original version, this interface was changed to public + // to allow access by JsonHelper::evaluateSpelExpression (through + // JsonHelper::unwrapSpelExpressionResult). + public interface JsonNodeWrapper extends Comparable { String toString(); diff --git a/fcli-core/fcli-common/src/main/java/org/springframework/integration/json/package-info.java b/fcli-core/fcli-common/src/main/java/org/springframework/integration/json/package-info.java index 6bf933274f..136f18dda5 100644 --- a/fcli-core/fcli-common/src/main/java/org/springframework/integration/json/package-info.java +++ b/fcli-core/fcli-common/src/main/java/org/springframework/integration/json/package-info.java @@ -1,7 +1,11 @@ /** * This package contains a copy of two classes from spring-integration that * fcli depends on, to avoid having to pull in the full spring-integration - * dependency and transitive dependencies. + * dependency and transitive dependencies. Compared to the original version, + * there's one minor change; the JsonNodeWrapper interface has been changed + * to public to allow access to the original JsonNode instance if an SpEL + * expression returns a JsonNode, as used by JsonHelper::evaluateSpelExpression + * (through JsonHelper::unwrapSpelExpressionResult). */ package org.springframework.integration.json; diff --git a/fcli-core/fcli-common/src/main/resources/com/fortify/cli/common/i18n/FortifyCLIMessages.properties b/fcli-core/fcli-common/src/main/resources/com/fortify/cli/common/i18n/FortifyCLIMessages.properties index 9bf8c02f6a..1cba3b7034 100644 --- a/fcli-core/fcli-common/src/main/resources/com/fortify/cli/common/i18n/FortifyCLIMessages.properties +++ b/fcli-core/fcli-common/src/main/resources/com/fortify/cli/common/i18n/FortifyCLIMessages.properties @@ -27,6 +27,7 @@ log-level = Set logging level. Note that DEBUG and TRACE levels may result in se being written to the log file. Allowed values: ${COMPLETION-CANDIDATES}. log-file = File where logging data will be written. Defaults to fcli.log in current directory \ if --log-level is specified. +fcli.data-extract.create.template-parameter = TODO # Generic, non command-specific output and query options arggroup.output.heading = Output options:%n diff --git a/fcli-core/fcli-common/src/test/java/com/fortify/cli/common/json/JsonNodeDeepCopyWalkerTest.java b/fcli-core/fcli-common/src/test/java/com/fortify/cli/common/json/JsonNodeDeepCopyWalkerTest.java new file mode 100644 index 0000000000..5c5a070b2e --- /dev/null +++ b/fcli-core/fcli-common/src/test/java/com/fortify/cli/common/json/JsonNodeDeepCopyWalkerTest.java @@ -0,0 +1,105 @@ +/******************************************************************************* + * Copyright 2021, 2023 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + *******************************************************************************/ +package com.fortify.cli.common.json; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + +import java.util.UUID; +import java.util.stream.Stream; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +import com.fasterxml.jackson.databind.node.ValueNode; +import com.fortify.cli.common.json.JsonHelper.JsonNodeDeepCopyWalker; + +public class JsonNodeDeepCopyWalkerTest { + private static final ObjectMapper objectMapper = JsonHelper.getObjectMapper(); + + @ParameterizedTest + @MethodSource("getSampleNodesStream") + public void testDeepCopy(JsonNode node) throws Exception { + var copy = new JsonNodeDeepCopyWalker().walk(node); + assertEquals(node, copy); + if ( node!=null && !(node instanceof ValueNode) ) { + System.out.println(copy.toPrettyString()); + assertFalse(node==copy); + } + } + + private static Stream getSampleNodesStream() { + return Stream.of( + Arguments.of(new Object[] {null}), + Arguments.of(createEmptyObjectNode()), + Arguments.of(createSampleValueObjectNode()), + Arguments.of(createSampleMultiLevelObjectNode()), + Arguments.of(createEmptyArrayNode()), + Arguments.of(createSampleValueArrayNode()), + Arguments.of(createSampleMultiLevelArrayNode()), + Arguments.of(createSampleValueNode()) + ); + } + + private static ObjectNode createEmptyObjectNode() { + return objectMapper.createObjectNode(); + } + + private static ArrayNode createEmptyArrayNode() { + return objectMapper.createArrayNode(); + } + + private static ObjectNode createSampleMultiLevelObjectNode() { + var result = createSampleValueObjectNode(); + result.set("array", createSampleValueArrayNode()); + result.set("obj", createSampleValueObjectNode()); + return result; + } + + private static ObjectNode createSampleValueObjectNode() { + var result = objectMapper.createObjectNode(); + result.put("int", 1); + result.put("bool", true); + result.put("float", 2.2f); + result.put("str1", "someString"); + result.set("val", createSampleValueNode()); + return result; + } + + private static ArrayNode createSampleMultiLevelArrayNode() { + var result = objectMapper.createArrayNode(); + for ( int i = 0 ; i < 20 ; i++ ) { + result.add(createSampleMultiLevelObjectNode()); + } + return result; + } + + private static ArrayNode createSampleValueArrayNode() { + var result = objectMapper.createArrayNode(); + for ( int i = 0 ; i < 10 ; i++ ) { + result.add(100+i); + } + return result; + } + + private static ValueNode createSampleValueNode() { + return new TextNode(UUID.randomUUID().toString()); + } + +} diff --git a/fcli-core/fcli-sc-sast/src/main/resources/com/fortify/cli/data_extract/templates/SC-SAST.zip b/fcli-core/fcli-sc-sast/src/main/resources/com/fortify/cli/data_extract/templates/SC-SAST.zip deleted file mode 100644 index 1b82ba32a2..0000000000 Binary files a/fcli-core/fcli-sc-sast/src/main/resources/com/fortify/cli/data_extract/templates/SC-SAST.zip and /dev/null differ diff --git a/fcli-core/fcli-sc-sast/src/main/resources/com/fortify/cli/sc_sast/i18n/SCSastMessages.properties b/fcli-core/fcli-sc-sast/src/main/resources/com/fortify/cli/sc_sast/i18n/SCSastMessages.properties index 9c79d702ee..174c2a32f7 100644 --- a/fcli-core/fcli-sc-sast/src/main/resources/com/fortify/cli/sc_sast/i18n/SCSastMessages.properties +++ b/fcli-core/fcli-sc-sast/src/main/resources/com/fortify/cli/sc_sast/i18n/SCSastMessages.properties @@ -101,6 +101,11 @@ fcli.sc-sast.session.list.usage.description.1 = For sessions created using user fcli.sc-sast.session.list.usage.description.2 = For sessions created using a pre-generated token, fcli cannot \ display session expiration date or status, as SSC doesn't allow for obtaining this information. +# fcli sc-sast data-extract +fcli.sc-sast.data-extract.usage.header = TODO +fcli.sc-sast.data-extract.create.usage.header = TODO +fcli.sc-sast.data-extract.create.template = TODO + # fcli sc-sast scan fcli.sc-sast.scan.usage.header = Manage ScanCentral SAST scans. fcli.sc-sast.scan.cancel.usage.header = Cancel a previously submitted scan request. diff --git a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/_main/cli/cmd/SSCCommands.java b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/_main/cli/cmd/SSCCommands.java index e9c36420a4..449615bee6 100644 --- a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/_main/cli/cmd/SSCCommands.java +++ b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/_main/cli/cmd/SSCCommands.java @@ -20,6 +20,7 @@ import com.fortify.cli.ssc.appversion.cli.cmd.SSCAppVersionCommands; import com.fortify.cli.ssc.artifact.cli.cmd.SSCArtifactCommands; import com.fortify.cli.ssc.attribute.cli.cmd.SSCAttributeCommands; +import com.fortify.cli.ssc.data_extract.cli.cmd.SSCDataExtractCommands; import com.fortify.cli.ssc.issue.cli.cmd.SSCIssueCommands; import com.fortify.cli.ssc.performance_indicator.cli.cmd.SSCPerformanceIndicatorCommands; import com.fortify.cli.ssc.plugin.cli.cmd.SSCPluginCommands; @@ -49,6 +50,7 @@ SSCAppVersionCommands.class, SSCArtifactCommands.class, SSCAttributeCommands.class, + SSCDataExtractCommands.class, SSCIssueCommands.class, SSCPerformanceIndicatorCommands.class, SSCVariableCommands.class, diff --git a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/data_extract/cli/cmd/SSCDataExtractCommands.java b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/data_extract/cli/cmd/SSCDataExtractCommands.java new file mode 100644 index 0000000000..63d9522b3d --- /dev/null +++ b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/data_extract/cli/cmd/SSCDataExtractCommands.java @@ -0,0 +1,26 @@ +/******************************************************************************* + * Copyright 2021, 2023 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + *******************************************************************************/ +package com.fortify.cli.ssc.data_extract.cli.cmd; + +import com.fortify.cli.common.cli.cmd.AbstractContainerCommand; + +import picocli.CommandLine.Command; + +@Command( + name = "data-extract", + subcommands = { + SSCDataExtractCreateCommand.class, + } +) +public class SSCDataExtractCommands extends AbstractContainerCommand { +} diff --git a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/data_extract/cli/cmd/SSCDataExtractCreateCommand.java b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/data_extract/cli/cmd/SSCDataExtractCreateCommand.java new file mode 100644 index 0000000000..5034526e5d --- /dev/null +++ b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/data_extract/cli/cmd/SSCDataExtractCreateCommand.java @@ -0,0 +1,101 @@ +/******************************************************************************* + * Copyright 2021, 2023 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + *******************************************************************************/ +package com.fortify.cli.ssc.data_extract.cli.cmd; + +import java.util.List; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fortify.cli.common.data_extract.cli.cmd.AbstractDataExtractCreateCommand; +import com.fortify.cli.common.data_extract.helper.DataExtractTemplateDescriptor.TemplateValidationException; +import com.fortify.cli.common.data_extract.helper.DataExtractTemplateRunner; +import com.fortify.cli.common.data_extract.helper.DataExtractTemplateRunner.IDataExtractRequestHelper.BasicDataExtractRequestHelper; +import com.fortify.cli.common.data_extract.helper.DataExtractTemplateRunner.ParameterTypeConverterArgs; +import com.fortify.cli.common.output.cli.mixin.OutputHelperMixins; +import com.fortify.cli.common.output.product.IProductHelper; +import com.fortify.cli.common.rest.unirest.IUnirestInstanceSupplier; +import com.fortify.cli.common.spring.expression.SpelHelper; +import com.fortify.cli.common.util.StringUtils; +import com.fortify.cli.ssc._common.rest.bulk.SSCBulkRequestBuilder; +import com.fortify.cli.ssc._common.rest.helper.SSCProductHelper; +import com.fortify.cli.ssc._common.session.cli.mixin.SSCUnirestInstanceSupplierMixin; +import com.fortify.cli.ssc.appversion.helper.SSCAppVersionHelper; +import com.fortify.cli.ssc.issue.helper.SSCIssueFilterSetHelper; + +import lombok.Getter; +import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; + +@Command(name = OutputHelperMixins.Create.CMD_NAME) +public class SSCDataExtractCreateCommand extends AbstractDataExtractCreateCommand { + @Getter @Mixin private SSCUnirestInstanceSupplierMixin unirestInstanceSupplier; + + @Override + protected final String getType() { + return "SSC"; + } + + @Override + protected void configure(DataExtractTemplateRunner templateExecutor) { + templateExecutor + .addParameterConverter("appversion_single", this::loadAppVersion) + .addParameterConverter("filterset", this::loadFilterSet) + .addRequestHelper("ssc", new SSCDataExtractRequestHelper(unirestInstanceSupplier::getUnirestInstance, SSCProductHelper.INSTANCE)); + } + + private final JsonNode loadAppVersion(String nameOrId, ParameterTypeConverterArgs args) { + args.getProgressWriter().writeProgress("Loading application version %s", nameOrId); + var result = SSCAppVersionHelper.getRequiredAppVersion(unirestInstanceSupplier.getUnirestInstance(), nameOrId, ":"); + args.getProgressWriter().writeProgress("Loaded application version %s", result.getAppAndVersionName()); + return result.asJsonNode(); + } + + private final JsonNode loadFilterSet(String titleOrId, ParameterTypeConverterArgs args) { + var progressMessage = StringUtils.isBlank(titleOrId) + ? "Loading default filter set" + : String.format("Loading filter set %s", titleOrId); + args.getProgressWriter().writeProgress(progressMessage); + var parameter = args.getParameter(); + var typeParameters = parameter.getTypeParameters(); + var appVersionIdExpression = typeParameters==null ? null : typeParameters.get("appversion.id"); + if ( appVersionIdExpression==null ) { + appVersionIdExpression = SpelHelper.parseTemplateExpression("${appversion?.id}"); + } + var appVersionId = args.getSpelEvaluator().evaluate(appVersionIdExpression, args.getParameters(), String.class); + if ( StringUtils.isBlank(appVersionId) ) { + throw new TemplateValidationException(String.format("Template parameter %s requires ${%s} to be available", parameter.getName(), appVersionIdExpression.getExpressionString())); + } + return new SSCIssueFilterSetHelper(unirestInstanceSupplier.getUnirestInstance(), appVersionId).getDescriptorByTitleOrId(titleOrId, false).asJsonNode(); + } + + private static final class SSCDataExtractRequestHelper extends BasicDataExtractRequestHelper { + public SSCDataExtractRequestHelper(IUnirestInstanceSupplier unirestInstanceSupplier, IProductHelper productHelper) { + super(unirestInstanceSupplier, productHelper); + } + + @Override + public void executeSimpleRequests(List requestDescriptors) { + var bulkRequestBuilder = new SSCBulkRequestBuilder(); + requestDescriptors.forEach(r->addRequest(bulkRequestBuilder, r)); + bulkRequestBuilder.execute(getUnirestInstance()); + } + + private void addRequest(SSCBulkRequestBuilder bulkRequestBuilder, DataExtractRequestDescriptor requestDescriptor) { + var request = getUnirestInstance() + .get(requestDescriptor.getUri()) + .queryString(requestDescriptor.getQueryParams()); + bulkRequestBuilder.request(request, requestDescriptor.getResponseConsumer()); + } + + + } +} diff --git a/fcli-core/fcli-ssc/src/main/resources/com/fortify/cli/ssc/data_extract/templates/github-code-scanning.yaml b/fcli-core/fcli-ssc/src/main/resources/com/fortify/cli/ssc/data_extract/templates/github-code-scanning.yaml new file mode 100644 index 0000000000..b6157ca838 --- /dev/null +++ b/fcli-core/fcli-ssc/src/main/resources/com/fortify/cli/ssc/data_extract/templates/github-code-scanning.yaml @@ -0,0 +1,132 @@ +description: | + bla + +parameters: + - name: output + description: "Output location; either 'stdout', 'stderr' or file name. Default value: gh-fortify-sast.sarif" + required: false + defaultValue: gh-fortify-sast.sarif + - name: appversion + description: "Required application version id or :" + type: appversion_single + - name: filterset + description: "Filter set from which to load issue data. Default value: Default filter set for given application version" + required: false + type: filterset + +steps: + - progress: Loading latest static scan + - requests: + - name: lastStaticScanId + uri: /api/v1/projectVersions/${#parameters.appversion.id}/issues?start=0&limit=1&showhidden=true&showremoved=true&showsuppressed=true&showshortfilenames=true + query: + orderby: lastScanId + fields: lastScanId + filter: ISSUE[11111111-1111-1111-1111-111111111151]:SCA + filterset: ${#parameters.filterset.guid} + - requests: + - name: lastStaticScan + if: lastStaticScanId?.size()>0 + uri: /api/v1/scans/${lastStaticScanId[0].lastScanId} + - progress: Processing issue data + - requests: + - name: issues + uri: /api/v1/projectVersions/${#parameters.appversion.id}/issues?limit=100 + query: + filter: ISSUE[11111111-1111-1111-1111-111111111151]:SCA + filterset: ${#parameters.filterset.guid} + pagingProgress: + postPageProcess: Processed ${#partialOutputs.results?.size()?:0} of ${issues_raw.count} issues + from: ssc + forEach: + name: issue + steps: + - requests: + - name: issue_details + uri: /api/v1/issueDetails/${issue.id} + from: ssc + - progress: Writing output + +outputWriters: + - outputName: github-code-scanning + to: ${#parameters.output} + +outputs: + - name: markdownTest + contents: |- + ## ${#parameters.appversion.project.name} - ${#parameters.appversion.name} + bla bla + + ${#join('\n', #partialOutputs.rules)} + - name: github-code-scanning + contents: + "$schema": https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json + version: '2.1.0' + runs: + - tool: + driver: + name: 'Fortify SCA' + version: ${lastStaticScan?.engineVersion?:'unknown'} + rules: ${#partialOutputs.rules?:{}} + results: ${#check(#partialOutputs.results?.size()>1000 , "GitHub does not support importing more than 1000 vulnerabilities. Please clean the scan results or update vulnerability search criteria.")?#partialOutputs.results?:{}:{}} + +partialOutputs: + issue_details: +# - name: mdTest +# contents: |- +# ### ${issue.id} +# ${issue_details?.brief} + - name: rules + contents: + id: ${issue.id+''} + shortDescription: + text: ${issue.issueName} + fullDescription: + text: ${issue_details?.brief} + help: + text: ${#htmlToText(issue_details?.detail)+'\n\n'+#htmlToText(issue_details?.recommendation)+"\n\nFor more information, see "+issue.deepLink} + properties: + tags: ${{issue.friority}} + precision: ${(issue.friority matches "(Critical|Medium)") ? "high":"low" } + security-severity: ${{Critical:10.0,High:8.9,Medium:6.9,Low:3.9}.get(issue.friority)+''} + - name: results + contents: + ruleId: ${issue.id+''} + message: + text: ${issue_details?.brief} + level: ${(issue.friority matches "(Critical|High)") ? "warning":"note" } + partialFingerprints: + issueInstanceId: ${issue.issueInstanceId} + locations: + - physicalLocation: + artifactLocation: + uri: ${issue.fullFileName} + region: + startLine: ${issue.lineNumber==0?1:issue.lineNumber} + endLine: ${issue.lineNumber==0?1:issue.lineNumber} + startColumn: ${1} # Needs to be specified as an expression in order to end up as integer instead of string in JSON + endColumn: ${80} + codeFlows: |- + ${ + issue_details?.traceNodes==null ? {} + : + {{ + threadFlows: issue_details?.traceNodes.![{ + locations: #this.![{ + location: { + message: { + text: text + }, + physicalLocation: { + artifactLocation: { + uri: fullPath + }, + region: { + startLine: line==0?1:line + } + } + } + }] + }] + }} + } \ No newline at end of file diff --git a/fcli-core/fcli-ssc/src/main/resources/com/fortify/cli/ssc/i18n/SSCMessages.properties b/fcli-core/fcli-ssc/src/main/resources/com/fortify/cli/ssc/i18n/SSCMessages.properties index 247df255de..503d6b3a36 100644 --- a/fcli-core/fcli-ssc/src/main/resources/com/fortify/cli/ssc/i18n/SSCMessages.properties +++ b/fcli-core/fcli-ssc/src/main/resources/com/fortify/cli/ssc/i18n/SSCMessages.properties @@ -333,6 +333,11 @@ fcli.ssc.attribute.get-definition.usage.header = Get attribute definition detail fcli.ssc.attribute.list-definitions.usage.header = List attribute definitions. fcli.ssc.attribute-definition.resolver.nameOrId = Attribute definition name or id. +# fcli ssc data-extract +fcli.ssc.data-extract.usage.header = TODO +fcli.ssc.data-extract.create.usage.header = TODO +fcli.ssc.data-extract.create.template = TODO + # fcli ssc issue fcli.ssc.issue.usage.header = Manage SSC issues (vulnerabilities) and related entities like issue templates, filters and groups. fcli.ssc.issue.count.usage.header = Count application version vulnerabilities by grouping. diff --git a/fcli-other/fcli-bom/build.gradle b/fcli-other/fcli-bom/build.gradle index 236a6cc0ed..b6dcdb95e7 100644 --- a/fcli-other/fcli-bom/build.gradle +++ b/fcli-other/fcli-bom/build.gradle @@ -53,7 +53,11 @@ dependencies { api 'org.junit.jupiter:junit-jupiter-params:5.9.3' api 'org.junit.jupiter:junit-jupiter-engine:5.9.3' - //required for unpacking tar.gz (debricked cli) + // Required for unpacking tar.gz (debricked cli) api('org.apache.commons:commons-compress:1.25.0') + + // Used for processing HTML text returned by SSC/FoD endpoints like issue summaries/details/... + api('org.jsoup:jsoup:1.17.2') + } } \ No newline at end of file diff --git a/fcli-other/fcli-gradle/fcli-java.gradle b/fcli-other/fcli-gradle/fcli-java.gradle index ee1eb57ca8..76ac6766c7 100644 --- a/fcli-other/fcli-gradle/fcli-java.gradle +++ b/fcli-other/fcli-gradle/fcli-java.gradle @@ -60,8 +60,11 @@ dependencies { testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine' testRuntimeOnly 'org.junit.platform:junit-platform-launcher' - //required for unpacking tar.gz (debricked cli) - implementation('org.apache.commons:commons-compress') + // Required for unpacking tar.gz (debricked cli) + implementation('org.apache.commons:commons-compress') + + // Used for processing HTML text returned by SSC/FoD endpoints like issue summaries/details/... + implementation('org.jsoup:jsoup') } compileJava {