chore: Partial data-extract implementation

fcli-core/fcli-common/src/main/java/com/fortify/cli/common/cli/mixin/CommonOptionMixins.java

@@ -13,13 +13,33 @@
 package com.fortify.cli.common.cli.mixin;
 
 import java.io.File;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Deque;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Stack;
+import java.util.function.Consumer;
+import java.util.function.Function;
 
+import com.formkiq.graalvm.annotations.Reflectable;
 import com.fortify.cli.common.util.PicocliSpecHelper;
 import com.fortify.cli.common.util.StringUtils;
+import com.github.freva.asciitable.AsciiTable;
+import com.github.freva.asciitable.Column;
+import com.github.freva.asciitable.HorizontalAlign;
 
+import lombok.Data;
 import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.IParameterPreprocessor;
 import picocli.CommandLine.Mixin;
+import picocli.CommandLine.Model.ArgSpec;
 import picocli.CommandLine.Model.CommandSpec;
+import picocli.CommandLine.Model.UnmatchedArgsBinding;
 import picocli.CommandLine.Option;
 import picocli.CommandLine.ParameterException;
 
@@ -78,4 +98,153 @@ private String getPlainPrompt(CommandSpec spec, Object... promptArgs) {
             return prompt;
         }
     }
+
+    /**
+     * This mixin allows for collecting any unmatched command line options 
+     * and parsing them into an [option name]=[value] map.
+     */
+    @Command(preprocessor = OptionParametersMixinPreprocessor.class)
+    public static final class OptionParametersMixin {
+        @Getter private Map<String, String> options = new LinkedHashMap<>();
+        @Getter private Map<String, String> optionNameToCurrentArgMap = new LinkedHashMap<>();
+        @Mixin private CommandHelperMixin commandHelper;
+
+        public final <T> T setUnmatchedArgs(T arg) {
+            parse((String[])arg);
+            return arg;
+        }
+
+        public final OptionParameterHelper getHelper(Consumer<OptionParameterHelper> helperConfigurer) {
+            var helper = new OptionParameterHelper();
+            helperConfigurer.accept(helper);
+            return helper;
+        }
+
+        public final class OptionParameterHelper {
+            private final Map<String, String> optionsWithDescriptions = new LinkedHashMap<>();
+            private final Map<String, Function<OptionParameterValidatorArgs, String>> optionValidators = new LinkedHashMap<>();
+            private final List<String> validationErrors = new ArrayList<>();
+            @Setter private String unknownOptionFormat = "Unknown option: %s";
+            @Setter private String validationExceptionFormat = "Invalid options:\n %s\nSupported options:\n%s\n";
+
+            public final void addSupportedOption(String name, String description) {
+                optionsWithDescriptions.put(name, description);
+            }
+
+            public final void addOptionValidator(String name, Function<OptionParameterValidatorArgs, String> validator) {
+                optionValidators.put(name, validator);
+            }
+
+            public final void requiredOption(String name, String validationErrorFormat) {
+                addOptionValidator(name, args->StringUtils.isBlank(args.getValue())
+                        ? String.format(validationErrorFormat, args.getArg())
+                        : null);
+            }
+
+            public final void validate() {
+                for ( var e : optionNameToCurrentArgMap.entrySet() ) {
+                    if ( !optionsWithDescriptions.containsKey(e.getKey()) ) {
+                        validationErrors.add(String.format(unknownOptionFormat, e.getValue()));
+                    }
+                }
+                optionValidators.entrySet().forEach(this::validate);
+                if ( validationErrors.size()>0 ) {
+                    var errorsString = String.join("\n ", validationErrors);
+                    var supportedOptionsString = getSupportedOptionsTable();
+                    var msg = String.format(validationExceptionFormat, errorsString, supportedOptionsString);
+                    throw new ParameterException(commandHelper.getCommandSpec().commandLine(), msg);
+                }
+            }
+
+            public final String getSupportedOptionsTable() {
+                return AsciiTable.builder()
+                    .border(AsciiTable.NO_BORDERS)
+                    .data(new Column[] {
+                            new Column().dataAlign(HorizontalAlign.LEFT),
+                            new Column().dataAlign(HorizontalAlign.LEFT),
+                        },
+                        optionsWithDescriptions.entrySet().stream()
+                            .map(e->new String[] {asArg(e.getKey()), e.getValue()})
+                            .toList().toArray(String[][]::new))
+                    .asString();
+            }
+
+            private static final String asArg(String name) {
+                return name.length()==1 ? "-"+name : "--"+name;
+            }
+
+            private final void validate(Map.Entry<String, Function<OptionParameterValidatorArgs, String>> entry) {
+                var name = entry.getKey();
+                var arg = asArg(name);
+                var value = options.get(name);
+                var validationError = entry.getValue().apply(new OptionParameterValidatorArgs(name, arg, value));
+                if ( StringUtils.isNotBlank(validationError) ) {
+                    validationErrors.add(validationError);
+                }
+            }
+        }
+
+        @Data
+        public static final class OptionParameterValidatorArgs {
+            private final String name;
+            private final String arg;
+            private final String value;
+        }
+
+        private final void parse(String[] argsArray) {
+            var args = asDeque(argsArray);
+            while ( !args.isEmpty() ) {
+                var arg = args.pop();
+                if ( !arg.startsWith("-") ) {
+                    throw new IllegalArgumentException("Unknown command line option: "+arg);
+                } else {
+                    var optName = arg.replaceFirst("-+", "");
+                    getOptionNameToCurrentArgMap().put(optName, arg);
+                    var nextArg = args.peek(); 
+                    if ( nextArg==null || nextArg.startsWith("-") ) {
+                        options.put(optName, "");
+                    } else {
+                        options.put(optName, args.pop());
+                    }
+                }
+            }
+        }
+
+        private final Deque<String> asDeque(String[] args) {
+            Deque<String> result = new ArrayDeque<>();
+            // Split --opt=value into separate args on the Deque
+            for ( var arg: args ) {
+                if ( arg.startsWith("--") ) {
+                    // Allow for --opt=val, adding option and value as separate args
+                    var elts = arg.split("=", 2);
+                    result.add(elts[0]);
+                    if ( elts.length==2 ) { result.add(elts[1]); }
+                } else if ( arg.startsWith("-") ) {
+                    result.add(arg.substring(0,2));
+                    if ( arg.length()>2 ) {
+                        var val = arg.substring(2);
+                        if ( val.startsWith("=") ) { val = val.substring(1); }
+                        result.add(val);
+                    }
+                } else {
+                    result.add(arg);
+                }
+            }
+            return result;
+        }
+    }
+
+    @Reflectable @NoArgsConstructor
+    static final class OptionParametersMixinPreprocessor implements IParameterPreprocessor {
+        @Override
+        public final boolean preprocess(Stack<String> args, CommandSpec commandSpec, ArgSpec argSpec, Map<String, Object> info) {
+            var mixin = commandSpec.mixins().values().stream()
+                    .map(CommandSpec::userObject)
+                    .filter(o->o instanceof OptionParametersMixin)
+                    .map(OptionParametersMixin.class::cast)
+                    .findFirst().get();
+            commandSpec.addUnmatchedArgsBinding(UnmatchedArgsBinding.forStringArrayConsumer(mixin::setUnmatchedArgs));
+            return false;
+        }
+    }
 }

fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/cli/cmd/AbstractDataExtractCreateCommand.java

@@ -0,0 +1,82 @@
+/*******************************************************************************
+ * Copyright 2021, 2023 Open Text.
+ *
+ * The only warranties for products and services of Open Text 
+ * and its affiliates and licensors ("Open Text") are as may 
+ * be set forth in the express warranty statements accompanying 
+ * such products and services. Nothing herein should be construed 
+ * as constituting an additional warranty. Open Text shall not be 
+ * liable for technical or editorial errors or omissions contained 
+ * herein. The information contained herein is subject to change 
+ * without notice.
+ *******************************************************************************/
+package com.fortify.cli.common.data_extract.cli.cmd;
+
+import java.util.List;
+
+import com.fortify.cli.common.cli.cmd.AbstractRunnableCommand;
+import com.fortify.cli.common.cli.mixin.CommonOptionMixins.OptionParametersMixin;
+import com.fortify.cli.common.cli.mixin.CommonOptionMixins.OptionParametersMixin.OptionParameterHelper;
+import com.fortify.cli.common.data_extract.helper.DataExtractTemplateDescriptor;
+import com.fortify.cli.common.data_extract.helper.DataExtractTemplateHelper;
+import com.fortify.cli.common.data_extract.helper.DataExtractTemplateRunner;
+import com.fortify.cli.common.progress.cli.mixin.ProgressWriterFactoryMixin;
+import com.fortify.cli.common.progress.helper.IProgressWriterI18n;
+import com.fortify.cli.common.util.DisableTest;
+import com.fortify.cli.common.util.DisableTest.TestType;
+
+import picocli.CommandLine.Mixin;
+import picocli.CommandLine.Option;
+
+public abstract class AbstractDataExtractCreateCommand extends AbstractRunnableCommand implements Runnable {
+    @Option(names={"-t", "--template"}, required=true, descriptionKey="fcli.data-extract.create.template") private String template;
+    @Option(names="--template-help", required=false, arity = "0", descriptionKey="fcli.data-extract.create.template-help") private boolean isTemplateHelpRequested; 
+    @DisableTest({TestType.MULTI_OPT_SPLIT, TestType.MULTI_OPT_PLURAL_NAME, TestType.OPT_LONG_NAME})
+    @Option(names="--<template-parameter>", paramLabel="<value>", descriptionKey="fcli.data-extract.create.template-parameter") 
+    private List<String> dummyForSynopsis;
+    @Mixin private OptionParametersMixin templateParameters;
+    @Mixin private ProgressWriterFactoryMixin progressWriterFactory;
+
+    @Override
+    public final void run() {
+        initMixins();
+        Runnable delayedConsoleWriter = null;
+        try ( var progressWriter = progressWriterFactory.create() ) {
+            progressWriter.writeProgress("Loading template %s", template);
+            var templateDescriptor = DataExtractTemplateHelper.load(getType(), template);
+            try ( var templateExecutor = DataExtractTemplateRunner.builder()
+                .template(templateDescriptor)
+                .inputParameters(templateParameters.getOptions())
+                .progressWriter(progressWriter).build() ) 
+            {
+                delayedConsoleWriter = run(templateDescriptor, templateExecutor, progressWriter);
+            }
+        }
+        delayedConsoleWriter.run();
+    }
+
+    private Runnable run(DataExtractTemplateDescriptor templateDescriptor, DataExtractTemplateRunner templateExecutor, IProgressWriterI18n progressWriter) {
+        configure(templateExecutor);
+        OptionParameterHelper helper = templateParameters.getHelper(templateExecutor::configureOptionParameterHelper);
+        if ( isTemplateHelpRequested ) {
+            String templateHelp = getTemplateHelp(templateDescriptor, helper);
+            return ()->{System.out.println(templateHelp);};
+        } else {
+            helper.validate();
+            progressWriter.writeProgress("Executing template %s", template);
+            return templateExecutor.execute();
+        }
+    }
+
+    private final String getTemplateHelp(DataExtractTemplateDescriptor template, OptionParameterHelper helper) {
+        return String.format(
+            "\nTemplate: %s\n"+
+            "\n%s\n"+
+            "\nTemplate options:\n"+
+            "%s",
+            template.getName(), template.getDescription(), helper.getSupportedOptionsTable());
+    }
+
+    protected abstract String getType();
+    protected abstract void configure(DataExtractTemplateRunner templateExecutor);
+}

fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/cli/cmd/AbstractDataExtractListTemplatesCommand.java

@@ -0,0 +1,37 @@
+/*******************************************************************************
+ * Copyright 2021, 2023 Open Text.
+ *
+ * The only warranties for products and services of Open Text 
+ * and its affiliates and licensors ("Open Text") are as may 
+ * be set forth in the express warranty statements accompanying 
+ * such products and services. Nothing herein should be construed 
+ * as constituting an additional warranty. Open Text shall not be 
+ * liable for technical or editorial errors or omissions contained 
+ * herein. The information contained herein is subject to change 
+ * without notice.
+ *******************************************************************************/
+package com.fortify.cli.common.data_extract.cli.cmd;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fortify.cli.common.data_extract.helper.DataExtractTemplateHelper;
+import com.fortify.cli.common.json.JsonHelper;
+import com.fortify.cli.common.output.cli.cmd.AbstractOutputCommand;
+import com.fortify.cli.common.output.cli.cmd.IJsonNodeSupplier;
+
+public abstract class AbstractDataExtractListTemplatesCommand extends AbstractOutputCommand implements IJsonNodeSupplier {
+    @Override
+    public final JsonNode getJsonNode() {
+        return DataExtractTemplateHelper.list(getType())
+                .map(JsonHelper.getObjectMapper()::valueToTree)
+                .map(ObjectNode.class::cast)
+                .collect(JsonHelper.arrayNodeCollector());
+    }    
+    @Override
+    public final boolean isSingular() {
+        return false;
+    }
+    protected abstract String getType();
+
+
+}

fcli-core/fcli-common/src/main/java/com/fortify/cli/common/data_extract/helper/DataExtractSpelFunctions.java

@@ -0,0 +1,83 @@
+/*******************************************************************************
+ * Copyright 2021, 2023 Open Text.
+ *
+ * The only warranties for products and services of Open Text 
+ * and its affiliates and licensors ("Open Text") are as may 
+ * be set forth in the express warranty statements accompanying 
+ * such products and services. Nothing herein should be construed 
+ * as constituting an additional warranty. Open Text shall not be 
+ * liable for technical or editorial errors or omissions contained 
+ * herein. The information contained herein is subject to change 
+ * without notice.
+ *******************************************************************************/
+package com.fortify.cli.common.data_extract.helper;
+
+import java.util.List;
+
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.safety.Safelist;
+
+import com.formkiq.graalvm.annotations.Reflectable;
+import com.fortify.cli.common.util.StringUtils;
+
+import lombok.NoArgsConstructor;
+
+@Reflectable @NoArgsConstructor
+public class DataExtractSpelFunctions {
+    public static final String join(String separator, List<Object> elts) {
+        switch (separator) {
+        case "\\n": separator="\n"; break;
+        case "\\t": separator="\t"; break;
+        }
+        return elts==null ? "" : String.join(separator, elts.stream().map(Object::toString).toList());
+    }
+
+    /**
+     * Convenience method to throw an exception if an expression evaluates to false
+     * @param throwError true if error should be thrown, false otherwise
+     * @param msg Message for exception to be thrown
+     * @return true if throwError is false
+     * @throws IllegalStateException with the given message if throwError is true
+     */
+    public static final boolean check(boolean throwError, String msg) {
+        if ( throwError ) {
+            throw new IllegalStateException(msg);
+        } else {
+            return true;
+        }
+    }
+
+    /**
+     * Abbreviate the given text to the given maximum width
+     * @param text to abbreviate
+     * @param maxWidth Maximum width
+     * @return Abbreviated text 
+     */
+    public static final String abbreviate(String text, int maxWidth) {
+        return StringUtils.abbreviate(text, maxWidth);
+    }
+
+    /**
+     * @param html to be converted to plain text
+     * @return Formatted plain-text string for the given HTML contents
+     */
+    public static final String htmlToText(String html) {
+        if( html==null ) { return null; }
+        Document document = Jsoup.parse(html);
+        document.outputSettings(new Document.OutputSettings().prettyPrint(false));//makes html() preserve linebreaks and spacing
+        document.select("br").append("\\n");
+        document.select("p").prepend("\\n\\n");
+        String s = document.html().replaceAll("\\\\n", "\n");
+        return Jsoup.clean(s, "", Safelist.none(), new Document.OutputSettings().prettyPrint(false));
+    }
+
+    /**
+     * @param html to be converted to plain text
+     * @return Single line of plain text for the given HTML contents
+     */
+    public static final String htmlToSingleLineText(String html) {
+        if( html==null ) { return null; }
+        return Jsoup.clean(html, "", Safelist.none());
+    }
+}