chore: Add support for updateable tool definitions, add support for f…

…cli & debricked (#510)

feat: `fcli tool`: Add `fcli tool definitions` commands, allowing tool definitions to be updated to make fcli aware of new tool versions that were released after the current fcli release. Customers may also host customized tool definitions, for example allowing for alternative tool download URLs or restricting the set of tool versions available to end users.

feat: `fcli tool`: Add `fcli tool debricked-cli` commands for installing Debricked CLI and managing those installations.

feat: `fcli tool`: Add `fcli tool fcli` commands for installing Fortify CLI and managing those installations.

feat: `fcli tool`: Add `fcli tool * install --base-dir` option to specify the base directory under which all tools will be installed. By default, fcli will now also install tool invocation scripts in a global `<base-dir>/bin` directory, unless the `--no-global-bin` option is specified. This allows for having a single bin-directory on the `PATH`, while managing the actual tool versions being invoked through the `fcli tool * install` commands.

feat: `fcli tool`: By default, the `fcli tool * install` commands will now install tools under the `<user.home>/fortify/tools` base directory (no dot/hidden directory), instead of `<user.home>/.fortify/tools`

feat: `fcli tool`: Deprecate `fcli tool * install --install-dir` option; the new `--base-dir` option is now preferred as it supports new functionality like global bin-scripts.

feat: `fcli tool`: Add `fcli tool * install --uninstall` option to remove existing tool installations while installing a new tool version, allowing for easy tool upgrades.

ftest: Update functional tests, add new tests

chore: Internal refactoring & code improvements

.github/workflows/functional-tests.yml

@@ -54,20 +54,20 @@ jobs:
           mv artifact/fcli-ftest.jar .
           case "${{ matrix.type }}" in
             "java" )
-              java -jar fcli-ftest.jar -Dft.fcli=build -Dft.run=core,config ;;
+              java -jar fcli-ftest.jar -Dft.fcli=build -Dft.run=core,config,tool ;;
             "jar" )
-              java -jar fcli-ftest.jar -Dft.fcli=fcli.jar -Dft.run=core,config ;;
+              java -jar fcli-ftest.jar -Dft.fcli=fcli.jar -Dft.run=core,config,tool ;;
             "native" )
               case "${{ matrix.os }}" in
                 "ubuntu-latest" ) 
                   tar -zxvf fcli-linux.tgz
-                  java -jar fcli-ftest.jar -Dft.fcli=./fcli -Dft.run=core,config ;;
+                  java -jar fcli-ftest.jar -Dft.fcli=./fcli -Dft.run=core,config,tool ;;
                 "windows-latest" )
                   7z e fcli-windows.zip
-                  java -jar fcli-ftest.jar -Dft.fcli=fcli.exe -Dft.run=core,config ;;
+                  java -jar fcli-ftest.jar -Dft.fcli=fcli.exe -Dft.run=core,config,tool ;;
                 "macos-latest" )
                   tar -zxvf fcli-mac.tgz
-                  java -jar fcli-ftest.jar -Dft.fcli=./fcli -Dft.run=core,config ;;
+                  java -jar fcli-ftest.jar -Dft.fcli=./fcli -Dft.run=core,config,tool ;;
               esac ;;
           esac 
             
@@ -153,4 +153,4 @@ jobs:
           name: test-log
           path: test-*.log
 
-
+

.gitignore

@@ -20,6 +20,9 @@ build/
 # We want to include everything in our functional test resources
 !**/ftest/resources/runtime/**/*
 
+# automatically downloaded during gradle build
+fcli-core/fcli-tool/src/main/resources/com/fortify/cli/tool/config/tool-definitions.yaml.zip
+
 ### STS ###
 .apt_generated
 .classpath

fcli-core/fcli-app/src/main/java/com/fortify/cli/app/runner/util/FortifyCLIStaticInitializer.java

@@ -30,6 +30,7 @@
 import com.fortify.cli.sc_sast.scan.helper.SCSastControllerScanJobArtifactState;
 import com.fortify.cli.sc_sast.scan.helper.SCSastControllerScanJobState;
 import com.fortify.cli.ssc.artifact.helper.SSCArtifactStatus;
+import com.fortify.cli.tool._common.helper.ToolUninstaller;
 
 import lombok.AccessLevel;
 import lombok.Getter;
@@ -48,6 +49,7 @@ public final class FortifyCLIStaticInitializer {
     private static final FortifyCLIStaticInitializer instance = new FortifyCLIStaticInitializer(); 
 
     public void initialize() {
+        ToolUninstaller.deleteAllPending();
         initializeTrustStore();
         initializeLocale();
         initializeFoDProperties();

fcli-core/fcli-common/src/main/java/com/fortify/cli/common/cli/mixin/CommonOptionMixins.java

@@ -45,10 +45,12 @@ public void checkConfirmed(Object... promptArgs) {
             if (!confirmed) {
                 CommandSpec spec = commandHelper.getCommandSpec();
                 if ( System.console()==null ) {
-                    throw new ParameterException(spec.commandLine(), "Missing option: Confirm operation with -y / --confirm (interactive prompt not available)");
+                    throw new ParameterException(spec.commandLine(), 
+                            PicocliSpecHelper.getRequiredMessageString(spec, 
+                                    "error.missing.confirmation", getPlainPrompt(spec, promptArgs).replace("\n", "\n  ")));
                 } else {
                     String expectedResponse = PicocliSpecHelper.getRequiredMessageString(spec, "expectedConfirmPromptResponse");
-                    String response = System.console().readLine(getPrompt(promptArgs));
+                    String response = System.console().readLine(getPrompt(spec, promptArgs));
                     if ( response.equalsIgnoreCase(expectedResponse) ) {
                         return;
                     } else {
@@ -58,19 +60,22 @@ public void checkConfirmed(Object... promptArgs) {
             }
         }
 
-        private String getPrompt(Object... promptArgs) {
-            CommandSpec spec = commandHelper.getCommandSpec();
-            String promptFormat = PicocliSpecHelper.getMessageString(spec, "confirmPrompt");
-            if ( StringUtils.isBlank(promptFormat) ) {
+        private String getPrompt(CommandSpec spec, Object... promptArgs) {
+            String prompt = getPlainPrompt(spec, promptArgs);
+            String promptOptions = PicocliSpecHelper.getRequiredMessageString(spec, "confirmPromptOptions");
+            return String.format("%s (%s) ", prompt, promptOptions);
+        }
+
+        private String getPlainPrompt(CommandSpec spec, Object... promptArgs) {
+            String prompt = PicocliSpecHelper.getMessageString(spec, "confirmPrompt", promptArgs);
+            if ( StringUtils.isBlank(prompt) ) {
                 String[] descriptionLines = spec.optionsMap().get("-y").description();
                 if ( descriptionLines==null || descriptionLines.length<1 ) {
                     throw new RuntimeException("No proper description found for generating prompt for --confirm option");
                 }
-                promptFormat = spec.optionsMap().get("-y").description()[0].replaceAll("[. ]+$", "")+"?";
+                prompt = spec.optionsMap().get("-y").description()[0].replaceAll("[. ]+$", "")+"?";
             }
-            String prompt = String.format(promptFormat, promptArgs);
-            String promptOptions = PicocliSpecHelper.getRequiredMessageString(spec, "confirmPromptOptions");
-            return String.format("%s (%s) ", prompt, promptOptions);
+            return prompt;
         }
     }
 }

fcli-core/fcli-common/src/main/java/com/fortify/cli/common/output/cli/mixin/OutputHelperMixins.java

@@ -124,6 +124,11 @@ public static class List extends TableWithQuery {
         public static final String CMD_NAME = "list";
     }
 
+    @Command(aliases = {"ls"})
+    public static class ListNoQuery extends TableNoQuery {
+        public static final String CMD_NAME = "list";
+    }
+
     @Command(aliases = {"lsd"})
     public static class ListDefinitions extends TableWithQuery {
         public static final String CMD_NAME = "list-definitions";

fcli-core/fcli-common/src/main/java/com/fortify/cli/common/output/writer/record/table/TableRecordWriter.java

@@ -68,7 +68,11 @@ private String getTable(String[] fields, String[][] data) {
 
     private String getHeader(String fieldName) {
         String header = getConfig().getMessageResolver().getMessageString("output.header."+fieldName);
-        return header!=null ? header : PropertyPathFormatter.humanReadable(fieldName);
+        return header!=null ? header : PropertyPathFormatter.humanReadable(getNormalizedFieldName(fieldName));
+    }
+
+    private String getNormalizedFieldName(String fieldName) {
+        return fieldName.replaceAll("String$", ""); 
     }
 
     private String[] getFields(ObjectNode firstObjectNode) {

fcli-core/fcli-common/src/main/java/com/fortify/cli/common/rest/unirest/UnirestHelper.java

@@ -0,0 +1,29 @@
+/**
+ * Copyright 2023 Open Text.
+ *
+ * The only warranties for products and services of Open Text 
+ * and its affiliates and licensors (“Open Text”) are as may 
+ * be set forth in the express warranty statements accompanying 
+ * such products and services. Nothing herein should be construed 
+ * as constituting an additional warranty. Open Text shall not be 
+ * liable for technical or editorial errors or omissions contained 
+ * herein. The information contained herein is subject to change 
+ * without notice.
+ */
+package com.fortify.cli.common.rest.unirest;
+
+import java.io.File;
+import java.nio.file.StandardCopyOption;
+
+import com.fortify.cli.common.http.proxy.helper.ProxyHelper;
+
+/**
+ * This class provides utility methods related to Unirest
+ */
+public class UnirestHelper {
+    public static final File download(String fcliModule, String url, File dest) {
+        GenericUnirestFactory.getUnirestInstance(fcliModule, u->ProxyHelper.configureProxy(u, fcliModule, url))
+                .get(url).asFile(dest.getAbsolutePath(), StandardCopyOption.REPLACE_EXISTING).getBody();
+        return dest;
+    }
+}

fcli-core/fcli-common/src/main/java/com/fortify/cli/common/util/FcliBuildPropertiesHelper.java

@@ -14,9 +14,13 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
 import java.util.Properties;
 
 public class FcliBuildPropertiesHelper {
+    private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
     private static final Properties buildProperties = loadProperties();
 
     public static final Properties getBuildProperties() {
@@ -31,15 +35,25 @@ public static final String getFcliVersion() {
         return buildProperties.getProperty("projectVersion", "unknown");
     }
 
-    public static final String getFcliBuildDate() {
+    public static final String getFcliBuildDateString() {
         return buildProperties.getProperty("buildDate", "unknown");
     }
 
+    public static final Date getFcliBuildDate() {
+        var dateString = getFcliBuildDateString();
+        if ( !StringUtils.isBlank(dateString) && !"unknown".equals(dateString) ) {
+            try {
+                return DATE_FORMAT.parse(dateString);
+            } catch (ParseException ignore) {}
+        }
+        return null;
+    }
+
     public static final String getFcliBuildInfo() {
         return String.format("%s version %s, built on %s" 
                 , FcliBuildPropertiesHelper.getFcliProjectName()
                 , FcliBuildPropertiesHelper.getFcliVersion()
-                , FcliBuildPropertiesHelper.getFcliBuildDate());
+                , FcliBuildPropertiesHelper.getFcliBuildDateString());
     }
 
     private static final Properties loadProperties() {

fcli-core/fcli-common/src/main/java/com/fortify/cli/common/util/FileUtils.java

@@ -16,26 +16,55 @@
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.channels.FileChannel;
+import java.nio.charset.Charset;
 import java.nio.file.CopyOption;
+import java.nio.file.FileSystemException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.StandardOpenOption;
 import java.util.Comparator;
 import java.util.stream.Stream;
+import java.util.zip.GZIPInputStream;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 
-import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+
+import lombok.SneakyThrows;
 
 // TODO For now, methods provided in this class are only used by the tools module,
 //      but potentially some methods or the full class could be moved to the common module.
 public final class FileUtils {
     private FileUtils() {}
 
+    public static final InputStream getResourceInputStream(String resourcePath) {
+        return Thread.currentThread().getContextClassLoader().getResourceAsStream(resourcePath);
+    }
+
+    @SneakyThrows
+    public static final String readResourceAsString(String resourcePath, Charset charset) {
+        return new String(readResourceAsBytes(resourcePath), charset);
+    }
+
+    @SneakyThrows
+    public static final byte[] readResourceAsBytes(String resourcePath) {
+        try ( InputStream in = getResourceInputStream(resourcePath) ) {
+            return in.readAllBytes();
+        }
+    }
+
     public static final void copyResource(String resourcePath, Path destinationFilePath, CopyOption... options) {
-        try ( InputStream in = Thread.currentThread().getContextClassLoader().getResourceAsStream(resourcePath) ) {
+        var parent = destinationFilePath.getParent();
+        try {
+            Files.createDirectories(parent);
+        } catch (IOException e) {
+            throw new RuntimeException(String.format("Error creating directory %s", parent), e);
+        }
+        try ( InputStream in = getResourceInputStream(resourcePath) ) {
             Files.copy( in, destinationFilePath, options);
         } catch ( IOException e ) {
             throw new RuntimeException(String.format("Error copying resource %s to %s", resourcePath, destinationFilePath), e);
@@ -47,37 +76,27 @@ public static final void copyResourceToDir(String resourcePath, Path destination
         copyResource(resourcePath, destinationPath.resolve(fileName), options);
     }
 
-    public static final String getFileDigest(File file, String algorithm) {
-        try {
-            MessageDigest digestInstance = MessageDigest.getInstance(algorithm);
-            return bytesToHex(DigestUtils.digest(digestInstance, file));
-        } catch ( IOException | NoSuchAlgorithmException e ) {
-            throw new RuntimeException("Error calculating file digest for file "+file.getAbsolutePath(), e);
+    @SneakyThrows
+    public static final void moveFiles(Path sourcePath, Path targetPath, String regex) {
+        Files.createDirectories(targetPath);
+        try ( var ls = Files.list(sourcePath) ) {
+            ls.map(Path::toFile)
+                .map(File::getName)
+                .filter(name->name.matches(regex))
+                .forEach(name->move(sourcePath.resolve(name), targetPath.resolve(name)));
         }
     }
 
-    public static final String getDigest(String inputName, InputStream input, String algorithm) {
+    public static final void move(Path source, Path target) {
         try {
-            MessageDigest digestInstance = MessageDigest.getInstance(algorithm);
-            return bytesToHex(DigestUtils.digest(digestInstance, input));
-        } catch ( IOException | NoSuchAlgorithmException e ) {
-            throw new RuntimeException("Error calculating file digest for file "+inputName, e);
-        }
-    }
-
-    private static final String bytesToHex(byte[] bytes) {
-        StringBuilder hexString = new StringBuilder(2 * bytes.length);
-        for (int i = 0; i < bytes.length; i++) {
-            String hex = Integer.toHexString(0xff & bytes[i]);
-            if(hex.length() == 1) {
-                hexString.append('0');
-            }
-            hexString.append(hex);
+            Files.move(source, target, StandardCopyOption.REPLACE_EXISTING);
+        } catch (IOException e) {
+            throw new RuntimeException(String.format("Error moving %s to %s", source, target), e);
         }
-        return hexString.toString();
     }
 
-    public static final void extractZip(File zipFile, Path targetDir) throws IOException {
+    @SneakyThrows
+    public static final void extractZip(File zipFile, Path targetDir) {
         try (FileInputStream fis = new FileInputStream(zipFile); ZipInputStream zipIn = new ZipInputStream(fis)) {
             for (ZipEntry ze; (ze = zipIn.getNextEntry()) != null; ) {
                 Path resolvedPath = targetDir.resolve(ze.getName()).normalize();
@@ -89,17 +108,72 @@ public static final void extractZip(File zipFile, Path targetDir) throws IOExcep
                     Files.createDirectories(resolvedPath);
                 } else {
                     Files.createDirectories(resolvedPath.getParent());
-                    Files.copy(zipIn, resolvedPath);
+                    Files.copy(zipIn, resolvedPath, StandardCopyOption.REPLACE_EXISTING);
+                }
+            }
+        }
+    }
+
+    @SneakyThrows
+    public static final void extractTarGZ(File tgzFile, Path targetDir) {
+        try (InputStream source = Files.newInputStream(tgzFile.toPath());
+                GZIPInputStream gzip = new GZIPInputStream(source);
+                TarArchiveInputStream tar = new TarArchiveInputStream(gzip)) {
+
+            TarArchiveEntry entry;
+            while ((entry = tar.getNextEntry()) != null) {
+                Path extractTo = targetDir.resolve(entry.getName());
+                if(entry.isDirectory()) {
+                    Files.createDirectories(extractTo);
+                } else {
+                    Files.copy(tar, extractTo, StandardCopyOption.REPLACE_EXISTING);
                 }
             }
         }
     }
 
-    public static final void deleteRecursive(Path installPath) throws IOException {
-        try (Stream<Path> walk = Files.walk(installPath)) {
+    /**
+     * Recursively delete the given path. As a best practice, this method should
+     * only be invoked if {@link #isDirPathInUse(Path)} returns false. The
+     * deleteRecursive() method itself doesn't invoke {@link #isDirPathInUse(Path)}
+     * for performance reasons, as callers may wish to explicitly check whether
+     * any files are in use in order to perform some alternative action.
+     * @param path
+     */
+    @SneakyThrows
+    public static final void deleteRecursive(Path path) {
+        try (Stream<Path> walk = Files.walk(path)) {
             walk.sorted(Comparator.reverseOrder())
                 .map(Path::toFile)
                 .forEach(File::delete);
         }
     }
+
+    @SneakyThrows
+    public static final boolean isDirPathInUse(Path path) {
+        if ( isDirPathInUseByCurrentExecutable(path) ) { return true; }
+        try (Stream<Path> walk = Files.walk(path)) {
+            return walk.anyMatch(FileUtils::isFilePathInUse);
+        }
+    }
+
+    @SneakyThrows
+    public static final boolean isDirPathInUseByCurrentExecutable(Path path) {
+        var currentExecutablePath = Path.of(FileUtils.class.getProtectionDomain().getCodeSource().getLocation().toURI());
+        return currentExecutablePath.normalize().startsWith(path.normalize());
+    }
+
+    @SneakyThrows
+    public static final boolean isFilePathInUse(Path path) {
+        if ( path.toFile().isFile() ) {
+            try ( var fc = FileChannel.open(path, StandardOpenOption.APPEND) ) {
+                if ( fc.tryLock()==null ) {
+                    return true;
+                }
+            } catch ( FileSystemException e ) {
+                return true;
+            }
+        }
+        return false;
+    }
 }