Maven Build Cache Extension: add support for state file suffix (#1063)

Introduce a new mechanism that can be used to aggregate partial caches.

E.g.

```
build -> | javadoc part#1 -|
         | javadoc part#2  |-> release
         | javadoc part#3 -|
```

The javadoc jobs can use -Dcache.recordSuffix=javadoc to produce augmented state files with a different name.
A simple glob like `**/target/state-javadoc.xml` to archive only the new state.

The release job can download all javadoc artifacts, without conflicts with the state files (unless the javadoc jobs overlap).
The javadoc state files can be loaded with -Dcache.loadSuffixes=javadoc, they will be merged with the default state files.

maven-plugins/build-cache-maven-extension/README.md

@@ -80,6 +80,23 @@ The configuration resides in `.mvn/cache-config.xml`.
         Can be overridden with -Dcache.enabled=true
      -->
     <enabled>false</enabled>
+    <!--
+        Enable state recording (target/state.xml).
+        Can be overridden with -Dcache.record=false
+    -->
+    <record>true</record>
+    <!--
+        Load additional state files (target/state-{suffix}.xml).
+        Can be overridden with -Dcache.loadSuffixes=foo,bar
+    -->
+    <loadSuffixes>
+        <suffix>foo</suffix>
+    </loadSuffixes>
+    <!--
+        Add a suffix to the state file name used for recording (target/state-{suffix}.xml).
+        Can be overridden with -Dcache.recordSuffix=bar
+    -->
+    <recordSuffix>foo</recordSuffix>
     <!-- Per project configuration -->
     <lifecycleConfig>
         <!-- Indicate if the project files checksum should be computed -->
@@ -156,12 +173,14 @@ The configuration resides in `.mvn/cache-config.xml`.
 
 ### Properties
 
-| Property      | Type    | Default<br/>Value | Description                              |
-|---------------|---------|-------------------|------------------------------------------|
-| cache.enabled | Boolean | `false`           | Enables the extension                    |
-| cache.record  | Boolean | `false`           | Update the recorded cache state          |
-| reactorRule   | String  | `null`            | The reactor rule to use                  |
-| moduleSet     | String  | `null`            | The moduleset in the reactor rule to use |
+| Property           | Type    | Default<br/>Value | Description                                    |
+|--------------------|---------|-------------------|------------------------------------------------|
+| cache.enabled      | Boolean | `false`           | Enables the extension                          |
+| cache.record       | Boolean | `false`           | Update the recorded cache state                |
+| cache.loadSuffixes | List    | `[]`              | List of additional state file suffixes to load |
+| cache.recordSuffix | String  | `null`            | State file suffix to use                       |
+| reactorRule        | String  | `null`            | The reactor rule to use                        |
+| moduleSet          | String  | `null`            | The moduleset in the reactor rule to use       |
 
 ## General usage
 

maven-plugins/build-cache-maven-extension/src/main/java/io/helidon/build/maven/cache/CacheConfig.java

@@ -16,9 +16,13 @@
 package io.helidon.build.maven.cache;
 
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Properties;
+import java.util.stream.Collectors;
 
 import io.helidon.build.common.Lists;
 import io.helidon.build.common.Strings;
@@ -36,11 +40,13 @@ public final class CacheConfig {
 
     private final boolean enabled;
     private final boolean record;
+    private final String recordSuffix;
+    private final List<String> loadSuffixes;
     private final String reactorRule;
     private final String moduleSet;
     private final boolean enableChecksums;
     private final boolean includeAllChecksums;
-    private final List<LifecycleConfig> lifecyleConfig = new ArrayList<>();
+    private final List<LifecycleConfig> lifecycleConfig = new ArrayList<>();
     private final List<ReactorRule> reactorRules = new ArrayList<>();
 
     /**
@@ -354,13 +360,29 @@ public String toString() {
         boolean enabled = parseBoolean(enabledValue, false);
         String recordValue = stringProperty(sysProps, userProps, "cache.record");
         boolean record = parseBoolean(recordValue, true);
+        String loadSuffixesValue = stringProperty(sysProps, userProps, "cache.loadSuffixes");
+        List<String> loadSuffixes;
+        if (loadSuffixesValue != null) {
+            loadSuffixes = Arrays.stream(loadSuffixesValue.split(","))
+                    .filter(Strings::isValid)
+                    .collect(Collectors.toList());
+        } else {
+            loadSuffixes = List.of();
+        }
+        String recordSuffix = stringProperty(sysProps, userProps, "cache.recordSuffix");
         if (xmlElt != null) {
             if (enabledValue == null) {
                 enabled = booleanElement(xmlElt, "enabled", false);
             }
             if (recordValue == null) {
                 record = booleanElement(xmlElt, "record", true);
             }
+            if (loadSuffixesValue == null) {
+                loadSuffixes = stringListElement(xmlElt, "loadSuffixes");
+            }
+            if (recordSuffix == null) {
+                recordSuffix = xmlElt.child("recordSuffix").map(XMLElement::value).orElse(null);
+            }
             XMLElement lifecycleConfigElt = xmlElt.child("lifecycleConfig").orElse(null);
             if (lifecycleConfigElt != null) {
                 enableChecksums = booleanElement(lifecycleConfigElt, "enableChecksums", false);
@@ -373,7 +395,7 @@ record = booleanElement(xmlElt, "record", true);
                     List<String> executionsIncludes = stringListElement(projectElt, "executionsIncludes");
                     List<String> executionsExcludes = stringListElement(projectElt, "executionsExcludes");
                     List<String> projectFilesExcludes = stringListElement(projectElt, "projectFilesExcludes");
-                    lifecyleConfig.add(new LifecycleConfig(path, glob, regex, projectEnabled, executionsIncludes,
+                    lifecycleConfig.add(new LifecycleConfig(path, glob, regex, projectEnabled, executionsIncludes,
                             executionsExcludes, projectFilesExcludes));
                 }
             }
@@ -393,6 +415,8 @@ record = booleanElement(xmlElt, "record", true);
         this.includeAllChecksums = includeAllChecksums;
         this.enabled = enabled;
         this.record = record;
+        this.recordSuffix = recordSuffix;
+        this.loadSuffixes = Collections.unmodifiableList(loadSuffixes);
         this.reactorRule = stringProperty(sysProps, userProps, "reactorRule");
         this.moduleSet = stringProperty(sysProps, userProps, "moduleSet");
     }
@@ -433,6 +457,24 @@ public boolean record() {
         return record;
     }
 
+    /**
+     * Get the state file suffixes to load.
+     *
+     * @return list, never {@code null}
+     */
+    public List<String> loadSuffixes() {
+        return loadSuffixes;
+    }
+
+    /**
+     * Get suffix to use for the recorded state files.
+     *
+     * @return Optional, never {@code null}
+     */
+    public Optional<String> recordSuffix() {
+        return Optional.ofNullable(recordSuffix);
+    }
+
     /**
      * Get the {@link ReactorRule} name.
      *
@@ -456,8 +498,8 @@ public String moduleSet() {
      *
      * @return list
      */
-    List<LifecycleConfig> lifecyleConfig() {
-        return lifecyleConfig;
+    List<LifecycleConfig> lifecycleConfig() {
+        return lifecycleConfig;
     }
 
     /**

maven-plugins/build-cache-maven-extension/src/main/java/io/helidon/build/maven/cache/CacheConfigManager.java

@@ -77,7 +77,7 @@ public CacheConfig cacheConfig() {
     public LifecycleConfig lifecycleConfig(MavenProject project) {
         return lifecycleConfigCache.computeIfAbsent(project, p -> {
             String projectPath = normalizePath(root().relativize(p.getFile().toPath().toAbsolutePath()));
-            return cacheConfig().lifecyleConfig().stream()
+            return cacheConfig().lifecycleConfig().stream()
                     .filter(c -> c.matches(projectPath))
                     .findFirst()
                     .orElse(LifecycleConfig.EMPTY);

maven-plugins/build-cache-maven-extension/src/main/java/io/helidon/build/maven/cache/ProjectState.java

@@ -25,6 +25,7 @@
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Properties;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -46,7 +47,6 @@
  */
 final class ProjectState {
 
-    private static final String STATE_FILE_NAME = "state.xml";
     private static final DefaultMavenProjectHelper PROJECT_HELPER = new DefaultMavenProjectHelper();
 
     private final Properties properties;
@@ -142,15 +142,16 @@ List<ExecutionEntry> executions() {
     /**
      * Load the project state from file.
      *
-     * @param project maven project
+     * @param project       maven project
+     * @param stateFileName state file name
      * @return state if state file exists, or {@code null}
      * @throws IOException  if an IO error occurs
      * @throws XMLException if a parsing error occurs
      */
-    static ProjectState load(MavenProject project) throws IOException, XMLException {
+    static ProjectState load(MavenProject project, String stateFileName) throws IOException, XMLException {
         return load(project.getModel().getProjectDirectory().toPath()
                 .resolve(project.getModel().getBuild().getDirectory())
-                .resolve(STATE_FILE_NAME));
+                .resolve(stateFileName));
     }
 
     /**
@@ -206,16 +207,17 @@ static ProjectState load(Path stateFile) throws IOException, XMLException {
     /**
      * Save the project state.
      *
-     * @param project Maven project
+     * @param project       Maven project
+     * @param stateFileName state file name
      * @throws IOException if an IO error occurs
      */
-    void save(MavenProject project) throws IOException {
+    void save(MavenProject project, String stateFileName) throws IOException {
         Model model = project.getModel();
         Path buildDir = model.getProjectDirectory().toPath().resolve(model.getBuild().getDirectory());
         if (!Files.exists(buildDir)) {
             Files.createDirectories(buildDir);
         }
-        save(buildDir.resolve(STATE_FILE_NAME));
+        save(buildDir.resolve(stateFileName));
     }
 
     /**
@@ -377,6 +379,41 @@ ExecutionEntry findMatchingExecution(ExecutionEntry execution) {
         }).orElse(null);
     }
 
+    /**
+     * Merge two project states.
+     *
+     * @param state1 state1, must be non {@code null}
+     * @param state2 state1, must be non {@code null}
+     * @return ProjectState
+     */
+    static ProjectState merge(ProjectState state1, ProjectState state2) {
+        Properties properties = new Properties();
+        properties.putAll(state1.properties);
+        properties.putAll(state2.properties);
+        ProjectFiles projectFiles1 = state1.projectFiles;
+        ProjectFiles projectFiles2 = state2.projectFiles;
+        return new ProjectState(
+                properties,
+                Optional.ofNullable(state1.artifact).orElse(state2.artifact),
+                Stream.of(state1.attachedArtifacts.stream(), state2.attachedArtifacts.stream())
+                        .flatMap(Function.identity())
+                        .distinct()
+                        .collect(Collectors.toList()),
+                Stream.of(state1.compileSourceRoots.stream(), state2.compileSourceRoots.stream())
+                        .flatMap(Function.identity())
+                        .distinct()
+                        .collect(Collectors.toList()),
+                Stream.of(state1.testCompileSourceRoots.stream(), state2.testCompileSourceRoots.stream())
+                        .flatMap(Function.identity())
+                        .distinct()
+                        .collect(Collectors.toList()),
+                projectFiles1.lastModified() > projectFiles2.lastModified() ? projectFiles1 : projectFiles2,
+                Stream.of(state1.executions.stream(), state2.executions.stream())
+                        .flatMap(Function.identity())
+                        .distinct()
+                        .collect(Collectors.toList()));
+    }
+
     /**
      * Create a state for the given project and merge it with the existing state for this project.
      *

maven-plugins/build-cache-maven-extension/src/main/java/io/helidon/build/maven/cache/ProjectStateManager.java

@@ -18,6 +18,7 @@
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.util.ArrayDeque;
+import java.util.ArrayList;
 import java.util.Deque;
 import java.util.HashMap;
 import java.util.List;
@@ -27,6 +28,7 @@
 import javax.inject.Named;
 
 import io.helidon.build.common.LazyValue;
+import io.helidon.build.common.Lists;
 import io.helidon.build.common.xml.XMLException;
 
 import org.apache.maven.SessionScoped;
@@ -114,7 +116,8 @@ private Map<MavenProject, ProjectStateStatus> initStates() {
      * @param project Maven project
      */
     public void save(MavenProject project) {
-        if (configManager.cacheConfig().record()) {
+        CacheConfig cacheConfig = configManager.cacheConfig();
+        if (cacheConfig.record()) {
             try {
                 ProjectState projectState = null;
                 ProjectFiles projectFiles = null;
@@ -125,7 +128,9 @@ public void save(MavenProject project) {
                 }
                 List<ExecutionEntry> newExecutions = executionManager.recordedExecutions(project);
                 ProjectState.merge(projectState, project, session, configManager, newExecutions, projectFiles)
-                        .save(project);
+                        .save(project, cacheConfig.recordSuffix()
+                                .map(suffix -> "state-" + suffix + ".xml")
+                                .orElse("state.xml"));
             } catch (IOException | UncheckedIOException ex) {
                 logger.error("Error while saving project state", ex);
             }
@@ -149,12 +154,31 @@ private ProjectStateStatus processState(MavenProject project) {
                     project.getGroupId(),
                     project.getArtifactId()));
         }
-        ProjectState state;
+        List<String> suffixes = cacheConfig.loadSuffixes();
+        List<String> stateFileNames = new ArrayList<>(Lists.map(suffixes, suffix -> "state-" + suffix + ".xml"));
+        stateFileNames.add("state.xml");
+        ProjectState state = null;
         try {
-            state = ProjectState.load(project);
+            for (String stateFileName : stateFileNames) {
+                ProjectState nextState = ProjectState.load(project, stateFileName);
+                if (nextState == null) {
+                    if (logger.isDebugEnabled()) {
+                        logger.debug(String.format("[%s:%s] - state file not found: %s",
+                                project.getGroupId(),
+                                project.getArtifactId(),
+                                stateFileName));
+                    }
+                    continue;
+                }
+                if (state == null) {
+                    state = nextState;
+                } else {
+                    state = ProjectState.merge(state, nextState);
+                }
+            }
             if (state == null) {
                 if (logger.isDebugEnabled()) {
-                    logger.debug(String.format("[%s:%s] - state file not found",
+                    logger.debug(String.format("[%s:%s] - state file(s) not found",
                             project.getGroupId(),
                             project.getArtifactId()));
                 }

maven-plugins/build-cache-maven-extension/src/test/java/io/helidon/build/maven/cache/CacheConfigTest.java

@@ -15,6 +15,7 @@
  */
 package io.helidon.build.maven.cache;
 
+import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.List;
@@ -44,12 +45,15 @@ void testConfig() throws Exception {
         CacheConfig config = new CacheConfig(elt, toProperties(Map.of()), toProperties(Map.of()));
 
         assertThat(config.enabled(), is(true));
+        assertThat(config.record(), is(true));
+        assertThat(config.loadSuffixes(), is(List.of("foo", "bar")));
+        assertThat(config.recordSuffix().orElse(null), is("foo"));
         assertThat(config.enableChecksums(), is(true));
         assertThat(config.includeAllChecksums(), is(true));
 
-        assertThat(config.lifecyleConfig().size(), is(2));
+        assertThat(config.lifecycleConfig().size(), is(2));
 
-        LifecycleConfig lifecycleConfig1 = config.lifecyleConfig().get(0);
+        LifecycleConfig lifecycleConfig1 = config.lifecycleConfig().get(0);
         assertThat(lifecycleConfig1.path(), is("a-path"));
         assertThat(lifecycleConfig1.glob(), is("a-glob"));
         assertThat(lifecycleConfig1.regex(), is("a-regex"));
@@ -58,7 +62,7 @@ void testConfig() throws Exception {
         assertThat(lifecycleConfig1.executionsIncludes(), is(List.of("exec-include")));
         assertThat(lifecycleConfig1.projectFilesExcludes(), is(List.of("project-exclude")));
 
-        LifecycleConfig lifecycleConfig2 = config.lifecyleConfig().get(1);
+        LifecycleConfig lifecycleConfig2 = config.lifecycleConfig().get(1);
         assertThat(lifecycleConfig2.glob(), is("foo/**"));
         assertThat(lifecycleConfig2.enabled(), is(false));
 
@@ -74,4 +78,21 @@ void testConfig() throws Exception {
         assertThat(moduleSet.includes(), is(List.of("module-include")));
         assertThat(moduleSet.excludes(), is(List.of("module-exclude")));
     }
+
+    @Test
+    void testConfigOverride() throws IOException {
+        Path configFile = TestFiles.testResourcePath(CacheConfigTest.class, "cache-config.xml");
+        XMLElement elt = XMLElement.parse(Files.newInputStream(configFile));
+        CacheConfig config = new CacheConfig(elt, toProperties(Map.of(
+                "cache.enabled", "false",
+                "cache.record", "false",
+                "cache.loadSuffixes", "one,two",
+                "cache.recordSuffix", "three"
+        )), toProperties(Map.of()));
+
+        assertThat(config.enabled(), is(false));
+        assertThat(config.record(), is(false));
+        assertThat(config.loadSuffixes(), is(List.of("one", "two")));
+        assertThat(config.recordSuffix().orElse(null), is("three"));
+    }
 }