From 57913719c9c653ec1df2ec91f0c815d910575007 Mon Sep 17 00:00:00 2001 From: Romain Grecourt Date: Wed, 21 Aug 2024 17:35:15 -0700 Subject: [PATCH] Maven Build Cache Extension: add support for state file suffix (#1063) Introduce a new mechanism that can be used to aggregate partial caches. E.g. ``` build -> | javadoc part#1 -| | javadoc part#2 |-> release | javadoc part#3 -| ``` The javadoc jobs can use -Dcache.recordSuffix=javadoc to produce augmented state files with a different name. A simple glob like `**/target/state-javadoc.xml` to archive only the new state. The release job can download all javadoc artifacts, without conflicts with the state files (unless the javadoc jobs overlap). The javadoc state files can be loaded with -Dcache.loadSuffixes=javadoc, they will be merged with the default state files. --- .../build-cache-maven-extension/README.md | 31 ++++++++--- .../build/maven/cache/CacheConfig.java | 50 ++++++++++++++++-- .../build/maven/cache/CacheConfigManager.java | 2 +- .../build/maven/cache/ProjectState.java | 51 ++++++++++++++++--- .../maven/cache/ProjectStateManager.java | 34 +++++++++++-- .../build/maven/cache/CacheConfigTest.java | 27 ++++++++-- .../src/test/resources/cache-config.xml | 6 +++ 7 files changed, 175 insertions(+), 26 deletions(-) diff --git a/maven-plugins/build-cache-maven-extension/README.md b/maven-plugins/build-cache-maven-extension/README.md index 63edf3c94..8d1c70e7b 100644 --- a/maven-plugins/build-cache-maven-extension/README.md +++ b/maven-plugins/build-cache-maven-extension/README.md @@ -80,6 +80,23 @@ The configuration resides in `.mvn/cache-config.xml`. Can be overridden with -Dcache.enabled=true --> false + + true + + + foo + + + foo @@ -156,12 +173,14 @@ The configuration resides in `.mvn/cache-config.xml`. ### Properties -| Property | Type | Default
Value | Description | -|---------------|---------|-------------------|------------------------------------------| -| cache.enabled | Boolean | `false` | Enables the extension | -| cache.record | Boolean | `false` | Update the recorded cache state | -| reactorRule | String | `null` | The reactor rule to use | -| moduleSet | String | `null` | The moduleset in the reactor rule to use | +| Property | Type | Default
Value | Description | +|--------------------|---------|-------------------|------------------------------------------------| +| cache.enabled | Boolean | `false` | Enables the extension | +| cache.record | Boolean | `false` | Update the recorded cache state | +| cache.loadSuffixes | List | `[]` | List of additional state file suffixes to load | +| cache.recordSuffix | String | `null` | State file suffix to use | +| reactorRule | String | `null` | The reactor rule to use | +| moduleSet | String | `null` | The moduleset in the reactor rule to use | ## General usage diff --git a/maven-plugins/build-cache-maven-extension/src/main/java/io/helidon/build/maven/cache/CacheConfig.java b/maven-plugins/build-cache-maven-extension/src/main/java/io/helidon/build/maven/cache/CacheConfig.java index e0fd5876f..6c03a2878 100644 --- a/maven-plugins/build-cache-maven-extension/src/main/java/io/helidon/build/maven/cache/CacheConfig.java +++ b/maven-plugins/build-cache-maven-extension/src/main/java/io/helidon/build/maven/cache/CacheConfig.java @@ -16,9 +16,13 @@ package io.helidon.build.maven.cache; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Objects; +import java.util.Optional; import java.util.Properties; +import java.util.stream.Collectors; import io.helidon.build.common.Lists; import io.helidon.build.common.Strings; @@ -36,11 +40,13 @@ public final class CacheConfig { private final boolean enabled; private final boolean record; + private final String recordSuffix; + private final List loadSuffixes; private final String reactorRule; private final String moduleSet; private final boolean enableChecksums; private final boolean includeAllChecksums; - private final List lifecyleConfig = new ArrayList<>(); + private final List lifecycleConfig = new ArrayList<>(); private final List reactorRules = new ArrayList<>(); /** @@ -354,6 +360,16 @@ public String toString() { boolean enabled = parseBoolean(enabledValue, false); String recordValue = stringProperty(sysProps, userProps, "cache.record"); boolean record = parseBoolean(recordValue, true); + String loadSuffixesValue = stringProperty(sysProps, userProps, "cache.loadSuffixes"); + List loadSuffixes; + if (loadSuffixesValue != null) { + loadSuffixes = Arrays.stream(loadSuffixesValue.split(",")) + .filter(Strings::isValid) + .collect(Collectors.toList()); + } else { + loadSuffixes = List.of(); + } + String recordSuffix = stringProperty(sysProps, userProps, "cache.recordSuffix"); if (xmlElt != null) { if (enabledValue == null) { enabled = booleanElement(xmlElt, "enabled", false); @@ -361,6 +377,12 @@ boolean record = parseBoolean(recordValue, true); if (recordValue == null) { record = booleanElement(xmlElt, "record", true); } + if (loadSuffixesValue == null) { + loadSuffixes = stringListElement(xmlElt, "loadSuffixes"); + } + if (recordSuffix == null) { + recordSuffix = xmlElt.child("recordSuffix").map(XMLElement::value).orElse(null); + } XMLElement lifecycleConfigElt = xmlElt.child("lifecycleConfig").orElse(null); if (lifecycleConfigElt != null) { enableChecksums = booleanElement(lifecycleConfigElt, "enableChecksums", false); @@ -373,7 +395,7 @@ record = booleanElement(xmlElt, "record", true); List executionsIncludes = stringListElement(projectElt, "executionsIncludes"); List executionsExcludes = stringListElement(projectElt, "executionsExcludes"); List projectFilesExcludes = stringListElement(projectElt, "projectFilesExcludes"); - lifecyleConfig.add(new LifecycleConfig(path, glob, regex, projectEnabled, executionsIncludes, + lifecycleConfig.add(new LifecycleConfig(path, glob, regex, projectEnabled, executionsIncludes, executionsExcludes, projectFilesExcludes)); } } @@ -393,6 +415,8 @@ record = booleanElement(xmlElt, "record", true); this.includeAllChecksums = includeAllChecksums; this.enabled = enabled; this.record = record; + this.recordSuffix = recordSuffix; + this.loadSuffixes = Collections.unmodifiableList(loadSuffixes); this.reactorRule = stringProperty(sysProps, userProps, "reactorRule"); this.moduleSet = stringProperty(sysProps, userProps, "moduleSet"); } @@ -433,6 +457,24 @@ public boolean record() { return record; } + /** + * Get the state file suffixes to load. + * + * @return list, never {@code null} + */ + public List loadSuffixes() { + return loadSuffixes; + } + + /** + * Get suffix to use for the recorded state files. + * + * @return Optional, never {@code null} + */ + public Optional recordSuffix() { + return Optional.ofNullable(recordSuffix); + } + /** * Get the {@link ReactorRule} name. * @@ -456,8 +498,8 @@ public String moduleSet() { * * @return list */ - List lifecyleConfig() { - return lifecyleConfig; + List lifecycleConfig() { + return lifecycleConfig; } /** diff --git a/maven-plugins/build-cache-maven-extension/src/main/java/io/helidon/build/maven/cache/CacheConfigManager.java b/maven-plugins/build-cache-maven-extension/src/main/java/io/helidon/build/maven/cache/CacheConfigManager.java index 49d74581d..de14e2fd7 100644 --- a/maven-plugins/build-cache-maven-extension/src/main/java/io/helidon/build/maven/cache/CacheConfigManager.java +++ b/maven-plugins/build-cache-maven-extension/src/main/java/io/helidon/build/maven/cache/CacheConfigManager.java @@ -77,7 +77,7 @@ public CacheConfig cacheConfig() { public LifecycleConfig lifecycleConfig(MavenProject project) { return lifecycleConfigCache.computeIfAbsent(project, p -> { String projectPath = normalizePath(root().relativize(p.getFile().toPath().toAbsolutePath())); - return cacheConfig().lifecyleConfig().stream() + return cacheConfig().lifecycleConfig().stream() .filter(c -> c.matches(projectPath)) .findFirst() .orElse(LifecycleConfig.EMPTY); diff --git a/maven-plugins/build-cache-maven-extension/src/main/java/io/helidon/build/maven/cache/ProjectState.java b/maven-plugins/build-cache-maven-extension/src/main/java/io/helidon/build/maven/cache/ProjectState.java index 6f9334d4e..f509a246f 100644 --- a/maven-plugins/build-cache-maven-extension/src/main/java/io/helidon/build/maven/cache/ProjectState.java +++ b/maven-plugins/build-cache-maven-extension/src/main/java/io/helidon/build/maven/cache/ProjectState.java @@ -25,6 +25,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Properties; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -46,7 +47,6 @@ */ final class ProjectState { - private static final String STATE_FILE_NAME = "state.xml"; private static final DefaultMavenProjectHelper PROJECT_HELPER = new DefaultMavenProjectHelper(); private final Properties properties; @@ -142,15 +142,16 @@ List executions() { /** * Load the project state from file. * - * @param project maven project + * @param project maven project + * @param stateFileName state file name * @return state if state file exists, or {@code null} * @throws IOException if an IO error occurs * @throws XMLException if a parsing error occurs */ - static ProjectState load(MavenProject project) throws IOException, XMLException { + static ProjectState load(MavenProject project, String stateFileName) throws IOException, XMLException { return load(project.getModel().getProjectDirectory().toPath() .resolve(project.getModel().getBuild().getDirectory()) - .resolve(STATE_FILE_NAME)); + .resolve(stateFileName)); } /** @@ -206,16 +207,17 @@ static ProjectState load(Path stateFile) throws IOException, XMLException { /** * Save the project state. * - * @param project Maven project + * @param project Maven project + * @param stateFileName state file name * @throws IOException if an IO error occurs */ - void save(MavenProject project) throws IOException { + void save(MavenProject project, String stateFileName) throws IOException { Model model = project.getModel(); Path buildDir = model.getProjectDirectory().toPath().resolve(model.getBuild().getDirectory()); if (!Files.exists(buildDir)) { Files.createDirectories(buildDir); } - save(buildDir.resolve(STATE_FILE_NAME)); + save(buildDir.resolve(stateFileName)); } /** @@ -377,6 +379,41 @@ ExecutionEntry findMatchingExecution(ExecutionEntry execution) { }).orElse(null); } + /** + * Merge two project states. + * + * @param state1 state1, must be non {@code null} + * @param state2 state1, must be non {@code null} + * @return ProjectState + */ + static ProjectState merge(ProjectState state1, ProjectState state2) { + Properties properties = new Properties(); + properties.putAll(state1.properties); + properties.putAll(state2.properties); + ProjectFiles projectFiles1 = state1.projectFiles; + ProjectFiles projectFiles2 = state2.projectFiles; + return new ProjectState( + properties, + Optional.ofNullable(state1.artifact).orElse(state2.artifact), + Stream.of(state1.attachedArtifacts.stream(), state2.attachedArtifacts.stream()) + .flatMap(Function.identity()) + .distinct() + .collect(Collectors.toList()), + Stream.of(state1.compileSourceRoots.stream(), state2.compileSourceRoots.stream()) + .flatMap(Function.identity()) + .distinct() + .collect(Collectors.toList()), + Stream.of(state1.testCompileSourceRoots.stream(), state2.testCompileSourceRoots.stream()) + .flatMap(Function.identity()) + .distinct() + .collect(Collectors.toList()), + projectFiles1.lastModified() > projectFiles2.lastModified() ? projectFiles1 : projectFiles2, + Stream.of(state1.executions.stream(), state2.executions.stream()) + .flatMap(Function.identity()) + .distinct() + .collect(Collectors.toList())); + } + /** * Create a state for the given project and merge it with the existing state for this project. * diff --git a/maven-plugins/build-cache-maven-extension/src/main/java/io/helidon/build/maven/cache/ProjectStateManager.java b/maven-plugins/build-cache-maven-extension/src/main/java/io/helidon/build/maven/cache/ProjectStateManager.java index e941e2f56..d22fd76fa 100644 --- a/maven-plugins/build-cache-maven-extension/src/main/java/io/helidon/build/maven/cache/ProjectStateManager.java +++ b/maven-plugins/build-cache-maven-extension/src/main/java/io/helidon/build/maven/cache/ProjectStateManager.java @@ -18,6 +18,7 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.util.ArrayDeque; +import java.util.ArrayList; import java.util.Deque; import java.util.HashMap; import java.util.List; @@ -27,6 +28,7 @@ import javax.inject.Named; import io.helidon.build.common.LazyValue; +import io.helidon.build.common.Lists; import io.helidon.build.common.xml.XMLException; import org.apache.maven.SessionScoped; @@ -114,7 +116,8 @@ private Map initStates() { * @param project Maven project */ public void save(MavenProject project) { - if (configManager.cacheConfig().record()) { + CacheConfig cacheConfig = configManager.cacheConfig(); + if (cacheConfig.record()) { try { ProjectState projectState = null; ProjectFiles projectFiles = null; @@ -125,7 +128,9 @@ public void save(MavenProject project) { } List newExecutions = executionManager.recordedExecutions(project); ProjectState.merge(projectState, project, session, configManager, newExecutions, projectFiles) - .save(project); + .save(project, cacheConfig.recordSuffix() + .map(suffix -> "state-" + suffix + ".xml") + .orElse("state.xml")); } catch (IOException | UncheckedIOException ex) { logger.error("Error while saving project state", ex); } @@ -149,12 +154,31 @@ private ProjectStateStatus processState(MavenProject project) { project.getGroupId(), project.getArtifactId())); } - ProjectState state; + List suffixes = cacheConfig.loadSuffixes(); + List stateFileNames = new ArrayList<>(Lists.map(suffixes, suffix -> "state-" + suffix + ".xml")); + stateFileNames.add("state.xml"); + ProjectState state = null; try { - state = ProjectState.load(project); + for (String stateFileName : stateFileNames) { + ProjectState nextState = ProjectState.load(project, stateFileName); + if (nextState == null) { + if (logger.isDebugEnabled()) { + logger.debug(String.format("[%s:%s] - state file not found: %s", + project.getGroupId(), + project.getArtifactId(), + stateFileName)); + } + continue; + } + if (state == null) { + state = nextState; + } else { + state = ProjectState.merge(state, nextState); + } + } if (state == null) { if (logger.isDebugEnabled()) { - logger.debug(String.format("[%s:%s] - state file not found", + logger.debug(String.format("[%s:%s] - state file(s) not found", project.getGroupId(), project.getArtifactId())); } diff --git a/maven-plugins/build-cache-maven-extension/src/test/java/io/helidon/build/maven/cache/CacheConfigTest.java b/maven-plugins/build-cache-maven-extension/src/test/java/io/helidon/build/maven/cache/CacheConfigTest.java index 91e9264ac..ef588fdea 100644 --- a/maven-plugins/build-cache-maven-extension/src/test/java/io/helidon/build/maven/cache/CacheConfigTest.java +++ b/maven-plugins/build-cache-maven-extension/src/test/java/io/helidon/build/maven/cache/CacheConfigTest.java @@ -15,6 +15,7 @@ */ package io.helidon.build.maven.cache; +import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; @@ -44,12 +45,15 @@ void testConfig() throws Exception { CacheConfig config = new CacheConfig(elt, toProperties(Map.of()), toProperties(Map.of())); assertThat(config.enabled(), is(true)); + assertThat(config.record(), is(true)); + assertThat(config.loadSuffixes(), is(List.of("foo", "bar"))); + assertThat(config.recordSuffix().orElse(null), is("foo")); assertThat(config.enableChecksums(), is(true)); assertThat(config.includeAllChecksums(), is(true)); - assertThat(config.lifecyleConfig().size(), is(2)); + assertThat(config.lifecycleConfig().size(), is(2)); - LifecycleConfig lifecycleConfig1 = config.lifecyleConfig().get(0); + LifecycleConfig lifecycleConfig1 = config.lifecycleConfig().get(0); assertThat(lifecycleConfig1.path(), is("a-path")); assertThat(lifecycleConfig1.glob(), is("a-glob")); assertThat(lifecycleConfig1.regex(), is("a-regex")); @@ -58,7 +62,7 @@ void testConfig() throws Exception { assertThat(lifecycleConfig1.executionsIncludes(), is(List.of("exec-include"))); assertThat(lifecycleConfig1.projectFilesExcludes(), is(List.of("project-exclude"))); - LifecycleConfig lifecycleConfig2 = config.lifecyleConfig().get(1); + LifecycleConfig lifecycleConfig2 = config.lifecycleConfig().get(1); assertThat(lifecycleConfig2.glob(), is("foo/**")); assertThat(lifecycleConfig2.enabled(), is(false)); @@ -74,4 +78,21 @@ void testConfig() throws Exception { assertThat(moduleSet.includes(), is(List.of("module-include"))); assertThat(moduleSet.excludes(), is(List.of("module-exclude"))); } + + @Test + void testConfigOverride() throws IOException { + Path configFile = TestFiles.testResourcePath(CacheConfigTest.class, "cache-config.xml"); + XMLElement elt = XMLElement.parse(Files.newInputStream(configFile)); + CacheConfig config = new CacheConfig(elt, toProperties(Map.of( + "cache.enabled", "false", + "cache.record", "false", + "cache.loadSuffixes", "one,two", + "cache.recordSuffix", "three" + )), toProperties(Map.of())); + + assertThat(config.enabled(), is(false)); + assertThat(config.record(), is(false)); + assertThat(config.loadSuffixes(), is(List.of("one", "two"))); + assertThat(config.recordSuffix().orElse(null), is("three")); + } } diff --git a/maven-plugins/build-cache-maven-extension/src/test/resources/cache-config.xml b/maven-plugins/build-cache-maven-extension/src/test/resources/cache-config.xml index 9b21e4194..29a26cf0b 100644 --- a/maven-plugins/build-cache-maven-extension/src/test/resources/cache-config.xml +++ b/maven-plugins/build-cache-maven-extension/src/test/resources/cache-config.xml @@ -18,6 +18,12 @@ --> true + true + + foo + bar + + foo true true