#597: Move workflows into ci-build.yml

project-keeper/error_code_config.yml

@@ -2,4 +2,4 @@ error-tags:
   PK-CORE:
     packages:
       - com.exasol.projectkeeper
-    highest-index: 207
+    highest-index: 208

project-keeper/src/main/java/com/exasol/projectkeeper/config/ProjectKeeperConfigReader.java

@@ -170,6 +170,11 @@ private List<StepCustomization> convertSteps(final List<RawStepCustomization> st
     }
 
     private StepCustomization convertStep(final RawStepCustomization step) {
+        if (step.job == null || step.job.isBlank()) {
+            throw new IllegalArgumentException(ExaError.messageBuilder("E-PK-CORE-208")
+                    .message("Missing job in step customization of file {{config file}}.", CONFIG_FILE_NAME)
+                    .mitigation("Add job to the step customization.").toString());
+        }
         if (step.action == null) {
             throw new IllegalArgumentException(ExaError.messageBuilder("E-PK-CORE-200")
                     .message("Missing action in step customization of file {{config file}}.", CONFIG_FILE_NAME)
@@ -188,6 +193,7 @@ private StepCustomization convertStep(final RawStepCustomization step) {
                     .mitigation("Add content to the step customization.").toString());
         }
         return StepCustomization.builder() //
+                .jobId(step.job) //
                 .type(step.action) //
                 .stepId(step.stepId) //
                 .step(WorkflowStep.createStep(step.content)) //

project-keeper/src/main/java/com/exasol/projectkeeper/config/ProjectKeeperRawConfig.java

@@ -460,6 +460,8 @@ public RawStepCustomization() {
             // Required for specifying Javadoc
         }
 
+        /** ID of the job to customize. */
+        public String job;
         /** Customization type (insert/replace). */
         public StepCustomization.Type action;
         /** ID of the step to replace or after which to insert. */

project-keeper/src/main/java/com/exasol/projectkeeper/validators/DeletedFilesValidator.java

@@ -25,6 +25,7 @@ public class DeletedFilesValidator implements Validator {
             Path.of(GITHUB_WORKFLOWS, "release_droid_release_on_maven_central.yml"), RELEASE_DROID_WORKFLOW_WARNING,
             Path.of(GITHUB_WORKFLOWS, "release_droid_upload_github_release_assets.yml"), RELEASE_DROID_WORKFLOW_WARNING,
             Path.of(GITHUB_WORKFLOWS, "release_droid_prepare_original_checksum.yml"), RELEASE_DROID_WORKFLOW_WARNING,
+            Path.of(GITHUB_WORKFLOWS, "ci-build-next-java.yml"), "Next Java build is now included in ci-build.yml",
             Path.of("release_config.yml"), "Release-droid configuration is replaced by release.yml");
     private final Path projectDirectory;
 

project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/CiBuildWorkflowGenerator.java

@@ -18,10 +18,13 @@ class CiBuildWorkflowGenerator {
     private static final String CI_BUILD_PATH_IN_PROJECT = WORKFLOW_PATH + CI_BUILD_WORKFLOW_NAME;
     private final BuildOptions buildOptions;
     private final List<String> javaVersions;
+    private final String nextJavaVersion;
 
-    CiBuildWorkflowGenerator(final BuildOptions buildOptions, final List<String> javaVersions) {
+    CiBuildWorkflowGenerator(final BuildOptions buildOptions, final List<String> javaVersions,
+            final String nextJavaVersion) {
         this.buildOptions = Objects.requireNonNull(buildOptions, "buildOptions");
         this.javaVersions = Objects.requireNonNull(javaVersions, "javaVersions");
+        this.nextJavaVersion = Objects.requireNonNull(nextJavaVersion, "nextJavaVersion");
     }
 
     FileTemplate createCiBuildWorkflow() {
@@ -31,6 +34,7 @@ FileTemplate createCiBuildWorkflow() {
                 CI_BUILD_PATH_IN_PROJECT, REQUIRE_EXACT);
         template.replacing("ciBuildRunnerOS", buildOptions.getRunnerOs());
         template.replacing("freeDiskSpace", String.valueOf(buildOptions.shouldFreeDiskSpace()));
+        template.replacing("nextJavaVersion", nextJavaVersion);
 
         if (buildType == CiTemplateType.EXASOL_VERSION_MATRIX) {
             template.replacing("matrixExasolDbVersions",
@@ -50,13 +54,13 @@ private FileTemplate createTemplate(final FileTemplateFromResource template, fin
         return new ContentCustomizingTemplate(template, new GitHubWorkflowCustomizer( //
                 javaVersionCustomizer(),
                 // [impl->dsn~customize-build-process.ci-build~0]
-                new GitHubWorkflowStepCustomizer(customizations, buildJobId),
+                new GitHubWorkflowStepCustomizer(customizations),
                 // [impl->dsn~customize-build-process.ci-build.environment~1]
                 new GitHubWorkflowEnvironmentCustomizer(buildJobId, environmentName)));
     }
 
     private GitHubWorkflowJavaVersionCustomizer javaVersionCustomizer() {
-        return new GitHubWorkflowJavaVersionCustomizer(javaVersions);
+        return new GitHubWorkflowJavaVersionCustomizer(javaVersions, nextJavaVersion);
     }
 
     private List<StepCustomization> findCustomizations(final String workflowName) {
@@ -81,7 +85,7 @@ private CiTemplateType getCiBuildType() {
     FileTemplate createReleaseWorkflow(final List<AnalyzedSource> sources) {
         final Consumer<FileTemplateFromResource> templateCustomizer = template -> template
                 .replacing("mavenCentralDeployment", mavenCentralDeploymentRequired(sources) ? "true" : "false");
-        return createCustomizedWorkflow("release.yml", "release", templateCustomizer);
+        return createCustomizedWorkflow("release.yml", templateCustomizer);
     }
 
     private boolean mavenCentralDeploymentRequired(final List<AnalyzedSource> sources) {
@@ -94,32 +98,32 @@ private boolean mavenCentralDeploymentRequired(final List<AnalyzedSource> source
 
     // [impl->dsn~customize-build-process.dependency-check~0]
     FileTemplate createDependenciesCheckWorkflow() {
-        return createCustomizedWorkflow("dependencies_check.yml", "report_security_issues");
+        return createCustomizedWorkflow("dependencies_check.yml");
     }
 
     // [impl->dsn~dependency-updater.workflow.generate~1]
     // [impl->dsn~customize-build-process.dependency-update~0]
     FileTemplate createDependenciesUpdateWorkflow() {
-        return createCustomizedWorkflow("dependencies_update.yml", "update_dependencies");
+        return createCustomizedWorkflow("dependencies_update.yml");
     }
 
-    private FileTemplate createCustomizedWorkflow(final String workflowName, final String jobName) {
-        return createCustomizedWorkflow(workflowName, jobName, template -> {
+    private FileTemplate createCustomizedWorkflow(final String workflowName) {
+        return createCustomizedWorkflow(workflowName, template -> {
         });
     }
 
-    private FileTemplate createCustomizedWorkflow(final String workflowName, final String jobName,
+    private FileTemplate createCustomizedWorkflow(final String workflowName,
             final Consumer<FileTemplateFromResource> templateCustomizer) {
         final FileTemplateFromResource template = new FileTemplateFromResource(WORKFLOW_PATH + workflowName,
                 REQUIRE_EXACT);
         templateCustomizer.accept(template);
         final List<StepCustomization> customizations = findCustomizations(workflowName);
         return new ContentCustomizingTemplate(template, new GitHubWorkflowCustomizer(javaVersionCustomizer(),
-                new GitHubWorkflowStepCustomizer(customizations, jobName)));
+                new GitHubWorkflowStepCustomizer(customizations)));
     }
 
     enum CiTemplateType {
-        DEFAULT(CI_BUILD_WORKFLOW_NAME, "build"),
+        DEFAULT(CI_BUILD_WORKFLOW_NAME, "build-and-test"),
         EXASOL_VERSION_MATRIX("ci-build-db-version-matrix.yml", "matrix-build");
 
         private final String templateName;

project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/FileTemplatesFactory.java

@@ -24,26 +24,29 @@ class FileTemplatesFactory {
     private final boolean hasNpmModule;
     private final BuildOptions buildOptions;
     private final CiBuildWorkflowGenerator workflowGenerator;
+    private final List<AnalyzedSource> sources;
 
     public FileTemplatesFactory(final Logger logger, final String ownVersion, final boolean hasNpmModule,
             final BuildOptions buildOptions, final List<AnalyzedSource> sources) {
         this.logger = Objects.requireNonNull(logger, "logger");
         this.ownVersion = Objects.requireNonNull(ownVersion, "ownVersion");
         this.hasNpmModule = hasNpmModule;
         this.buildOptions = Objects.requireNonNull(buildOptions, "buildOptions");
-        this.workflowGenerator = new CiBuildWorkflowGenerator(this.buildOptions,
-                new JavaVersionExtractor(sources).extractVersions());
+        this.sources = sources;
+        final JavaVersionExtractor javaVersionExtractor = new JavaVersionExtractor(sources);
+        this.workflowGenerator = new CiBuildWorkflowGenerator(this.buildOptions, javaVersionExtractor.extractVersions(),
+                javaVersionExtractor.getNextVersion());
     }
 
-    List<FileTemplate> getGlobalTemplates(final List<AnalyzedSource> sources) {
+    List<FileTemplate> getGlobalTemplates() {
         final List<FileTemplate> templates = new ArrayList<>();
         templates.add(new FileTemplateFromResource(".github/workflows/broken_links_checker.yml", REQUIRE_EXACT));
         templates.add(new FileTemplateFromResource(".vscode/settings.json", REQUIRE_EXIST));
         final Optional<AnalyzedSource> mvnRoot = sources.stream().filter(this::isMvnRootProject).findFirst();
         templates.add(new FileTemplateFromResource("templates/gitattributes", ".gitattributes", REQUIRE_EXIST)
                 .replacing("pomFiles", mvnRoot.isPresent() ? POM_FILES_GENERATED : ""));
         if (mvnRoot.isPresent()) {
-            templates.addAll(getGenericMavenTemplates(sources, mvnRoot.get().getModules()));
+            templates.addAll(getGenericMavenTemplates(mvnRoot.get().getModules()));
         } else {
             templates.addAll(getProjectKeeperVerifyWorkflowTemplates());
             this.logger.warn(ExaError.messageBuilder("W-PK-CORE-91")
@@ -53,27 +56,15 @@ List<FileTemplate> getGlobalTemplates(final List<AnalyzedSource> sources) {
         return templates;
     }
 
-    private List<FileTemplate> getGenericMavenTemplates(final List<AnalyzedSource> sources,
-            final Set<ProjectKeeperModule> modules) {
+    private List<FileTemplate> getGenericMavenTemplates(final Set<ProjectKeeperModule> modules) {
         final List<FileTemplate> templates = new ArrayList<>();
         templates.add(getCiBuildTemplate(modules));
-        templates.add(getNextJavaWorkflow(modules, new JavaVersionExtractor(sources).getNextVersion()));
         templates.add(this.workflowGenerator.createDependenciesCheckWorkflow());
         templates.add(this.workflowGenerator.createDependenciesUpdateWorkflow());
         templates.add(this.workflowGenerator.createReleaseWorkflow(sources));
         return templates;
     }
 
-    private FileTemplate getNextJavaWorkflow(final Set<ProjectKeeperModule> modules, final String nextJavaVersion) {
-        final FileTemplateFromResource template = new FileTemplateFromResource(
-                ".github/workflows/ci-build-next-java.yml", REQUIRE_EXACT) //
-                .replacing("skipNativeImage",
-                        modules.contains(ProjectKeeperModule.NATIVE_IMAGE) ? "-P skipNativeImage" : "")
-                .replacing("nextJavaVersion", nextJavaVersion);
-        return new ContentCustomizingTemplate(template,
-                new GitHubWorkflowCustomizer(new GitHubWorkflowJavaVersionCustomizer(List.of(nextJavaVersion))));
-    }
-
     private FileTemplate getCiBuildTemplate(final Set<ProjectKeeperModule> modules) {
         if (modules.contains(NATIVE_IMAGE)) {
             return new FileTemplateFromResource("templates/.github/workflows/ci-build-native-build.yml",

project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/GitHubWorkflow.java

@@ -35,15 +35,15 @@ Job getJob(final String jobId) {
         if (rawJob == null) {
             return null;
         }
-        return new Job(asMap(rawJob));
+        return new Job(jobId, asMap(rawJob));
     }
 
     private Map<String, Object> getJobMap() {
         return asMap(rawWorkflow.get("jobs"));
     }
 
     List<Job> getJobs() {
-        return getJobMap().values().stream().map(GitHubWorkflow::asMap).map(Job::new).toList();
+        return getJobMap().entrySet().stream().map(entry -> new Job(entry.getKey(), asMap(entry.getValue()))).toList();
     }
 
     Map<String, Object> getOnTrigger() {
@@ -59,12 +59,18 @@ Object getRawObject() {
      * well as insert and replace steps.
      */
     static class Job {
+        private final String id;
         private final Map<String, Object> rawJob;
 
-        private Job(final Map<String, Object> rawJob) {
+        private Job(final String id, final Map<String, Object> rawJob) {
+            this.id = id;
             this.rawJob = rawJob;
         }
 
+        String getId() {
+            return id;
+        }
+
         Step getStep(final String id) {
             return getSteps().stream() //
                     .filter(hasId(id)) //
@@ -137,7 +143,9 @@ private int findStepIndex(final String stepId) {
                     .filter(index -> steps.get(index).getId().equals(stepId)) //
                     .findFirst() //
                     .orElseThrow(() -> new IllegalStateException(ExaError.messageBuilder("E-PK-CORE-205")
-                            .message("No step found for id {{step id}} in {{raw job}}", stepId, rawJob).toString()));
+                            .message("No step found for id {{step id}} in {{available step ids}}", stepId,
+                                    steps.stream().map(Step::getId).toList())
+                            .toString()));
         }
     }
 

project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowJavaVersionCustomizer.java

@@ -10,22 +10,30 @@
 class GitHubWorkflowJavaVersionCustomizer implements GitHubWorkflowCustomizer.WorkflowCustomizer {
 
     private final List<String> javaVersions;
+    private final String nextJavaVersion;
 
-    GitHubWorkflowJavaVersionCustomizer(final List<String> javaVersions) {
+    GitHubWorkflowJavaVersionCustomizer(final List<String> javaVersions, final String nextJavaVersion) {
         this.javaVersions = javaVersions;
+        this.nextJavaVersion = nextJavaVersion;
     }
 
     @Override
     public void applyCustomization(final GitHubWorkflow workflow) {
-        workflow.getJobs().forEach(this::updateJavaVersion);
+        for (final Job job : workflow.getJobs()) {
+            if ("next-java-compatibility".equals(job.getId())) {
+                updateJavaVersion(job, nextJavaVersion);
+            } else {
+                updateJavaVersion(job, formatJavaVersions());
+            }
+        }
     }
 
-    private void updateJavaVersion(final Job job) {
-        job.getSteps().stream().filter(this::isSetupJavaStep).forEach(this::updateJavaVersion);
+    private void updateJavaVersion(final Job job, final String version) {
+        job.getSteps().stream().filter(this::isSetupJavaStep).forEach(step -> updateJavaVersion(step, version));
     }
 
-    private void updateJavaVersion(final Step step) {
-        step.getWith().put("java-version", formatJavaVersions());
+    private void updateJavaVersion(final Step step, final String version) {
+        step.getWith().put("java-version", version);
     }
 
     private String formatJavaVersions() {

project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/GitHubWorkflowStepCustomizer.java

@@ -7,17 +7,15 @@
 
 class GitHubWorkflowStepCustomizer implements GitHubWorkflowCustomizer.WorkflowCustomizer {
     private final List<StepCustomization> customizations;
-    private final String jobId;
 
-    GitHubWorkflowStepCustomizer(final List<StepCustomization> customizations, final String jobId) {
+    GitHubWorkflowStepCustomizer(final List<StepCustomization> customizations) {
         this.customizations = customizations;
-        this.jobId = jobId;
     }
 
     @Override
     public void applyCustomization(final GitHubWorkflow workflow) {
-        final Job job = workflow.getJob(jobId);
         for (final StepCustomization customization : customizations) {
+            final Job job = workflow.getJob(customization.getJobId());
             applyCustomization(job, customization);
         }
     }

project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/JavaVersionExtractor.java

@@ -39,6 +39,9 @@ private static double parseVersion(final String version) {
     }
 
     private Set<String> getSourceJavaVersions() {
+        if (sources.isEmpty()) {
+            throw new IllegalStateException("No sources, can't get java versions");
+        }
         return sources.stream().filter(AnalyzedMavenSource.class::isInstance) //
                 .map(AnalyzedMavenSource.class::cast) //
                 .map(AnalyzedMavenSource::getJavaVersion) //
@@ -63,7 +66,8 @@ private double getCurrentLatestVersion() {
         return getSourceJavaVersions().stream() //
                 .map(JavaVersionExtractor::parseVersion) //
                 .reduce(JavaVersionExtractor::takeLast) //
-                .orElseThrow();
+                .orElseThrow(() -> new IllegalStateException(
+                        "No latest java version found in source java versions " + getSourceJavaVersions()));
     }
 
     private static double takeLast(final double first, final double second) {

project-keeper/src/main/java/com/exasol/projectkeeper/validators/files/ProjectFilesValidator.java

@@ -57,7 +57,7 @@ private List<ValidationFinding> validateTemplatesRelativeToSource(final FileTemp
     }
 
     private List<ValidationFinding> validateTemplatesRelativeToRepo(final FileTemplatesFactory templatesFactory) {
-        final List<FileTemplate> templates = templatesFactory.getGlobalTemplates(this.sources);
+        final List<FileTemplate> templates = templatesFactory.getGlobalTemplates();
         return runValidation(templates, this.projectDirectory);
     }
 

project-keeper/src/main/resources/non_maven_templates/.github/workflows/project-keeper-verify.yml

@@ -16,6 +16,7 @@ jobs:
       cancel-in-progress: true
     steps:
       - name: Checkout the repository
+        id: checkout
         uses: actions/checkout@v4
         with:
           fetch-depth: 0

project-keeper/src/main/resources/templates/.github/workflows/broken_links_checker.yml

@@ -5,10 +5,6 @@ name: Broken Links Checker
 on:
   schedule:
     - cron: "0 5 * * 0"
-  push:
-    branches:
-      - main
-  pull_request:
 
 jobs:
   linkChecker: