From d9bb0245155c8a2fc3d4ddf50357886fbf231438 Mon Sep 17 00:00:00 2001 From: Glenn Renfro Date: Tue, 13 Aug 2024 09:51:18 -0400 Subject: [PATCH] Update SCDF so that multipleComposedTaskWithArguments succeeds Step 1. Make sure to remove the version from the docker compose. It is no longer needed and causes older versions of docker to fail Step 2. Update compose files to use the latest version of SCDF 3.x instead of 2.11.x Step 3. Update build image script so that uses java 17 when creating containers Update the DataFlowIT and the Abstract classes it is built on so that multipleComposedTaskWithArguments test passes. Notice that JobParameterJacksonDeserializer and JobParametersJacksonMixIn have been updated. These changes mirror those in https://github.com/spring-cloud/spring-cloud-dataflow/pull/5850. These were required for the test to pass. At the time this PR is merged we can merge accepting those from #5850. Provide docs on how SCDF images are created and pushed Also update the DEFAULT_JDK to Java 17 Update PR based on code review comments * Added log message in case a JobParameter Type is invalid * cleaned up workflow.adoc --- .github/workflows/build-image.sh | 13 ++-- .github/workflows/build-images.yml | 2 +- .github/workflows/workflow.adoc | 71 +++++++++++++++++++ .../JobParameterJacksonDeserializer.java | 1 + .../dataflow/integration/test/DataFlowIT.java | 2 - .../test/db/AbstractDataflowTests.java | 6 +- .../integration/test/tags/TagNames.java | 6 ++ .../docker-compose-maven-it-task-import.yml | 2 - .../docker-compose-prometheus.yml | 2 - src/docker-compose/docker-compose.yml | 5 +- 10 files changed, 90 insertions(+), 20 deletions(-) create mode 100644 .github/workflows/workflow.adoc diff --git a/.github/workflows/build-image.sh b/.github/workflows/build-image.sh index 72670f255c..c6ee19bdb6 100755 --- a/.github/workflows/build-image.sh +++ b/.github/workflows/build-image.sh @@ -13,8 +13,8 @@ if [ "$TAG" == "" ]; then exit 1 fi if [ "$DEFAULT_JDK" = "" ]; then - echo "DEFAULT_JDK not found using 11" - DEFAULT_JDK=11 + echo "DEFAULT_JDK not found using 17" + DEFAULT_JDK=17 else echo "DEFAULT_JDK=$DEFAULT_JDK" fi @@ -55,7 +55,8 @@ if [ ! -f "$JAR" ]; then exit $RC fi fi -for v in 8 11 17; do +# TODO add Java 21 when packeto supports it +for v in 17; do echo "Creating: $REPO:$TAG-jdk$v" pack build --builder gcr.io/paketo-buildpacks/builder:base \ --path "$JAR" \ @@ -93,9 +94,5 @@ for v in 8 11 17; do fi fi done -#if [ "$PUSH" == "true" ]; then -# echo "Pruning Docker" -# docker system prune -f -# docker system prune --volumes -f -#fi + diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index 23cf8926a4..b1f7a59832 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -81,7 +81,7 @@ jobs: shell: bash env: TAG: ${{ needs.prepare.outputs.version }} - DEFAULT_JDK: '11' + DEFAULT_JDK: '17' ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }} ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }} DELETE_TAGS: ${{ inputs.delete-tags }} diff --git a/.github/workflows/workflow.adoc b/.github/workflows/workflow.adoc new file mode 100644 index 0000000000..bbd98775dc --- /dev/null +++ b/.github/workflows/workflow.adoc @@ -0,0 +1,71 @@ += Workflow Reference + +This README serves as a guide to the GitHub Action workflows included in this repository. +It outlines the purpose and functionality of each workflow, detailing their role in the CI and release processes. +Additionally, this document provides an overview of the scripts and actions utilized in these workflows, offering insights into how they work together in SCDF's CI/CD pipeline. + +This document is a work in progress, and as various workflows are updated, the documentation will be revised to reflect both existing and new behaviors. + + +== Building Docker Images and pushing the containers to DockerHub +.This diagram shows the flow of execution of how workflows create Docker imges. +``` +┌─────────────────────────┐ +│ │ +│ │ +│build-snapshot-worker.yml┼────┐ +│ │ │ +│ │ │ +└─────────────────────────┘ │ +┌─────────────────────────┐ │ +│ │ │ +│ │ │ +│ ci-images.yml ┼────┤ ┌─────────────────────────┐ ┌─────────────────────────┐ +│ │ │ │ │ │ │ +│ │ │ │ │ │ │ +└─────────────────────────┘ ├────►│ build-images.yml ┼────────►│ build-image.sh │ +┌─────────────────────────┐ │ │ │ │ │ +│ │ │ │ │ │ │ +│ │ │ └───────────┬─────────────┘ └─────────────────────────┘ +│ ci.yml ┼────┤ │ +│ │ │ │ +│ │ │ ┌───────────┴─────────────┐ +└─────────────────────────┘ │ │ │ +┌─────────────────────────┐ │ │ │ +│ │ │ │ images.json │ +│ │ │ │ │ +│ release-worker.yml ┼────┘ │ │ +│ │ └─────────────────────────┘ +│ │ +└─────────────────────────┘ +``` + +Part of the release and CI process involves creating and pushing images to a registry (such as DockerHub) for the Dataflow server, Skipper server, CTR app, and other components. +This process is managed by the `build-images` (build-images.yml) workflow. While the `build-images` workflow is typically not run independently, it is invoked by other workflows that handle CI builds and releases. +The `build-images` workflow determines which images to create based on the `images.json` file. +This file contains metadata on the primary SCDF components that need to have an associated image. +Each entry specifies the location (directory) where the jar can be found, jar name, and image name for each artifact that will be used to construct the image. +For each entry in the `images.json` file, the workflow calls the `build-image.sh` script, which retrieves the jar, builds the image, and then pushes it to the registry. + +SCDF also provides images for external applications that support some of the optional features that are offered by dataflow. +These include Grafana and Prometheus local. +These images are created and pushed using the docker/build-push-action@v2 action. + +=== Scripts used to build images +As mentioned above, the `build-image.sh` script is responsible for building the specified image based on the parameters provided and then pushing the image to Dockerhub. +This script uses Paketo to build an image for each of the supported Java versions using the corresponding jar file. +The resulting image name will look something like `spring-cloud-dataflow-server:3.0.0-SNAPSHOT-jdk17`. +Additionally, the script creates a default image using the default Java version as specified by the `DEFAULT_JDK` environment variable. + +The format for running the `build-image.sh` is as follows: +[source, bash] +``` +bash +./build-image.sh +``` + +There is an optional `DEFAULT_JDK` environment variable that allows you to set the JDK version for the default image created. +If not the script will set it to its current setting (which as of the writing of this document is `17`). + +NOTE: When new releases of Java are available and are compliant with the SCDF release, they need to be added to the `build-image.sh` script. + diff --git a/spring-cloud-dataflow-rest-resource/src/main/java/org/springframework/cloud/dataflow/rest/support/jackson/JobParameterJacksonDeserializer.java b/spring-cloud-dataflow-rest-resource/src/main/java/org/springframework/cloud/dataflow/rest/support/jackson/JobParameterJacksonDeserializer.java index 64441e7100..02c584aee3 100644 --- a/spring-cloud-dataflow-rest-resource/src/main/java/org/springframework/cloud/dataflow/rest/support/jackson/JobParameterJacksonDeserializer.java +++ b/spring-cloud-dataflow-rest-resource/src/main/java/org/springframework/cloud/dataflow/rest/support/jackson/JobParameterJacksonDeserializer.java @@ -54,6 +54,7 @@ public JobParameter deserialize(JsonParser jsonParser, DeserializationContext de try { jobParameter = new JobParameter(value, Class.forName(type), identifying); } catch (ClassNotFoundException e) { + logger.warn("JobParameter type %s is not supported by DataFlow. Verify type is valid or in classpath.".formatted(type) ); throw new IllegalArgumentException("JobParameter type %s is not supported by DataFlow".formatted(type), e); } } diff --git a/spring-cloud-dataflow-server/src/test/java/org/springframework/cloud/dataflow/integration/test/DataFlowIT.java b/spring-cloud-dataflow-server/src/test/java/org/springframework/cloud/dataflow/integration/test/DataFlowIT.java index 23ea329fde..19e8b263c4 100644 --- a/spring-cloud-dataflow-server/src/test/java/org/springframework/cloud/dataflow/integration/test/DataFlowIT.java +++ b/spring-cloud-dataflow-server/src/test/java/org/springframework/cloud/dataflow/integration/test/DataFlowIT.java @@ -1314,8 +1314,6 @@ public void composedTask() { assertThat(taskBuilder.allTasks().size()).isEqualTo(0); } - //TODO: Boot3x followup - @Disabled("TODO: Boot3x followup Wait for composed Task runner to be ported to 3.x") @Test public void multipleComposedTaskWithArguments() { logger.info("task-multiple-composed-task-with-arguments-test"); diff --git a/spring-cloud-dataflow-server/src/test/java/org/springframework/cloud/dataflow/integration/test/db/AbstractDataflowTests.java b/spring-cloud-dataflow-server/src/test/java/org/springframework/cloud/dataflow/integration/test/db/AbstractDataflowTests.java index c7f8b48816..d2527afb3b 100644 --- a/spring-cloud-dataflow-server/src/test/java/org/springframework/cloud/dataflow/integration/test/db/AbstractDataflowTests.java +++ b/spring-cloud-dataflow-server/src/test/java/org/springframework/cloud/dataflow/integration/test/db/AbstractDataflowTests.java @@ -64,7 +64,8 @@ protected static class EmptyConfig { ClusterContainer.from(TagNames.DATAFLOW_2_8, DATAFLOW_IMAGE_PREFIX + "2.8.4"), ClusterContainer.from(TagNames.DATAFLOW_2_9, DATAFLOW_IMAGE_PREFIX + "2.9.6"), ClusterContainer.from(TagNames.DATAFLOW_2_10, DATAFLOW_IMAGE_PREFIX + "2.10.3"), - ClusterContainer.from(TagNames.DATAFLOW_2_11, DATAFLOW_IMAGE_PREFIX + "2.11.3") + ClusterContainer.from(TagNames.DATAFLOW_2_11, DATAFLOW_IMAGE_PREFIX + "2.11.4"), + ClusterContainer.from(TagNames.DATAFLOW_3_0, DATAFLOW_IMAGE_PREFIX + "3.0.0") ); public final static List SKIPPER_CONTAINERS = Arrays.asList( @@ -72,7 +73,8 @@ protected static class EmptyConfig { ClusterContainer.from(TagNames.SKIPPER_2_7, SKIPPER_IMAGE_PREFIX + "2.7.4"), ClusterContainer.from(TagNames.SKIPPER_2_8, SKIPPER_IMAGE_PREFIX + "2.8.6"), ClusterContainer.from(TagNames.SKIPPER_2_9, SKIPPER_IMAGE_PREFIX + "2.9.3"), - ClusterContainer.from(TagNames.SKIPPER_2_11, SKIPPER_IMAGE_PREFIX + "2.11.3") + ClusterContainer.from(TagNames.SKIPPER_2_11, SKIPPER_IMAGE_PREFIX + "2.11.4"), + ClusterContainer.from(TagNames.SKIPPER_3_0, SKIPPER_IMAGE_PREFIX + "3.0.0") ); public final static List DATABASE_CONTAINERS = Arrays.asList( diff --git a/spring-cloud-dataflow-server/src/test/java/org/springframework/cloud/dataflow/integration/test/tags/TagNames.java b/spring-cloud-dataflow-server/src/test/java/org/springframework/cloud/dataflow/integration/test/tags/TagNames.java index 074b4e5a8a..da55beb2c3 100644 --- a/spring-cloud-dataflow-server/src/test/java/org/springframework/cloud/dataflow/integration/test/tags/TagNames.java +++ b/spring-cloud-dataflow-server/src/test/java/org/springframework/cloud/dataflow/integration/test/tags/TagNames.java @@ -73,8 +73,11 @@ public abstract class TagNames { public static final String SKIPPER_2_8 = "skipper_2_8"; public static final String SKIPPER_2_9 = "skipper_2_9"; + public static final String SKIPPER_2_11 = "skipper_2_11"; + public static final String SKIPPER_3_0 = "skipper_3_0"; + public static final String SKIPPER_main = "skipper_main"; public static final String DATAFLOW = "dataflow"; @@ -86,7 +89,10 @@ public abstract class TagNames { public static final String DATAFLOW_2_9 = "dataflow_2_9"; public static final String DATAFLOW_2_10 = "dataflow_2_10"; + public static final String DATAFLOW_2_11 = "dataflow_2_11"; + public static final String DATAFLOW_3_0 = "dataflow_3_0"; + public static final String DATAFLOW_main = "dataflow_main"; } diff --git a/spring-cloud-dataflow-server/src/test/resources/docker-compose-maven-it-task-import.yml b/spring-cloud-dataflow-server/src/test/resources/docker-compose-maven-it-task-import.yml index 4280774905..116a1dce3e 100644 --- a/spring-cloud-dataflow-server/src/test/resources/docker-compose-maven-it-task-import.yml +++ b/spring-cloud-dataflow-server/src/test/resources/docker-compose-maven-it-task-import.yml @@ -1,5 +1,3 @@ -version: '3' - # CI specific test Task (scenario) registered from maven resource services: diff --git a/src/docker-compose/docker-compose-prometheus.yml b/src/docker-compose/docker-compose-prometheus.yml index d06f5b0a23..a4a2ba178f 100644 --- a/src/docker-compose/docker-compose-prometheus.yml +++ b/src/docker-compose/docker-compose-prometheus.yml @@ -1,5 +1,3 @@ -version: '3' - # Extends the default docker-compose.yml with Prometheus/Grafana monitoring configuration # Usage: docker-compose -f ./docker-compose.yml -f ./docker-compose-prometheus.yml up services: diff --git a/src/docker-compose/docker-compose.yml b/src/docker-compose/docker-compose.yml index 86b6e63ba1..cf7d830325 100644 --- a/src/docker-compose/docker-compose.yml +++ b/src/docker-compose/docker-compose.yml @@ -1,4 +1,3 @@ -version: '3' # Configuration environment variables: # - DATAFLOW_VERSION and SKIPPER_VERSION specify what DataFlow and Skipper image versions to use. # - STREAM_APPS_URI and TASK_APPS_URI are used to specify what Stream and Task applications to pre-register. @@ -21,7 +20,7 @@ version: '3' services: dataflow-server: user: root - image: springcloud/spring-cloud-dataflow-server:${DATAFLOW_VERSION:-2.11.2-SNAPSHOT}${BP_JVM_VERSION:-} + image: springcloud/spring-cloud-dataflow-server:${DATAFLOW_VERSION:-3.0.0-SNAPSHOT}${BP_JVM_VERSION:-} container_name: dataflow-server ports: - "9393:9393" @@ -64,7 +63,7 @@ services: skipper-server: user: root - image: springcloud/spring-cloud-skipper-server:${SKIPPER_VERSION:-2.11.2-SNAPSHOT}${BP_JVM_VERSION:-} + image: springcloud/spring-cloud-skipper-server:${SKIPPER_VERSION:-3.0.0-SNAPSHOT}${BP_JVM_VERSION:-} container_name: skipper-server ports: - "7577:7577"