diff --git a/.github/actions/build-images/build-images.sh b/.github/actions/build-images/build-images.sh index 0c64723eea..5a4bd64f7a 100755 --- a/.github/actions/build-images/build-images.sh +++ b/.github/actions/build-images/build-images.sh @@ -31,37 +31,11 @@ function pack_image { fi echo "Created: $REPO:$TAG-jdk$v" } - -TARGETS=("spring-cloud-dataflow-server/target/spring-cloud-dataflow-server" \ - "spring-cloud-skipper/spring-cloud-skipper-server/target/spring-cloud-skipper-server" \ - "spring-cloud-dataflow-composed-task-runner/target/spring-cloud-dataflow-composed-task-runner" \ - "spring-cloud-dataflow-single-step-batch-job/target/spring-cloud-dataflow-single-step-batch-job" \ - "spring-cloud-dataflow-tasklauncher/spring-cloud-dataflow-tasklauncher-sink-kafka/target/spring-cloud-dataflow-tasklauncher-sink-kafka" \ - "spring-cloud-dataflow-tasklauncher/spring-cloud-dataflow-tasklauncher-sink-rabbit/target/spring-cloud-dataflow-tasklauncher-sink-rabbit") - -IMAGES=("springcloud/spring-cloud-dataflow-server" \ - "springcloud/spring-cloud-skipper-server" \ - "springcloud/spring-cloud-dataflow-composed-task-runner" \ - "springcloud/spring-cloud-dataflow-single-step-batch-job" \ - "springcloud/spring-cloud-dataflow-tasklauncher-sink-kafka" \ - "springcloud/spring-cloud-dataflow-tasklauncher-sink-rabbit") - -MAVEN_ARTIFACT_IDS=("spring-cloud-dataflow-server" \ - "spring-cloud-dataflow-server" \ - "spring-cloud-dataflow-composed-task-runner" \ - "spring-cloud-dataflow-tasklauncher-sink-kafka" \ - "spring-cloud-dataflow-tasklauncher-sink-rabbit") -len=${#TARGETS[@]} -imageLen=${#IMAGES[@]} -if ((len != imageLen)); then - echo "Expected $len == $imageLen" - exit 1 -fi - -for ((i = 0; i < len; i++)); do - TARGET="${TARGETS[i]}" - IMAGE="${IMAGES[i]}" - ARTIFACT_ID="${MAVEN_ARTIFACT_IDS[i]}" +LEN=$(jq '.include | length' .github/workflows/images.json) +for ((i = 0; i < LEN; i++)); do + TARGET="$(jq -r --arg index $i '.include[$index | tonumber] | .path' .github/workflows/images.json)" + IMAGE="$(jq -r --arg index $i '.include[$index | tonumber ] | .image' .github/workflows/images.json)" + ARTIFACT_ID="$(jq -r --arg index $i '.include[$index | tonumber ] | .name' .github/workflows/images.json)" for v in 8 11 17 21; do pack_image $TARGET $IMAGE $v $ARTIFACT_ID RC=$? diff --git a/.github/workflows/images.json b/.github/workflows/images.json index 76b3b1d2e5..83652cd335 100644 --- a/.github/workflows/images.json +++ b/.github/workflows/images.json @@ -15,6 +15,11 @@ "path": "spring-cloud-dataflow-composed-task-runner/target", "image": "springcloud/spring-cloud-dataflow-composed-task-runner" }, + { + "name": "spring-cloud-dataflow-single-step-batch-job", + "path": "spring-cloud-dataflow-single-step-batch-job/target", + "image": "springcloud/spring-cloud-dataflow-single-step-batch-job" + }, { "name": "spring-cloud-dataflow-tasklauncher-sink-kafka", "path": "spring-cloud-dataflow-tasklauncher/spring-cloud-dataflow-tasklauncher-sink-kafka/target", diff --git a/.github/workflows/release-controller.yml b/.github/workflows/release-controller.yml index 678486e0e2..92df60d36d 100644 --- a/.github/workflows/release-controller.yml +++ b/.github/workflows/release-controller.yml @@ -14,101 +14,111 @@ jobs: runs-on: ubuntu-latest steps: - # initial is when user starts workflow from UI(context is empty) - # then train build goes through via repos using defined hander rules - - name: Handle Zoo Control - uses: jvalkeal/build-zoo-handler@v0.0.4 - with: - dispatch-handler-token: ${{ secrets.SCDF_ACCESS_TOKEN }} - dispatch-handler-max: 20 - dispatch-handler-config: > - [ - { - "if": "initial == true", - "action": "workflow_dispatch", - "workflow_dispatch": { - "owner": "spring-cloud", - "repo": "spring-cloud-deployer", - "ref": "main", - "workflow": "release-worker.yml" + # initial is when user starts workflow from UI(context is empty) + # then train build goes through via repos using defined hander rules + - name: Handle Zoo Control + uses: jvalkeal/build-zoo-handler@v0.0.4 + with: + dispatch-handler-token: ${{ secrets.SCDF_ACCESS_TOKEN }} + dispatch-handler-max: 20 + dispatch-handler-config: > + [ + { + "if": "initial == true", + "action": "workflow_dispatch", + "workflow_dispatch": { + "owner": "spring-cloud", + "repo": "spring-cloud-deployer", + "ref": "main", + "workflow": "release-worker.yml" + } + }, + { + "if": "data.event == 'build-succeed' && data.repo == 'spring-cloud-deployer' && data.owner == 'spring-cloud'", + "action": "workflow_dispatch", + "workflow_dispatch": { + "owner": "spring-cloud", + "repo": "spring-cloud-dataflow-ui", + "ref": "main", + "workflow": "release-worker.yml" + } + }, + { + "if": "data.event == 'build-succeed' && data.repo == 'spring-cloud-dataflow-ui' && data.owner == 'spring-cloud'", + "action": "workflow_dispatch", + "workflow_dispatch": { + "owner": "spring-cloud", + "repo": "spring-cloud-dataflow", + "ref": "main", + "workflow": "release-worker.yml" + } + }, + { + "if": "data.event == 'build-succeed' && data.repo == 'spring-cloud-dataflow' && data.owner == 'spring-cloud'", + "action": "workflow_dispatch", + "workflow_dispatch": { + "owner": "spring-cloud", + "repo": "spring-cloud-dataflow", + "ref": "main", + "workflow": "carvel-worker.yml" + } + }, + { + "if": "data.event == 'carvel-succeed' && data.repo == 'spring-cloud-dataflow' && data.owner == 'spring-cloud'", + "action": "workflow_dispatch", + "workflow_dispatch": { + "owner": "spring-cloud", + "repo": "spring-cloud-dataflow", + "ref": "main", + "workflow": "promote-release.yml" + } + }, + { + "if": "data.event == 'promote-succeed' && data.repo == 'spring-cloud-dataflow' && data.owner == 'spring-cloud'", + "action": "workflow_dispatch", + "workflow_dispatch": { + "owner": "spring-cloud", + "repo": "spring-cloud-dataflow", + "ref": "main", + "workflow": "central-release.yml" + } + }, + { + "if": "data.event == 'central-succeed' && data.repo == 'spring-cloud-dataflow' && data.owner == 'spring-cloud'", + "action": "workflow_dispatch", + "workflow_dispatch": { + "owner": "spring-cloud", + "repo": "spring-cloud-dataflow", + "ref": "main", + "workflow": "github-release.yml" + } + }, + { + "if": "data.event == 'promotion-failed'", + "action": "fail", + "fail": { + "message": "artifactory promotion failed" + } + }, + { + "if": "data.event == 'central-failed'", + "action": "fail", + "fail": { + "message": "central sync failed" + } + }, + { + "if": "data.event == 'ghrelease-failed'", + "action": "fail", + "fail": { + "message": "github release failed" + } + }, + { + "if": "data.event == 'build-failed'", + "action": "fail", + "fail": { + "message": "something went wrong in build train" + } } - }, - { - "if": "data.event == 'build-succeed' && data.repo == 'spring-cloud-deployer' && data.owner == 'spring-cloud'", - "action": "workflow_dispatch", - "workflow_dispatch": { - "owner": "spring-cloud", - "repo": "spring-cloud-dataflow-ui", - "ref": "main", - "workflow": "release-worker.yml" - } - }, - { - "if": "data.event == 'build-succeed' && data.repo == 'spring-cloud-dataflow-ui' && data.owner == 'spring-cloud'", - "action": "workflow_dispatch", - "workflow_dispatch": { - "owner": "spring-cloud", - "repo": "spring-cloud-dataflow", - "ref": "main", - "workflow": "release-worker.yml" - } - }, - { - "if": "data.event == 'build-succeed' && data.repo == 'spring-cloud-dataflow' && data.owner == 'spring-cloud'", - "action": "workflow_dispatch", - "workflow_dispatch": { - "owner": "spring-cloud", - "repo": "spring-cloud-dataflow", - "ref": "main", - "workflow": "promote-release.yml" - } - }, - { - "if": "data.event == 'promote-succeed' && data.repo == 'spring-cloud-dataflow' && data.owner == 'spring-cloud'", - "action": "workflow_dispatch", - "workflow_dispatch": { - "owner": "spring-cloud", - "repo": "spring-cloud-dataflow", - "ref": "main", - "workflow": "central-release.yml" - } - }, - { - "if": "data.event == 'central-succeed' && data.repo == 'spring-cloud-dataflow' && data.owner == 'spring-cloud'", - "action": "workflow_dispatch", - "workflow_dispatch": { - "owner": "spring-cloud", - "repo": "spring-cloud-dataflow", - "ref": "main", - "workflow": "github-release.yml" - } - }, - { - "if": "data.event == 'promotion-failed'", - "action": "fail", - "fail": { - "message": "artifactory promotion failed" - } - }, - { - "if": "data.event == 'central-failed'", - "action": "fail", - "fail": { - "message": "central sync failed" - } - }, - { - "if": "data.event == 'ghrelease-failed'", - "action": "fail", - "fail": { - "message": "github release failed" - } - }, - { - "if": "data.event == 'build-failed'", - "action": "fail", - "fail": { - "message": "something went wrong in build train" - } - } - ] + ] diff --git a/.github/workflows/release-worker.yml b/.github/workflows/release-worker.yml index 2450d2e8c4..7fd0dc692a 100644 --- a/.github/workflows/release-worker.yml +++ b/.github/workflows/release-worker.yml @@ -15,119 +15,119 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-java@v3 - with: - java-version: '8' - distribution: 'liberica' - - uses: jvalkeal/setup-maven@v1 - with: - maven-version: 3.8.8 - maven-mirror: 'https://dlcdn.apache.org/maven/maven-3/' - - uses: jfrog/setup-jfrog-cli@v1 - with: - version: 1.46.4 - env: - JF_ARTIFACTORY_SPRING: ${{ secrets.JF_ARTIFACTORY_SPRING }} - - uses: actions/cache@v2 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2- + - uses: actions/checkout@v2 + - uses: actions/setup-java@v3 + with: + java-version: '8' + distribution: 'liberica' + - uses: jvalkeal/setup-maven@v1 + with: + maven-version: 3.8.8 + maven-mirror: 'https://dlcdn.apache.org/maven/maven-3/' + - uses: jfrog/setup-jfrog-cli@v1 + with: + version: 1.46.4 + env: + JF_ARTIFACTORY_SPRING: ${{ secrets.JF_ARTIFACTORY_SPRING }} + - uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-m2- - # target deploy repos - - name: Configure JFrog Cli - run: | - jfrog rt mvnc \ - --server-id-resolve=repo.spring.io \ - --server-id-deploy=repo.spring.io \ - --repo-resolve-releases=libs-spring-dataflow-private-staging-release \ - --repo-resolve-snapshots=libs-snapshot \ - --repo-deploy-releases=libs-staging-local \ - --repo-deploy-snapshots=libs-snapshot-local - echo JFROG_CLI_BUILD_NAME=spring-cloud-dataflow-main-release >> $GITHUB_ENV - echo JFROG_CLI_BUILD_NUMBER=$GITHUB_RUN_NUMBER >> $GITHUB_ENV + # target deploy repos + - name: Configure JFrog Cli + run: | + jfrog rt mvnc \ + --server-id-resolve=repo.spring.io \ + --server-id-deploy=repo.spring.io \ + --repo-resolve-releases=libs-spring-dataflow-private-staging-release \ + --repo-resolve-snapshots=libs-snapshot \ + --repo-deploy-releases=libs-staging-local \ + --repo-deploy-snapshots=libs-snapshot-local + echo JFROG_CLI_BUILD_NAME=spring-cloud-dataflow-main-release >> $GITHUB_ENV + echo JFROG_CLI_BUILD_NUMBER=$GITHUB_RUN_NUMBER >> $GITHUB_ENV - # zoo extract and ensure - - name: Extract Zoo Context Properties - uses: jvalkeal/build-zoo-handler@v0.0.4 - with: - dispatch-handler-extract-context-properties: true - ensure-env: | - BUILD_ZOO_HANDLER_spring_cloud_deployer_version - BUILD_ZOO_HANDLER_spring_cloud_dataflow_ui_version + # zoo extract and ensure + - name: Extract Zoo Context Properties + uses: jvalkeal/build-zoo-handler@v0.0.4 + with: + dispatch-handler-extract-context-properties: true + ensure-env: | + BUILD_ZOO_HANDLER_spring_cloud_deployer_version + BUILD_ZOO_HANDLER_spring_cloud_dataflow_ui_version - # build and publish to configured target - - name: Build and Publish - run: | - jfrog rt mvn build-helper:parse-version versions:set \ - -gs .github/settings.xml \ - -Pstagingrelease \ - -DskipResolution=true \ - -DprocessAllModules=true \ - -DgenerateBackupPoms=false \ - -Dartifactory.publish.artifacts=false \ - -DnewVersion='${parsedVersion.majorVersion}.${parsedVersion.minorVersion}.${parsedVersion.incrementalVersion}' \ - -B - BUILD_ZOO_HANDLER_spring_cloud_dataflow_version=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) - echo BUILD_ZOO_HANDLER_spring_cloud_dataflow_version=$BUILD_ZOO_HANDLER_spring_cloud_dataflow_version >> $GITHUB_ENV - echo "::notice ::set dataflow.version=${BUILD_ZOO_HANDLER_spring_cloud_dataflow_version}" - jfrog rt mvn versions:set-property \ - -gs .github/settings.xml \ - -Pstagingrelease \ - -DskipResolution=true \ - -DgenerateBackupPoms=false \ - -Dproperty=dataflow.version \ - -DnewVersion=${BUILD_ZOO_HANDLER_spring_cloud_dataflow_version} \ - -B - echo "::notice ::set spring-cloud-deployer.version=${BUILD_ZOO_HANDLER_spring_cloud_deployer_version}" - jfrog rt mvn versions:set-property \ - -gs .github/settings.xml \ - -Pstagingrelease \ - -DskipResolution=true \ - -DgenerateBackupPoms=false \ - -Dproperty=spring-cloud-deployer.version \ - -DnewVersion=${BUILD_ZOO_HANDLER_spring_cloud_deployer_version} \ - -B - echo "::notice ::set spring-cloud-dataflow-ui.version=${BUILD_ZOO_HANDLER_spring_cloud_dataflow_ui_version}" - jfrog rt mvn versions:set-property \ - -gs .github/settings.xml \ - -DskipResolution=true \ - -Pstagingrelease \ - -DgenerateBackupPoms=false \ - -Dproperty=spring-cloud-dataflow-ui.version \ - -DnewVersion=${BUILD_ZOO_HANDLER_spring_cloud_dataflow_ui_version} \ - -B - echo "::notice ::build-clean" - jfrog rt build-clean - echo "::notice ::set-package-version" - ./spring-cloud-dataflow-package/set-package-version.sh - echo "::notice ::install" - jfrog rt mvn -gs .github/settings.xml -Pstagingrelease,full,deploymentfiles -B install -DskipTests - echo "::notice ::build-publish" - jfrog rt build-publish - spring_cloud_dataflow_version=$(mvn -gs .github/settings.xml -Pstagingrelease help:evaluate -Dexpression=project.version -q -DforceStdout) - echo BUILD_ZOO_HANDLER_spring_cloud_dataflow_version=$spring_cloud_dataflow_version >> $GITHUB_ENV - echo BUILD_ZOO_HANDLER_spring_cloud_dataflow_buildname=spring-cloud-dataflow-main-release >> $GITHUB_ENV - echo BUILD_ZOO_HANDLER_spring_cloud_dataflow_buildnumber=$GITHUB_RUN_NUMBER >> $GITHUB_ENV - echo BUILD_ZOO_HANDLER_spring_cloud_skipper_version=$spring_cloud_dataflow_version >> $GITHUB_ENV - - echo "::info ::BUILD_ZOO_HANDLER_spring_cloud_dataflow_version=$spring_cloud_dataflow_version" - echo "::info ::BUILD_ZOO_HANDLER_spring_cloud_dataflow_buildname=spring-cloud-dataflow-main-release" - echo "::info ::BUILD_ZOO_HANDLER_spring_cloud_dataflow_buildnumber=$GITHUB_RUN_NUMBER" - echo "::info ::BUILD_ZOO_HANDLER_spring_cloud_skipper_version=$spring_cloud_dataflow_version" - # zoo tag - - name: Tag Release - uses: jvalkeal/build-zoo-handler@v0.0.4 - with: - tag-release-branch: ${{ env.BUILD_ZOO_HANDLER_spring_cloud_dataflow_version }} - tag-release-tag: ${{ env.BUILD_ZOO_HANDLER_spring_cloud_dataflow_version }} - tag-release-tag-prefix: v - # clean m2 cache - - name: Clean cache - run: | - find ~/.m2/repository -type d -name '*SNAPSHOT' | xargs rm -fr + # build and publish to configured target + - name: Build and Publish + run: | + jfrog rt mvn build-helper:parse-version versions:set \ + -gs .github/settings.xml \ + -Pstagingrelease \ + -DskipResolution=true \ + -DprocessAllModules=true \ + -DgenerateBackupPoms=false \ + -Dartifactory.publish.artifacts=false \ + -DnewVersion='${parsedVersion.majorVersion}.${parsedVersion.minorVersion}.${parsedVersion.incrementalVersion}' \ + -B + BUILD_ZOO_HANDLER_spring_cloud_dataflow_version=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) + echo BUILD_ZOO_HANDLER_spring_cloud_dataflow_version=$BUILD_ZOO_HANDLER_spring_cloud_dataflow_version >> $GITHUB_ENV + echo "::notice ::set dataflow.version=${BUILD_ZOO_HANDLER_spring_cloud_dataflow_version}" + jfrog rt mvn versions:set-property \ + -gs .github/settings.xml \ + -Pstagingrelease \ + -DskipResolution=true \ + -DgenerateBackupPoms=false \ + -Dproperty=dataflow.version \ + -DnewVersion=${BUILD_ZOO_HANDLER_spring_cloud_dataflow_version} \ + -B + echo "::notice ::set spring-cloud-deployer.version=${BUILD_ZOO_HANDLER_spring_cloud_deployer_version}" + jfrog rt mvn versions:set-property \ + -gs .github/settings.xml \ + -Pstagingrelease \ + -DskipResolution=true \ + -DgenerateBackupPoms=false \ + -Dproperty=spring-cloud-deployer.version \ + -DnewVersion=${BUILD_ZOO_HANDLER_spring_cloud_deployer_version} \ + -B + echo "::notice ::set spring-cloud-dataflow-ui.version=${BUILD_ZOO_HANDLER_spring_cloud_dataflow_ui_version}" + jfrog rt mvn versions:set-property \ + -gs .github/settings.xml \ + -DskipResolution=true \ + -Pstagingrelease \ + -DgenerateBackupPoms=false \ + -Dproperty=spring-cloud-dataflow-ui.version \ + -DnewVersion=${BUILD_ZOO_HANDLER_spring_cloud_dataflow_ui_version} \ + -B + echo "::notice ::build-clean" + jfrog rt build-clean + echo "::notice ::set-package-version" + ./spring-cloud-dataflow-package/set-package-version.sh + echo "::notice ::install" + jfrog rt mvn -gs .github/settings.xml -Pstagingrelease,full,deploymentfiles -B install -DskipTests + echo "::notice ::build-publish" + jfrog rt build-publish + spring_cloud_dataflow_version=$(mvn -gs .github/settings.xml -Pstagingrelease help:evaluate -Dexpression=project.version -q -DforceStdout) + echo BUILD_ZOO_HANDLER_spring_cloud_dataflow_version=$spring_cloud_dataflow_version >> $GITHUB_ENV + echo BUILD_ZOO_HANDLER_spring_cloud_dataflow_buildname=spring-cloud-dataflow-main-release >> $GITHUB_ENV + echo BUILD_ZOO_HANDLER_spring_cloud_dataflow_buildnumber=$GITHUB_RUN_NUMBER >> $GITHUB_ENV + echo BUILD_ZOO_HANDLER_spring_cloud_skipper_version=$spring_cloud_dataflow_version >> $GITHUB_ENV + + echo "::info ::BUILD_ZOO_HANDLER_spring_cloud_dataflow_version=$spring_cloud_dataflow_version" + echo "::info ::BUILD_ZOO_HANDLER_spring_cloud_dataflow_buildname=spring-cloud-dataflow-main-release" + echo "::info ::BUILD_ZOO_HANDLER_spring_cloud_dataflow_buildnumber=$GITHUB_RUN_NUMBER" + echo "::info ::BUILD_ZOO_HANDLER_spring_cloud_skipper_version=$spring_cloud_dataflow_version" + # zoo tag + - name: Tag Release + uses: jvalkeal/build-zoo-handler@v0.0.4 + with: + tag-release-branch: ${{ env.BUILD_ZOO_HANDLER_spring_cloud_dataflow_version }} + tag-release-tag: ${{ env.BUILD_ZOO_HANDLER_spring_cloud_dataflow_version }} + tag-release-tag-prefix: v + # clean m2 cache + - name: Clean cache + run: | + find ~/.m2/repository -type d -name '*SNAPSHOT' | xargs rm -fr outputs: version: ${{ env.BUILD_ZOO_HANDLER_spring_cloud_dataflow_version }} BUILD_ZOO_HANDLER_spring_cloud_dataflow_version: ${{ env.BUILD_ZOO_HANDLER_spring_cloud_dataflow_version }} @@ -160,7 +160,7 @@ jobs: echo "BUILD_ZOO_HANDLER_spring_cloud_dataflow_buildnumber=${{ needs.build.outputs.BUILD_ZOO_HANDLER_spring_cloud_dataflow_buildnumber }}" >> $GITHUB_ENV echo "BUILD_ZOO_HANDLER_spring_cloud_skipper_version=${{ needs.build.outputs.BUILD_ZOO_HANDLER_spring_cloud_skipper_version }}" >> $GITHUB_ENV - # zoo success + # zoo success - name: Notify Build Success Zoo Handler Controller if: ${{ success() }} uses: jvalkeal/build-zoo-handler@v0.0.4