From 7ed9099ac52f76ece4b2f1d9f09949c53ce2a806 Mon Sep 17 00:00:00 2001 From: Chan <55515281+sichanyoo@users.noreply.github.com> Date: Tue, 24 Dec 2024 10:48:09 -0800 Subject: [PATCH] chore: Make GH workflows pass in staging & refactor using composite actions (#1848) * Update workflows so they pass in staging repo too. * Use composite action to reduce duplication --------- Co-authored-by: Sichan Yoo --- .../action.yml | 51 ++++++++++++ .github/workflows/codegen-build-test.yml | 40 +++++----- .github/workflows/continuous-integration.yml | 65 +++++++-------- .github/workflows/integration-test.yml | 80 +++++++++---------- 4 files changed, 137 insertions(+), 99 deletions(-) create mode 100644 .github/actions/checkout-smithy-swift-composite-action/action.yml diff --git a/.github/actions/checkout-smithy-swift-composite-action/action.yml b/.github/actions/checkout-smithy-swift-composite-action/action.yml new file mode 100644 index 00000000000..74f50db2c67 --- /dev/null +++ b/.github/actions/checkout-smithy-swift-composite-action/action.yml @@ -0,0 +1,51 @@ +name: 'Checkout smithy-swift composite action' +description: 'A composite action that sets up smithy-swift for the workflow.' +inputs: + AUTOMATION_USER_SSH_PRIVATE_KEY: + description: 'SSH private key for the automation user' + required: true + STAGING_PARTNER_REPO: + description: 'The staging partner repository' + required: true +runs: + using: 'composite' + steps: + - name: Set up SSH key + if: ${{ github.repository != 'awslabs/aws-sdk-swift' }} + run: | + mkdir -p ~/.ssh + echo "${{ inputs.AUTOMATION_USER_SSH_PRIVATE_KEY }}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + ssh-keyscan github.com >> ~/.ssh/known_hosts + shell: bash + - name: Select staging smithy-swift branch + if: ${{ github.repository != 'awslabs/aws-sdk-swift' }} + run: | + cd aws-sdk-swift + ORIGINAL_REPO_HEAD_REF="$GITHUB_HEAD_REF" \ + DEPENDENCY_REPO_URL="git@github.com:${{ inputs.STAGING_PARTNER_REPO }}.git" \ + ./scripts/ci_steps/select_dependency_branch.sh + shell: bash + - name: Checkout staging smithy-swift + if: ${{ github.repository != 'awslabs/aws-sdk-swift' }} + uses: actions/checkout@v4 + with: + repository: ${{ inputs.STAGING_PARTNER_REPO }} + ref: ${{ env.DEPENDENCY_REPO_SHA }} + path: smithy-swift + ssh-key: ${{ inputs.AUTOMATION_USER_SSH_PRIVATE_KEY }} + - name: Select smithy-swift branch + if: ${{ github.repository == 'awslabs/aws-sdk-swift' }} + run: | + cd aws-sdk-swift + ORIGINAL_REPO_HEAD_REF="$GITHUB_HEAD_REF" \ + DEPENDENCY_REPO_URL="https://github.com/smithy-lang/smithy-swift.git" \ + ./scripts/ci_steps/select_dependency_branch.sh + shell: bash + - name: Checkout smithy-swift + if: ${{ github.repository == 'awslabs/aws-sdk-swift' }} + uses: actions/checkout@v4 + with: + repository: smithy-lang/smithy-swift + ref: ${{ env.DEPENDENCY_REPO_SHA }} + path: smithy-swift \ No newline at end of file diff --git a/.github/workflows/codegen-build-test.yml b/.github/workflows/codegen-build-test.yml index b7317296a76..277b802dd42 100644 --- a/.github/workflows/codegen-build-test.yml +++ b/.github/workflows/codegen-build-test.yml @@ -19,22 +19,16 @@ jobs: DEVELOPER_DIR: /Applications/Xcode_16.1.app/Contents/Developer steps: - name: Checkout aws-sdk-swift - uses: actions/checkout@v3 - - name: Select smithy-swift branch - run: | - ORIGINAL_REPO_HEAD_REF="${GITHUB_HEAD_REF:-$GITHUB_REF_NAME}" \ - DEPENDENCY_REPO_URL="https://github.com/smithy-lang/smithy-swift.git" \ - ./scripts/ci_steps/select_dependency_branch.sh - - name: Checkout smithy-swift - uses: actions/checkout@v3 + uses: actions/checkout@v4 + with: + path: aws-sdk-swift + - name: Checkout smithy-swift with composite action + uses: ./aws-sdk-swift/.github/actions/checkout-smithy-swift-composite-action with: - repository: smithy-lang/smithy-swift - ref: ${{ env.DEPENDENCY_REPO_SHA }} - path: smithy-swift - - name: Move smithy-swift into place - run: mv smithy-swift .. + AUTOMATION_USER_SSH_PRIVATE_KEY: ${{ secrets.AUTOMATION_USER_SSH_PRIVATE_KEY }} + STAGING_PARTNER_REPO: ${{ secrets.STAGING_PARTNER_REPO }} - name: Cache Gradle - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | ~/.gradle/caches @@ -44,7 +38,7 @@ jobs: 2-${{ runner.os }}-gradle-${{ hashFiles('*.gradle.kts', 'gradle/wrapper/gradle-wrapper.properties') }} 2-${{ runner.os }}-gradle- - name: Cache Swift - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | ~/Library/Caches/org.swift.swiftpm @@ -54,15 +48,21 @@ jobs: 1-${{ runner.os }}-${{ matrix.xcode }}-${{ hashFiles('Package.swift', 'AWSSDKSwiftCLI/Package.swift') }} 1-${{ runner.os }}-${{ matrix.xcode }}- - name: Setup Java - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: corretto java-version: 17 - name: Tools Versions - run: ./scripts/ci_steps/log_tool_versions.sh + run: ./aws-sdk-swift/scripts/ci_steps/log_tool_versions.sh - name: Code-Generate SDK - run: ./scripts/ci_steps/codegen_sdk.sh + run: | + cd aws-sdk-swift + ./scripts/ci_steps/codegen_sdk.sh - name: Build SDK with Unit Tests - run: swift build --build-tests + run: | + cd aws-sdk-swift + swift build --build-tests - name: Run Unit Tests - run: swift test --skip-build + run: | + cd aws-sdk-swift + swift test --skip-build diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index b066f12ffc2..f5a6a3e3b22 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -64,19 +64,13 @@ jobs: sudo xcodebuild -runFirstLaunch - name: Checkout aws-sdk-swift uses: actions/checkout@v4 - - name: Select smithy-swift branch - run: | - ORIGINAL_REPO_HEAD_REF="$GITHUB_HEAD_REF" \ - DEPENDENCY_REPO_URL="https://github.com/smithy-lang/smithy-swift.git" \ - ./scripts/ci_steps/select_dependency_branch.sh - - name: Checkout smithy-swift - uses: actions/checkout@v4 with: - repository: smithy-lang/smithy-swift - ref: ${{ env.DEPENDENCY_REPO_SHA }} - path: smithy-swift - - name: Move smithy-swift into place - run: mv smithy-swift .. + path: aws-sdk-swift + - name: Checkout smithy-swift with composite action + uses: ./aws-sdk-swift/.github/actions/checkout-smithy-swift-composite-action + with: + AUTOMATION_USER_SSH_PRIVATE_KEY: ${{ secrets.AUTOMATION_USER_SSH_PRIVATE_KEY }} + STAGING_PARTNER_REPO: ${{ secrets.STAGING_PARTNER_REPO }} - name: Cache Gradle uses: actions/cache@v4 with: @@ -103,20 +97,22 @@ jobs: distribution: corretto java-version: 17 - name: Tools Versions - run: ./scripts/ci_steps/log_tool_versions.sh + run: ./aws-sdk-swift/scripts/ci_steps/log_tool_versions.sh - name: Run CLI Unit Tests if: ${{ matrix.destination == 'platform=macOS' }} run: | - cd AWSSDKSwiftCLI + cd aws-sdk-swift/AWSSDKSwiftCLI swift test cd ../SPRCLI unset AWS_SWIFT_SDK_USE_LOCAL_DEPS swift build - name: Prepare Protocol & Unit Tests run: | + cd aws-sdk-swift ./scripts/ci_steps/prepare_protocol_and_unit_tests.sh - name: Build and Run Unit Tests run: | + cd aws-sdk-swift set -o pipefail && \ NSUnbufferedIO=YES xcodebuild \ -scheme aws-sdk-swift-Package \ @@ -125,7 +121,7 @@ jobs: | xcbeautify - name: Build and Run Protocol Tests run: | - cd codegen/ + cd aws-sdk-swift/codegen/ set -o pipefail && \ NSUnbufferedIO=YES xcodebuild \ -scheme aws-sdk-swift-protocol-tests-Package \ @@ -150,22 +146,16 @@ jobs: ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true steps: - name: Checkout aws-sdk-swift - uses: actions/checkout@v3 - - name: Select smithy-swift branch - run: | - ORIGINAL_REPO_HEAD_REF="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME:-main}}" \ - DEPENDENCY_REPO_URL="https://github.com/smithy-lang/smithy-swift.git" \ - ./scripts/ci_steps/select_dependency_branch.sh - - name: Checkout smithy-swift - uses: actions/checkout@v3 + uses: actions/checkout@v4 + with: + path: aws-sdk-swift + - name: Checkout smithy-swift with composite action + uses: ./aws-sdk-swift/.github/actions/checkout-smithy-swift-composite-action with: - repository: smithy-lang/smithy-swift - ref: ${{ env.DEPENDENCY_REPO_SHA }} - path: smithy-swift - - name: Move smithy-swift into place - run: mv smithy-swift .. + AUTOMATION_USER_SSH_PRIVATE_KEY: ${{ secrets.AUTOMATION_USER_SSH_PRIVATE_KEY }} + STAGING_PARTNER_REPO: ${{ secrets.STAGING_PARTNER_REPO }} - name: Cache Gradle - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | ~/.gradle/caches @@ -175,7 +165,7 @@ jobs: 2-${{ runner.os }}-gradle-${{ hashFiles('*.gradle.kts', 'gradle/wrapper/gradle-wrapper.properties') }} 2-${{ runner.os }}-gradle- - name: Cache Swift - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | ~/Library/Caches/org.swift.swiftpm @@ -185,28 +175,31 @@ jobs: 1-${{ runner.os }}-swift-${{ matrix.version }}-spm-${{ hashFiles('Package.swift', 'AWSSDKSwiftCLI/Package.swift') }} 1-${{ runner.os }}-swift-${{ matrix.version }}-spm- - name: Setup Java - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: corretto java-version: 17 - name: Install OpenSSL (all OS) and which (AL2 only) - run: ./scripts/ci_steps/install_native_linux_dependencies.sh + run: ./aws-sdk-swift/scripts/ci_steps/install_native_linux_dependencies.sh - name: Tools Versions - run: ./scripts/ci_steps/log_tool_versions.sh + run: ./aws-sdk-swift/scripts/ci_steps/log_tool_versions.sh - name: Run CLI Unit Tests run: | - cd AWSSDKSwiftCLI + cd aws-sdk-swift/AWSSDKSwiftCLI swift test cd ../SPRCLI unset AWS_SWIFT_SDK_USE_LOCAL_DEPS swift build - name: Prepare Protocol & Unit Tests run: | + cd aws-sdk-swift ./scripts/ci_steps/prepare_protocol_and_unit_tests.sh - name: Build and Run Unit Tests on Linux - run: swift test + run: | + cd aws-sdk-swift + swift test - name: Build and Run Protocol Tests on Linux run: | - cd codegen/ + cd aws-sdk-swift/codegen/ swift test cd .. diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index a26d5146d53..ec7ce42a0df 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -11,6 +11,8 @@ env: permissions: id-token: write + contents: read + actions: read jobs: apple: @@ -71,22 +73,16 @@ jobs: role-to-assume: ${{ secrets.INTEGRATION_TEST_ROLE_ARN }} aws-region: us-west-2 - name: Checkout aws-sdk-swift - uses: actions/checkout@v3 - - name: Select smithy-swift branch - run: | - ORIGINAL_REPO_HEAD_REF="$GITHUB_HEAD_REF" \ - DEPENDENCY_REPO_URL="https://github.com/smithy-lang/smithy-swift.git" \ - ./scripts/ci_steps/select_dependency_branch.sh - - name: Checkout smithy-swift - uses: actions/checkout@v3 + uses: actions/checkout@v4 + with: + path: aws-sdk-swift + - name: Checkout smithy-swift with composite action + uses: ./aws-sdk-swift/.github/actions/checkout-smithy-swift-composite-action with: - repository: smithy-lang/smithy-swift - ref: ${{ env.DEPENDENCY_REPO_SHA }} - path: smithy-swift - - name: Move smithy-swift into place - run: mv smithy-swift .. + AUTOMATION_USER_SSH_PRIVATE_KEY: ${{ secrets.AUTOMATION_USER_SSH_PRIVATE_KEY }} + STAGING_PARTNER_REPO: ${{ secrets.STAGING_PARTNER_REPO }} - name: Cache Gradle - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | ~/.gradle/caches @@ -96,7 +92,7 @@ jobs: 2-${{ runner.os }}-gradle-${{ hashFiles('*.gradle.kts', 'gradle/wrapper/gradle-wrapper.properties') }} 2-${{ runner.os }}-gradle- - name: Cache Swift - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | ~/Library/Caches/org.swift.swiftpm @@ -106,14 +102,15 @@ jobs: 1-${{ runner.os }}-${{ matrix.xcode }}-${{ hashFiles('Package.swift', 'AWSSDKSwiftCLI/Package.swift') }} 1-${{ runner.os }}-${{ matrix.xcode }}- - name: Setup Java - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: corretto java-version: 17 - name: Tools Versions - run: ./scripts/ci_steps/log_tool_versions.sh + run: ./aws-sdk-swift/scripts/ci_steps/log_tool_versions.sh - name: Add Credentials to Test Plan run: | + cd aws-sdk-swift # JSON-escape the credentials. They are also surrounded with double quotes. AKID_ESCAPED=`echo -n "$AWS_ACCESS_KEY_ID" | jq -Rsa .` SECRET_ESCAPED=`echo -n "$AWS_SECRET_ACCESS_KEY" | jq -Rsa .` @@ -124,10 +121,12 @@ jobs: jq ".defaultOptions.environmentVariableEntries += [{\"key\": \"AWS_ACCESS_KEY_ID\", \"value\": $AKID_ESCAPED}, {\"key\": \"AWS_SECRET_ACCESS_KEY\", \"value\": $SECRET_ESCAPED}, {\"key\": \"AWS_DEFAULT_REGION\", \"value\": $REGION_ESCAPED}, {\"key\": \"AWS_SESSION_TOKEN\", \"value\": $TOKEN_ESCAPED}]" IntegrationTests/XCTestPlans/AWSIntegrationTestsOnCI.xctestplan > testplan.tmp mv testplan.tmp IntegrationTests/XCTestPlans/AWSIntegrationTestsOnCI.xctestplan - name: Prepare Integration Tests - run: ./scripts/ci_steps/prepare_integration_tests.sh + run: | + cd aws-sdk-swift + ./scripts/ci_steps/prepare_integration_tests.sh - name: Run Integration Tests run: | - cd IntegrationTests + cd aws-sdk-swift/IntegrationTests set -o pipefail && \ NSUnbufferedIO=YES xcodebuild \ -scheme AWSIntegrationTestsOnCI \ @@ -137,6 +136,7 @@ jobs: | xcbeautify linux: + # Details on why AL2 isn't included: https://github.com/awslabs/aws-sdk-swift/pull/1833 if: github.repository == 'awslabs/aws-sdk-swift' || github.event_name == 'pull_request' runs-on: ubuntu-latest container: swift:${{ matrix.version }}-${{ matrix.os }} @@ -159,22 +159,16 @@ jobs: role-to-assume: ${{ secrets.INTEGRATION_TEST_ROLE_ARN }} aws-region: us-west-2 - name: Checkout aws-sdk-swift - uses: actions/checkout@v3 - - name: Select smithy-swift branch - run: | - ORIGINAL_REPO_HEAD_REF="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME:-main}}" \ - DEPENDENCY_REPO_URL="https://github.com/smithy-lang/smithy-swift.git" \ - ./scripts/ci_steps/select_dependency_branch.sh - - name: Checkout smithy-swift - uses: actions/checkout@v3 + uses: actions/checkout@v4 + with: + path: aws-sdk-swift + - name: Checkout smithy-swift with composite action + uses: ./aws-sdk-swift/.github/actions/checkout-smithy-swift-composite-action with: - repository: smithy-lang/smithy-swift - ref: ${{ env.DEPENDENCY_REPO_SHA }} - path: smithy-swift - - name: Move smithy-swift into place - run: mv smithy-swift .. + AUTOMATION_USER_SSH_PRIVATE_KEY: ${{ secrets.AUTOMATION_USER_SSH_PRIVATE_KEY }} + STAGING_PARTNER_REPO: ${{ secrets.STAGING_PARTNER_REPO }} - name: Cache Gradle - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | ~/.gradle/caches @@ -184,7 +178,7 @@ jobs: 2-${{ runner.os }}-gradle-${{ hashFiles('*.gradle.kts', 'gradle/wrapper/gradle-wrapper.properties') }} 2-${{ runner.os }}-gradle- - name: Cache Swift - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | ~/Library/Caches/org.swift.swiftpm @@ -194,23 +188,23 @@ jobs: 1-${{ runner.os }}-swift-${{ matrix.version }}-spm-${{ hashFiles('Package.swift', 'AWSSDKSwiftCLI/Package.swift') }} 1-${{ runner.os }}-swift-${{ matrix.version }}-spm- - name: Setup Java - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: corretto java-version: 17 - name: Install OpenSSL (all OS) and which (AL2 only) - run: ./scripts/ci_steps/install_native_linux_dependencies.sh + run: ./aws-sdk-swift/scripts/ci_steps/install_native_linux_dependencies.sh - name: Tools Versions - run: ./scripts/ci_steps/log_tool_versions.sh + run: ./aws-sdk-swift/scripts/ci_steps/log_tool_versions.sh - name: Prepare Integration Tests - run: ./scripts/ci_steps/prepare_integration_tests.sh + run: | + cd aws-sdk-swift + ./scripts/ci_steps/prepare_integration_tests.sh - name: Build Integration Tests run: | - cd IntegrationTests + cd aws-sdk-swift/IntegrationTests swift build --build-tests - cd .. - name: Run Integration Tests run: | - cd IntegrationTests - swift test - cd .. + cd aws-sdk-swift/IntegrationTests + swift test \ No newline at end of file