ci: speed up releases (#271) #635
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: "Commit" | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| branches: | |
| - main | |
| env: | |
| python_version: "3.10" | |
| jobs: | |
| lint: | |
| name: Lint | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout the repository | |
| uses: actions/checkout@v3 | |
| - uses: seisollc/goat@main | |
| with: | |
| exclude: (.*tests/(ansible|terraform|cloudformation)/.*|.*build/Dockerfile\.j2$) | |
| disable_mypy: true | |
| generate-matrixes: | |
| name: Generate matrixes for future use in pipelines | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| image-matrix: ${{ steps.set-image-outputs.outputs.image-matrix }} | |
| test-matrix: ${{ steps.set-testing-outputs.outputs.test-matrix }} | |
| steps: | |
| - name: Checkout the repository | |
| uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.python_version }} | |
| - uses: actions/cache@v3 | |
| with: | |
| path: ~/.local/share/virtualenvs | |
| key: ${{ runner.os }}-python-${{ env.python_version }}-pipenv-${{ hashFiles('Pipfile.lock') }} | |
| - name: Install the dependencies | |
| run: | | |
| python -m pip install --upgrade pipenv | |
| pipenv install --python ${{ env.python_version }} --deploy --ignore-pipfile --dev | |
| - name: Gather the image matrix | |
| id: set-image-outputs | |
| run: | | |
| pipenv run python -c \ | |
| 'from easy_infra import utils; \ | |
| print(utils.get_github_actions_matrix())' >> "${GITHUB_OUTPUT}" | |
| - name: Gather the testing matrix | |
| id: set-testing-outputs | |
| run: | | |
| pipenv run python -c \ | |
| 'from easy_infra import utils; \ | |
| print(utils.get_github_actions_matrix(testing=True))' >> "${GITHUB_OUTPUT}" | |
| test: | |
| name: Test | |
| needs: [generate-matrixes] | |
| if: github.event_name == 'pull_request' | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.generate-matrixes.outputs.test-matrix) }} | |
| steps: | |
| - name: Checkout the repository | |
| uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.python_version }} | |
| - uses: actions/cache@v3 | |
| with: | |
| path: ~/.local/share/virtualenvs | |
| key: ${{ runner.os }}-python-${{ env.python_version }}-pipenv-${{ hashFiles('Pipfile.lock') }} | |
| - name: Install the dependencies | |
| run: | | |
| python -m pip install --upgrade pipenv | |
| pipenv install --python ${{ env.python_version }} --deploy --ignore-pipfile --dev | |
| mkdir "${RUNNER_TEMP}/bin" | |
| curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b "${RUNNER_TEMP}/bin" | |
| chmod +x "${RUNNER_TEMP}/bin/syft" | |
| curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b "${RUNNER_TEMP}/bin" | |
| chmod +x "${RUNNER_TEMP}/bin/grype" | |
| echo "${RUNNER_TEMP}/bin" >> "${GITHUB_PATH}" | |
| - name: Build the image | |
| run: pipenv run invoke build --tool=${{ matrix.tool }} --environment=${{ matrix.environment }} | |
| - name: Generate the SBOM | |
| run: pipenv run invoke sbom --tool=${{ matrix.tool }} --environment=${{ matrix.environment }} | |
| - name: Upload the SBOM | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: SBOM_${{ matrix.tool }}_${{ matrix.environment }} | |
| path: sbom.*.json | |
| if-no-files-found: error | |
| - name: Generate Vuln scan results | |
| run: pipenv run invoke vulnscan --tool=${{ matrix.tool }} --environment=${{ matrix.environment }} | |
| - name: Upload Vuln scan result | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: Vulns_${{ matrix.tool }}_${{ matrix.environment }} | |
| path: vulns.*.json | |
| if-no-files-found: error | |
| - name: Run tests | |
| run: | | |
| find tests -mindepth 1 -type d -exec chmod o+w {} \; | |
| pipenv run invoke test --tool=${{ matrix.tool }} --environment=${{ matrix.environment }} --user=${{ matrix.user }} --debug | |
| bump-version: | |
| name: Bump version | |
| needs: [lint] | |
| if: "${{ github.event_name == 'push' && !startsWith(github.event.head_commit.message, 'Bump version: 2') }}" | |
| permissions: | |
| contents: write | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| git_tag: ${{ steps.bump-version.outputs.git_tag }} | |
| steps: | |
| - name: Checkout the repository | |
| uses: actions/checkout@v3 | |
| with: | |
| token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.python_version }} | |
| - uses: actions/cache@v3 | |
| with: | |
| path: ~/.local/share/virtualenvs | |
| key: ${{ runner.os }}-python-${{ env.python_version }}-pipenv-${{ hashFiles('Pipfile.lock') }} | |
| - name: Install the dependencies | |
| run: | | |
| python -m pip install --upgrade pipenv | |
| pipenv install --python ${{ env.python_version }} --deploy --ignore-pipfile --dev | |
| - name: Bump the version | |
| id: bump-version | |
| run: | | |
| git config --global user.name 'Seiso Automation' | |
| git config --global user.email '[email protected]' | |
| pipenv run invoke release | |
| GIT_TAG="$(git describe --tags)" | |
| BRANCH="$(git branch --show-current)" | |
| git push --atomic origin "${BRANCH}" "${GIT_TAG}" | |
| echo "git_tag=${GIT_TAG}" >> "${GITHUB_OUTPUT}" | |
| distribute: | |
| name: Distribute | |
| needs: [generate-matrixes, bump-version] | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.generate-matrixes.outputs.image-matrix) }} | |
| if: "${{ github.event_name == 'push' && !startsWith(github.event.head_commit.message, 'Bump version: 2') }}" | |
| environment: "${{ needs.bump-version.outputs.git_tag }}" | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout the repository | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| ref: "${{ needs.bump-version.outputs.git_tag }}" | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.python_version }} | |
| - uses: actions/cache@v3 | |
| with: | |
| path: ~/.local/share/virtualenvs | |
| key: ${{ runner.os }}-python-${{ env.python_version }}-pipenv-${{ hashFiles('Pipfile.lock') }} | |
| - name: Install the dependencies | |
| run: | | |
| python -m pip install --upgrade pipenv | |
| pipenv install --python ${{ env.python_version }} --deploy --ignore-pipfile --dev | |
| mkdir "${RUNNER_TEMP}/bin" | |
| curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b "${RUNNER_TEMP}/bin" | |
| chmod +x "${RUNNER_TEMP}/bin/syft" | |
| curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b "${RUNNER_TEMP}/bin" | |
| chmod +x "${RUNNER_TEMP}/bin/grype" | |
| echo "${RUNNER_TEMP}/bin" >> "${GITHUB_PATH}" | |
| - name: Build the image | |
| run: pipenv run invoke build --tool=${{ matrix.tool }} --environment=${{ matrix.environment }} | |
| - name: Generate the SBOM | |
| run: pipenv run invoke sbom --tool=${{ matrix.tool }} --environment=${{ matrix.environment }} | |
| - name: Upload the SBOM | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: SBOM_${{ matrix.tool }}_${{ matrix.environment }} | |
| path: sbom.*.json | |
| if-no-files-found: error | |
| - name: Generate Vuln scan results | |
| run: pipenv run invoke vulnscan --tool=${{ matrix.tool }} --environment=${{ matrix.environment }} | |
| - name: Upload Vuln scan result | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: Vulns_${{ matrix.tool }}_${{ matrix.environment }} | |
| path: vulns.*.json | |
| if-no-files-found: error | |
| - name: Run tests | |
| run: | | |
| find tests -mindepth 1 -type d -exec chmod o+w {} \; | |
| pipenv run invoke test --tool=${{ matrix.tool }} --environment=${{ matrix.environment }} --user=${{ matrix.user }} --debug | |
| - name: Log in to Docker Hub | |
| uses: docker/login-action@v2 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Publish the image to Docker Hub | |
| run: pipenv run invoke publish --tool=${{ matrix.tool }} --environment=${{ matrix.environment }} | |
| - name: Install cosign | |
| uses: sigstore/cosign-installer@main | |
| - name: Sign the image | |
| run: | | |
| image_and_versioned_tag=$(pipenv run python -c \ | |
| "from easy_infra.utils import get_image_and_tag; \ | |
| print(get_image_and_tag(tool='${TOOL}', environment='${ENVIRONMENT}'))") | |
| versioned_tag="${image_and_versioned_tag#*:}" | |
| # Requires that the image is available in the local daemon and was pushed to the remote repo | |
| image_and_digest=$(docker inspect --format='{{index .RepoDigests 0}}' \ | |
| "${image_and_versioned_tag}" ) | |
| echo -n "${COSIGN_PASSWORD}" | | |
| cosign sign --yes --key cosign.key \ | |
| -a git_sha="${GITHUB_SHA}" \ | |
| -a tag="${versioned_tag}" \ | |
| "${image_and_digest}" | |
| env: | |
| COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }} | |
| TOOL: ${{ matrix.tool }} | |
| ENVIRONMENT: ${{ matrix.environment }} | |
| finalize-the-release: | |
| name: Finalize the release | |
| needs: [bump-version, distribute] | |
| if: "${{ github.event_name == 'push' && !startsWith(github.event.head_commit.message, 'Bump version: 2') }}" | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout the repository | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| ref: "${{ needs.bump-version.outputs.git_tag }}" | |
| - name: Download the SBOMs and Vuln scan results | |
| uses: actions/download-artifact@v3 | |
| with: | |
| path: ${{ runner.temp }} | |
| - name: Publish the release to GitHub | |
| uses: softprops/action-gh-release@v1 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| name: ${{ steps.bump-version.outputs.git_tag }} | |
| tag_name: ${{ steps.bump-version.outputs.git_tag }} | |
| generate_release_notes: true | |
| files: | | |
| Vulns_*/vulns.*.json | |
| SBOM_*/sbom.*.json | |
| fail_on_unmatched_files: true | |
| draft: false | |
| prerelease: false | |
| - name: Publish the release README to Docker Hub | |
| uses: peter-evans/dockerhub-description@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| repository: seiso/easy_infra |