diff --git a/.github/workflows/bootstrap.yaml b/.github/workflows/bootstrap.yaml index a1ebb30..41dac5a 100644 --- a/.github/workflows/bootstrap.yaml +++ b/.github/workflows/bootstrap.yaml @@ -12,7 +12,7 @@ jobs: name: "Update DevOps tooling" runs-on: ubuntu-latest permissions: - # IMPORTANT: mandatory to update content/actions/PRs + # IMPORTANT: mandatory to create or update content/actions/pr contents: write actions: write pull-requests: write @@ -20,11 +20,8 @@ jobs: steps: - name: "Checkout primary repository" uses: actions/checkout@v4 - with: - # Note: Requires a specific/defined Personal Access Token - token: ${{ secrets.ACTIONS_WORKFLOW }} - - name: "Pull workflows from central repository" + - name: "Pull devops content from repository" uses: actions/checkout@v4 with: repository: "os-climate/devops-toolkit" @@ -36,115 +33,307 @@ jobs: GH_TOKEN: ${{ github.token }} # yamllint disable rule:line-length run: | - ### SHELL CODE START ### + #SHELLCODESTART + set -euo pipefail + # set -x + + # Define variables + + DEVOPS_DIR=".devops" + AUTOMATION_BRANCH="update-devops-tooling" REPO_DIR=$(git rev-parse --show-toplevel) - # Ensure working from top-level of GIT repository - CURRENT_DIR=$(pwd) - if [ "$REPO_DIR" != "$CURRENT_DIR" ]; then - echo "Changing directory to: $REPO_DIR" - if ! (cd "$REPO_DIR"); then - echo "Error: unable to change directory"; exit 1 - fi - fi + GIT_ORIGIN=$(git config --get remote.origin.url) + REPO_NAME=$(basename -s .git "$GIT_ORIGIN") + EXCLUDE_FILE=".devops-exclusions" + DEVOPS_REPO='git@github.com:os-climate/devops-toolkit.git' + HEAD_BRANCH=$(git rev-parse --abbrev-ref HEAD) + + # Content folder defines the files and folders to update + FILES="$DEVOPS_DIR/content/files.txt" + FOLDERS="$DEVOPS_DIR/content/folders.txt" - # Define a function to allow selective opt-out of devops tooling - OPT_OUT=".devops-exclusions" + # Define functions + + perform_folder_operation() { + FS_PATH="$1" + if [ -d "$DEVOPS_DIR"/"$FS_PATH" ]; then + echo "Scanning target folder content at: $FS_PATH" + return 0 + else + echo "Upstream folder NOT found: $FS_PATH [skipping]" + return 1 + fi + } + + # Allows for selective opt-out components on a per-path basis perform_operation() { - ELEMENT="$1" - if [ ! -f "$OPT_OUT" ]; then - # Opt-out file does not exist; all operations will be performed + FS_PATH="$1" + if [ ! -f "$DEVOPS_DIR"/"$FS_PATH" ]; then + echo "Skipping missing upstream file at: $FS_PATH" + return 1 + fi + # Elements excluded from processing return exit status 1 + if [ ! -f "$EXCLUDE_FILE" ]; then + return 0 + elif [ "$FS_PATH" = "$EXCLUDE_FILE" ]; then + # The exclusion file itself is never updated by automation + return 1 + elif (grep -Fxq "$FS_PATH" "$EXCLUDE_FILE" > /dev/null); then + # Element listed; exclude from processing return 1 else - if grep -Fxq "$ELEMENT" "$OPT_OUT" - then - # Element is excluded from processing + # Element not found in exclusion file; process it + return 0 + fi + } + + # Only updates file if it has changed + selective_file_copy() { + # Receives a single file path as argument + SHA_SRC=$(sha1sum "$DEVOPS_DIR"/"$1" | awk '{print $1}') + SHA_DST=$(sha1sum "$1" 2>/dev/null | awk '{print $1}' || :) + if [ "$SHA_SRC" != "$SHA_DST" ]; then + echo "Copying: $1" + cp "$DEVOPS_DIR/$1" "$1" + git add "$1" + fi + } + + check_pr_for_author() { + AUTHOR="$1" + printf "Checking for pull requests by: %s" "$AUTHOR" + # Capture the existing PR number + PR_NUM=$(gh pr list --state open -L 1 \ + --author "$AUTHOR" --json number | \ + grep "number" | sed "s/:/ /g" | awk '{print $2}' | \ + sed "s/}//g" | sed "s/]//g") + if [ -z "$PR_NUM" ]; then + echo " [none]" + return 1 + else + echo " [$PR_NUM]" + echo "Running: gh pr checkout $PR_NUM" + if (gh pr checkout "$PR_NUM"); then return 0 else - # Element should be processed - return 1 + echo "Failed to checkout GitHub pull request" + echo "Check errors/output for the cause" + return 2 fi fi } - echo "Removing remote branch if it exists: update-devops-tooling" - git push origin --delete update-devops-tooling || : - STRING=$(dd if=/dev/urandom bs=1k count=1 2>/dev/null | tr -dc 'a-zA-Z0-9' | head -c 10) - git checkout -b "update-$STRING" + check_prs() { + # Define users to check for pre-existing pull requests + AUTOMATION_USER="github-actions[bot]" + if [[ -n ${GH_TOKEN+x} ]]; then + GITHUB_USERS="$AUTOMATION_USER" + else + GITHUB_USERS=$(gh api user | jq -r '.login') + # Check local user account first, if enumerated + GITHUB_USERS+=" $AUTOMATION_USER" + fi + + # Check for existing pull requests opened by this automation + for USER in $GITHUB_USERS; do + if (check_pr_for_author "$USER"); then + return 0 + else + STATUS="$?" + fi + if [ "$STATUS" -eq 1 ]; then + continue + elif [ "$STATUS" -eq 2 ]; then + echo "Failed to checkout pull request"; exit 1 + fi + done + return 1 + } + + # Check if script is running in GHA workflow + in_github() { + if [ -z ${GITHUB_RUN_ID+x} ]; then + echo "Script is NOT running in GitHub" + return 1 + else + echo "Script is running in GitHub" + return 0 + fi + } - # Configure GIT - TEST=$(git config -l) + # Check if user is logged into GitHub + logged_in_github() { + if (gh auth status); then + echo "Logged in and authenticated to GitHb" + return 0 + else + echo "Not logged into GitHub, some script operations unavailable" + return 1 + fi + } + + # Main script entry point + + echo "Repository name and HEAD branch: $REPO_NAME [$HEAD_BRANCH]" + + # Ensure working from top-level of GIT repository + CURRENT_DIR=$(pwd) + if [ "$REPO_DIR" != "$CURRENT_DIR" ]; then + echo "Changing directory to: $REPO_DIR" + if ! (cd "$REPO_DIR"); then + echo "Error: unable to change directory"; exit 1 + fi + fi + + # Stashing only used during development/testing + # Check if there are unstaged changes + # if ! (git diff --exit-code --quiet); then + # echo "Stashing unstaged changes in current repository" + # git stash -q + # fi + + # Configure GIT environment only if NOT already configured + # i.e. when running in a GitHub Actions workflow + TEST=$(git config -l > /dev/null 2>&1) if [ -n "$TEST" ]; then git config user.name "github-actions[bot]" git config user.email \ "41898282+github-actions[bot]@users.noreply.github.com" fi - FOLDERS=".github .github/workflows scripts" - for FOLDER in ${FOLDERS}; do - # Check to see if operation should be skipped - if (perform_operation "$FOLDER"); then - echo "Opted out of DevOps folder: $FOLDER" - continue + + if ! (check_prs); then + # No existing open pull requests found for this repository + + # Remove remote branch if it exists + git push origin --delete "$AUTOMATION_BRANCH" > /dev/null 2>&1 || : + git branch -D "$AUTOMATION_BRANCH" || : + git checkout -b "$AUTOMATION_BRANCH" + else + # The -B flag swaps branch and creates it if NOT present + git checkout -B "$AUTOMATION_BRANCH" + fi + + # Only if NOT running in GitHub + # (checkout is otherwise performed by earlier steps) + if ! (in_github); then + # Remove any stale local copy of the upstream repository + if [ -d "$DEVOPS_DIR" ]; then + rm -Rf "$DEVOPS_DIR" + fi + printf "Cloning DevOps repository into: %s" "$DEVOPS_DIR" + if (git clone "$DEVOPS_REPO" "$DEVOPS_DIR" > /dev/null 2>&1); then + echo " [success]" else - # If necessary, create target folder - if [ ! -d "$FOLDER" ]; then - echo "Creating target folder: $FOLDER" - mkdir "$FOLDER" - fi - # Update folder contents - echo "Updating folder contents: $FOLDER" - cp -a .devops/"$FOLDER"/. "$FOLDER" + echo " [failed]"; exit 1 fi - done + fi + + # Process upstream DevOps repository content and update + + LOCATIONS="" + # Populate list of files to be updated/sourced + while read -ra LINE; + do + for FILE in "${LINE[@]}"; + do + LOCATIONS+="$FILE " + done + done < "$FILES" + + # Gather files from specified folders and append to locations list + while read -ra LINE; + do + for FOLDER in "${LINE[@]}"; + do + # Check to see if this folder should be skipped + if (perform_folder_operation "$FOLDER"); then + # If necessary, create target folder + if [ ! -d "$FOLDER" ]; then + echo "Creating target folder: $FOLDER" + mkdir "$FOLDER" + fi + # Add folder contents to list of file LOCATIONS + FILES=$(cd "$DEVOPS_DIR/$FOLDER"; find . -maxdepth 1 -type f -exec basename {} \;) + for LOCATION in $FILES; do + # Also check if individual files in the folder are excluded + if (perform_operation "$FOLDER/$LOCATION"); then + LOCATIONS+=" $FOLDER/$LOCATION" + fi + done + else + echo "Opted out of folder: $FOLDER" + continue + fi + done; + done < "$FOLDERS" # Copy specified files into repository root - FILES=".pre-commit-config.yaml .prettierignore .gitignore" - for FILE in ${FILES}; do - if (perform_operation "$FILE"); then - echo "Opted out of DevOps file: $FILE" + for LOCATION in ${LOCATIONS}; do + if (perform_operation "$LOCATION"); then + selective_file_copy "$LOCATION" else - echo "Copying file: $FILE" - cp .devops/"$FILE" "$FILE" + echo "Not updating: $LOCATION" fi done # If no changes required, do not throw an error if [ -z "$(git status --porcelain)" ]; then echo "No updates/changes to commit"; exit 0 - else - # Set a flag for use by the next action/step + fi + + # Temporarily disable exit on unbound variable + set +eu +o pipefail + + # Next step is only performed if running as GitHub Action + if [[ -n ${GH_TOKEN+x} ]]; then + # Script is running in a GitHub actions workflow + # Set outputs for use by the next actions/steps + # shellcheck disable=SC2129 echo "changed=true" >> "$GITHUB_OUTPUT" + echo "branchname=$AUTOMATION_BRANCH" >> "$GITHUB_OUTPUT" + echo "headbranch=$HEAD_BRANCH" >> "$GITHUB_OUTPUT" + # Move to the next workflow step to raise the PR + git push --set-upstream origin "$AUTOMATION_BRANCH" + exit 0 fi - if [ -n "$GITHUB_TOKEN" ]; then - git add . - if ! (git commit -as -S -m "Chore: Update DevOps tooling from central repository [skip-ci]" \ - -m "This commit created by automation/scripting" --no-verify); then - echo "Commit failed; aborting"; exit 1 - else - git push --set-upstream origin update-devops-tooling - # ToDo: need to verify if we are running in a GHA - gh pr create --title \ - "Chore: Pull DevOps tooling from upstream repository" \ - --body 'Automated by a GitHub workflow: bootstrap.yaml' - fi + + # If running shell code locally, continue to raise the PR + + # Reinstate exit on unbound variables + set -euo pipefail + + git status + if ! (git commit -as -S -m "Chore: Update DevOps tooling from central repository [skip ci]" \ + -m "This commit created by automation/scripting" --no-verify); then + echo "Commit failed; aborting"; exit 1 else - echo "Script running in GitHub Actions workflow; proceeding to next step" + # Push branch to remote repository + git push --set-upstream origin "$AUTOMATION_BRANCH" + # Create PR request + gh pr create \ + --title "Chore: Pull DevOps tooling from upstream repository" \ + --body 'Automated by a GitHub workflow: bootstrap.yaml' fi - ### SHELL CODE END ### + # echo "Unstashing unstaged changes, if any exist" + # git stash pop -q || : + #SHELLCODEEND - name: Create Pull Request if: steps.update-repository.outputs.changed == 'true' - uses: peter-evans/create-pull-request@v5 - env: - GITHUB_TOKEN: ${{ github.token }} + uses: peter-evans/create-pull-request@v6 + # env: + # GITHUB_TOKEN: ${{ github.token }} with: - token: ${{ github.token }} - commit-message: "Chore: Update DevOps tooling from central repository [skip-ci]" + # Note: Requires a specific/defined Personal Access Token + token: ${{ secrets.ACTIONS_WORKFLOW }} + commit-message: "Chore: Update DevOps tooling from central repository [skip ci]" signoff: "true" - branch: update-devops-tooling + base: ${{ steps.update-repository.outputs.headbranch }} + branch: ${{ steps.update-repository.outputs.branchname }} delete-branch: true - title: "Chore: Update DevOps tooling from central repository [skip-ci]" + title: "Chore: Update DevOps tooling from central repository [skip ci]" body: | Update repository with content from upstream: os-climate/devops-toolkit labels: | diff --git a/.github/workflows/builds.yaml b/.github/workflows/builds.yaml index 8de5ef0..1c7f171 100644 --- a/.github/workflows/builds.yaml +++ b/.github/workflows/builds.yaml @@ -1,5 +1,5 @@ --- -name: "🧪 Test builds (Matrix)" +name: "🧱 Builds (Matrix)" # yamllint disable-line rule:truthy on: @@ -11,15 +11,21 @@ on: - "!update-devops-tooling" jobs: - pre-release: + parse-project-metadata: + name: "Determine Python versions" + # yamllint disable-line rule:line-length + uses: os-climate/devops-reusable-workflows/.github/workflows/pyproject-toml-fetch-matrix.yaml@main + + test-builds: + name: "Build: Python" + needs: [parse-project-metadata] runs-on: "ubuntu-latest" continue-on-error: true # Don't run when pull request is merged if: github.event.pull_request.merged == false strategy: fail-fast: false - matrix: - python-version: ["3.9", "3.10", "3.11"] + matrix: ${{ fromJson(needs.parse-project-metadata.outputs.matrix) }} steps: - name: "Populate environment variables" @@ -41,10 +47,10 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: "Install dependencies" - run: | - python -m pip install --upgrade pip - pip install tox tox-gh-actions + - name: "Setup PDM for build commands" + uses: pdm-project/setup-pdm@v4 + with: + python-version: ${{ matrix.python-version }} - name: "Tag for test release" # Delete all local tags, then create a synthetic tag for testing @@ -56,6 +62,27 @@ jobs: git checkout "tags/v${{ steps.setenv.outputs.vernum }}" grep version pyproject.toml - - name: "Build with TOX" + - name: "Performing build" + run: | + python -m pip install --upgrade pip + if [ -f tox.ini ]; then + pip install tox tox-gh-actions + echo "Found file: tox.ini" + echo "Building with command: tox -e build" + tox -e build + elif [ -f pyproject.toml ]; then + echo "Found file: pyproject.toml" + echo "Building with command: pdm build" + pdm build + else + echo "Neither file found: tox.ini/pyproject.toml" + pip install --upgrade build + echo "Attempting build with: python -m build" + python -m build + fi + + - name: "Validating Artefacts with Twine" run: | - tox -e build + echo "Validating artefacts with: twine check dist/*" + pip install --upgrade twine + twine check dist/* diff --git a/.github/workflows/documentation.yaml b/.github/workflows/documentation.yaml index 3796a60..0430fae 100644 --- a/.github/workflows/documentation.yaml +++ b/.github/workflows/documentation.yaml @@ -1,5 +1,5 @@ --- -name: "🗒️ Build documentation (Matrix)" +name: "📘 Documentation build/publish" # yamllint disable-line rule:truthy on: @@ -12,31 +12,30 @@ on: jobs: build_and_deploy: - # Don't run if pull request is NOT merged + # Only run when pull request is merged if: github.event.pull_request.merged == true name: "Rebuild documentation" runs-on: ubuntu-latest continue-on-error: true - strategy: - matrix: - python-version: ["3.11"] + permissions: # IMPORTANT: mandatory for documentation updates; used in final step id-token: write pull-requests: write contents: write repository-projects: write + steps: - name: "Checkout repository" uses: actions/checkout@v4 - - name: "Set up Python ${{ matrix.python-version }}" + - name: "Set up Python" uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} + python-version: "3.11" - name: "Setup PDM for build commands" - uses: pdm-project/setup-pdm@v3 + uses: pdm-project/setup-pdm@v4 - name: "Install dependencies" run: | @@ -48,6 +47,7 @@ jobs: - name: "Build documentation: (tox/sphinx)" run: | + pip install --upgrade tox tox -e docs - name: "Publish documentation" diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 7b99e21..a5bd95e 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -9,8 +9,10 @@ on: # workflow_dispatch: push: # Only invoked on release tag pushes + branches: + - '**' tags: - - v*.*.* + - 'v*.*.*' env: python-version: "3.10" @@ -18,8 +20,11 @@ env: ### BUILD ### jobs: + build: name: "🐍 Build packages" + # Only publish on tag pushes + if: startsWith(github.ref, 'refs/tags/') runs-on: ubuntu-latest permissions: # IMPORTANT: mandatory for Sigstore @@ -36,7 +41,7 @@ jobs: python-version: ${{ env.python-version }} - name: "Setup PDM for build commands" - uses: pdm-project/setup-pdm@v3 + uses: pdm-project/setup-pdm@v4 - name: "Update version from tags for production release" run: | @@ -50,7 +55,7 @@ jobs: ### SIGNING ### - name: "Sign packages with Sigstore" - uses: sigstore/gh-action-sigstore-python@v2.1.0 + uses: sigstore/gh-action-sigstore-python@v2.1.1 with: inputs: >- ./dist/*.tar.gz @@ -81,15 +86,16 @@ jobs: name: ${{ github.ref_name }} path: dist/ - - name: "📦 Publish release to GitHub" - uses: ModeSevenIndustrialSolutions/action-automatic-releases@latest + - name: "📦 Publish artefacts to GitHub" + # https://github.com/softprops/action-gh-release + uses: softprops/action-gh-release@v2 with: - # Valid inputs are: - # repo_token, automatic_release_tag, draft, prerelease, title, files - repo_token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ secrets.GITHUB_TOKEN }} prerelease: false - automatic_release_tag: ${{ github.ref_name }} - title: ${{ github.ref_name }} + tag_name: ${{ github.ref_name }} + name: "Test/Development Build \ + ${{ github.ref_name }}" + # body_path: ${{ github.workspace }}/CHANGELOG.rst files: | dist/*.tar.gz dist/*.whl @@ -158,7 +164,7 @@ jobs: rm dist/*.sigstore - name: "Setup PDM for build commands" - uses: pdm-project/setup-pdm@v3 + uses: pdm-project/setup-pdm@v4 - name: "Publish release to PyPI" uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/security.yaml b/.github/workflows/security.yaml index 9d34824..26251da 100644 --- a/.github/workflows/security.yaml +++ b/.github/workflows/security.yaml @@ -16,15 +16,21 @@ on: - "!update-devops-tooling" jobs: + + parse-project-metadata: + name: "Determine Python versions" + # yamllint disable-line rule:line-length + uses: os-climate/devops-reusable-workflows/.github/workflows/pyproject-toml-fetch-matrix.yaml@main + build: name: "Audit Python dependencies" + needs: [parse-project-metadata] runs-on: ubuntu-latest # Don't run when pull request is merged if: github.event.pull_request.merged == false strategy: fail-fast: false - matrix: - python-version: ["3.9", "3.10", "3.11"] + matrix: ${{ fromJson(needs.parse-project-metadata.outputs.matrix) }} steps: - name: "Checkout repository" @@ -36,7 +42,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: "Setup PDM for build commands" - uses: pdm-project/setup-pdm@v3 + uses: pdm-project/setup-pdm@v4 with: python-version: ${{ matrix.python-version }} @@ -47,6 +53,7 @@ jobs: pdm export -o requirements.txt python -m pip install -r requirements.txt python -m pip install . + pip install --upgrade setuptools pdm list --graph - name: "Run: pip-audit" diff --git a/.github/workflows/test-release.yaml b/.github/workflows/test-release.yaml index ad2a83b..9b54d9f 100644 --- a/.github/workflows/test-release.yaml +++ b/.github/workflows/test-release.yaml @@ -32,7 +32,7 @@ jobs: python-version: ${{ env.python-version }} - name: "Setup PDM for build commands" - uses: pdm-project/setup-pdm@v3 + uses: pdm-project/setup-pdm@v4 with: python-version: ${{ env.python-version }} @@ -62,7 +62,8 @@ jobs: ### SIGNING ### - name: "Sign packages with Sigstore" - uses: sigstore/gh-action-sigstore-python@v2.1.0 + uses: sigstore/gh-action-sigstore-python@v2.1.1 + with: inputs: >- ./dist/*.tar.gz @@ -103,16 +104,16 @@ jobs: echo "tarball=$(ls dist/*.tgz)" >> "$GITHUB_OUTPUT" echo "wheel=$(ls dist/*.whl)" >> "$GITHUB_OUTPUT" - - name: "📦 Publish packages to GitHub" - uses: ModeSevenIndustrialSolutions/action-automatic-releases@latest + - name: "📦 Publish artefacts to GitHub" + # https://github.com/softprops/action-gh-release + uses: softprops/action-gh-release@v2 with: - # Valid inputs are: - # repo_token, automatic_release_tag, draft, prerelease, title, files - repo_token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ secrets.GITHUB_TOKEN }} prerelease: true - automatic_release_tag: ${{ steps.setenv.outputs.vernum }} - title: "Development Build \ + tag_name: ${{ steps.setenv.outputs.vernum }} + name: "Test/Development Build \ ${{ steps.setenv.outputs.vernum }}" + # body_path: ${{ github.workspace }}/CHANGELOG.rst files: | dist/*.tar.gz dist/*.whl diff --git a/.github/workflows/testing.yaml b/.github/workflows/testing.yaml index bb80cec..0167d07 100644 --- a/.github/workflows/testing.yaml +++ b/.github/workflows/testing.yaml @@ -11,15 +11,21 @@ on: - "!update-devops-tooling" jobs: - build: + + parse-project-metadata: + name: "Determine Python versions" + # yamllint disable-line rule:line-length + uses: os-climate/devops-reusable-workflows/.github/workflows/pyproject-toml-fetch-matrix.yaml@main + + testing: name: "Run unit tests" + needs: [parse-project-metadata] runs-on: ubuntu-latest # Don't run when pull request is merged if: github.event.pull_request.merged == false strategy: fail-fast: false - matrix: - python-version: ["3.9", "3.10", "3.11"] + matrix: ${{ fromJson(needs.parse-project-metadata.outputs.matrix) }} steps: - name: "Checkout repository" @@ -31,7 +37,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: "Setup PDM for build commands" - uses: pdm-project/setup-pdm@v3 + uses: pdm-project/setup-pdm@v4 with: python-version: ${{ matrix.python-version }} @@ -40,6 +46,7 @@ jobs: python -m pip install --upgrade pip pdm export -o requirements.txt pip install -r requirements.txt + pip install --upgrade pytest pip install . - name: "Run unit tests: pytest"