From ac959a33bb83f6b8abacc2a3983920de0592492c Mon Sep 17 00:00:00 2001 From: pulpbot Date: Thu, 15 Feb 2024 18:17:45 +0000 Subject: [PATCH] Update CI files [noissue] --- .ci/ansible/Containerfile.j2 | 4 +- .ci/scripts/check_release.py | 13 +- .ci/scripts/collect_changes.py | 8 +- .github/template_gitref | 2 +- .github/workflows/create-branch.yml | 15 +- .github/workflows/publish.yml | 322 ++++++++++++++++ .github/workflows/release.yml | 361 +----------------- .github/workflows/scripts/install.sh | 4 +- .../scripts/install_python_client.sh | 15 +- .../workflows/scripts/install_ruby_client.sh | 16 +- .github/workflows/scripts/release.sh | 26 ++ .github/workflows/scripts/script.sh | 105 +++-- .../scripts/update_backport_labels.py | 8 +- functest_requirements.txt | 2 +- lint_requirements.txt | 2 +- template_config.yml | 16 +- unittest_requirements.txt | 1 + 17 files changed, 495 insertions(+), 425 deletions(-) create mode 100644 .github/workflows/publish.yml create mode 100755 .github/workflows/scripts/release.sh diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index c6c21fdb..338ff697 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -1,4 +1,4 @@ -FROM {{ ci_base | default("ghcr.io/pulp/pulp-ci-centos:" + pulp_container_tag) }} +FROM {{ ci_base | default(pulp_default_container) }} # Add source directories to container {% for item in plugins %} @@ -11,7 +11,7 @@ ADD ./{{ item.name }} ./{{ item.name }} RUN pip3 install {%- if s3_test | default(false) -%} -{{ " " }}git+https://github.com/fabricio-aguiar/botocore.git@fix-100-continue +{{ " " }}git+https://github.com/gerrod3/botocore.git@fix-100-continue {%- endif -%} {%- for item in plugins -%} {{ " " }}{{ item.source }} diff --git a/.ci/scripts/check_release.py b/.ci/scripts/check_release.py index 1ae7713d..0ff2e147 100755 --- a/.ci/scripts/check_release.py +++ b/.ci/scripts/check_release.py @@ -29,7 +29,7 @@ def main(): "--branches", default="supported", help="A comma separated list of branches to check for releases. Can also use keyword: " - "'supported'. Defaults to 'supported', see `ci_update_branches` in " + "'supported'. Defaults to 'supported', see `supported_release_branches` in " "`plugin_template.yml`.", ) opts = parser.parse_args() @@ -46,12 +46,15 @@ def main(): if branches == "supported": with open(f"{d}/template_config.yml", mode="r") as f: tc = yaml.safe_load(f) - branches = tc["ci_update_branches"] - branches.append(DEFAULT_BRANCH) + branches = set(tc["supported_release_branches"]) + latest_release_branch = tc["latest_release_branch"] + if latest_release_branch is not None: + branches.add(latest_release_branch) + branches.add(DEFAULT_BRANCH) else: - branches = branches.split(",") + branches = set(branches.split(",")) - if diff := set(branches) - set(available_branches): + if diff := branches - set(available_branches): print(f"Supplied branches contains non-existent branches! {diff}") exit(1) diff --git a/.ci/scripts/collect_changes.py b/.ci/scripts/collect_changes.py index 3c8c3f22..4a7aae7b 100755 --- a/.ci/scripts/collect_changes.py +++ b/.ci/scripts/collect_changes.py @@ -19,9 +19,11 @@ CHANGELOG_FILE = tc_settings.get("filename", "NEWS.rst") START_STRING = tc_settings.get( "start_string", - "\n" - if CHANGELOG_FILE.endswith(".md") - else ".. towncrier release notes start\n", + ( + "\n" + if CHANGELOG_FILE.endswith(".md") + else ".. towncrier release notes start\n" + ), ) TITLE_FORMAT = tc_settings.get("title_format", "{name} {version} ({project_date})") diff --git a/.github/template_gitref b/.github/template_gitref index bff7b993..2b533f2f 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-301-g83f0607 +2021.08.26-315-g8ecb63d diff --git a/.github/workflows/create-branch.yml b/.github/workflows/create-branch.yml index 2472f0aa..52460fc3 100644 --- a/.github/workflows/create-branch.yml +++ b/.github/workflows/create-branch.yml @@ -33,7 +33,7 @@ jobs: - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install bump2version jinja2 pyyaml + pip install bump2version jinja2 pyyaml packaging echo ::endgroup:: - name: "Setting secrets" @@ -71,6 +71,19 @@ jobs: run: | find CHANGES -type f -regex ".*\.\(bugfix\|doc\|feature\|misc\|deprecation\|removal\)" -exec git rm {} + + - name: Checkout plugin template + uses: actions/checkout@v3 + with: + repository: pulp/plugin_template + path: plugin_template + fetch-depth: 0 + + - name: Update CI branches in template_config + working-directory: plugin_template + run: | + python3 ./plugin-template pulp_file --github --latest-release-branch "${NEW_BRANCH}" + git add -A + - name: Make a PR with version bump and without CHANGES/* uses: peter-evans/create-pull-request@v4 with: diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 00000000..f74995ff --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,322 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_file' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +--- +name: "File Publish Release" +on: + push: + tags: + - "[0-9]+.[0-9]+.[0-9]+" + +defaults: + run: + working-directory: "pulp_file" + +jobs: + build: + uses: "./.github/workflows/build.yml" + + build-bindings-docs: + needs: + - "build" + runs-on: "ubuntu-latest" + # Install scripts expect TEST to be set, 'docs' is most appropriate even though we don't run tests + env: + TEST: "docs" + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + path: "pulp_file" + + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + repository: "pulp/pulp-openapi-generator" + path: "pulp-openapi-generator" + + - uses: "actions/setup-python@v4" + with: + python-version: "3.8" + + - uses: "actions/download-artifact@v3" + with: + name: "plugin_package" + path: "pulp_file/dist/" + - uses: ruby/setup-ruby@v1 + with: + ruby-version: "2.6" + + - name: "Install python dependencies" + run: | + echo ::group::PYDEPS + pip install towncrier twine wheel httpie docker netaddr boto3 ansible mkdocs + echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/pulp_file/.ci/assets/httpie/" >> $GITHUB_ENV + echo ::endgroup:: + + # Building the bindings and docs requires accessing the OpenAPI specs endpoint, so we need to + # setup the Pulp instance. + - name: "Before Install" + run: | + .github/workflows/scripts/before_install.sh + shell: "bash" + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + + - name: "Install" + run: | + .github/workflows/scripts/install.sh + shell: "bash" + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + + - name: "Install Python client" + run: | + .github/workflows/scripts/install_python_client.sh + shell: "bash" + - name: "Install Ruby client" + run: | + .github/workflows/scripts/install_ruby_client.sh + shell: "bash" + - name: "Upload python client packages" + uses: "actions/upload-artifact@v3" + with: + name: "python-client.tar" + path: "pulp_file/file-python-client.tar" + if-no-files-found: "error" + + - name: "Upload python client docs" + uses: "actions/upload-artifact@v3" + with: + name: "python-client-docs.tar" + path: "pulp_file/file-python-client-docs.tar" + if-no-files-found: "error" + - name: "Upload ruby client packages" + uses: "actions/upload-artifact@v3" + with: + name: "ruby-client.tar" + path: "pulp_file/file-ruby-client.tar" + if-no-files-found: "error" + - name: Build docs + run: | + export DJANGO_SETTINGS_MODULE=pulpcore.app.settings + export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py + make -C docs/ PULP_URL="https://pulp" diagrams html + tar -cvf docs/docs.tar docs/_build + + - name: "Upload built docs" + uses: actions/upload-artifact@v3 + with: + name: "docs.tar" + path: "pulp_file/docs/docs.tar" + + - name: "Logs" + if: always() + run: | + echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" + http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true + docker images || true + docker ps -a || true + docker logs pulp || true + docker exec pulp ls -latr /etc/yum.repos.d/ || true + docker exec pulp cat /etc/yum.repos.d/* || true + docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" + publish-package: + runs-on: "ubuntu-latest" + needs: + - "build-bindings-docs" + + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + path: "pulp_file" + + - uses: "actions/download-artifact@v3" + with: + name: "plugin_package" + path: "pulp_file/dist/" + + - uses: "actions/setup-python@v4" + with: + python-version: "3.8" + + - name: "Install python dependencies" + run: | + echo ::group::PYDEPS + pip install twine + echo ::endgroup:: + + - name: "Setting secrets" + run: | + python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" + env: + SECRETS_CONTEXT: "${{ toJson(secrets) }}" + + - name: "Deploy plugin to pypi" + run: | + .github/workflows/scripts/publish_plugin_pypi.sh ${{ github.ref_name }} + publish-python-bindings: + runs-on: "ubuntu-latest" + needs: + - "build-bindings-docs" + + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + path: "pulp_file" + + - name: "Download Python client" + uses: "actions/download-artifact@v3" + with: + name: "python-client.tar" + path: "pulp_file/" + + - name: "Untar python client packages" + run: | + tar -xvf file-python-client.tar + + - uses: "actions/setup-python@v4" + with: + python-version: "3.8" + + - name: "Install python dependencies" + run: | + echo ::group::PYDEPS + pip install twine + echo ::endgroup:: + + - name: "Setting secrets" + run: | + python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" + env: + SECRETS_CONTEXT: "${{ toJson(secrets) }}" + + - name: "Publish client to pypi" + run: | + bash .github/workflows/scripts/publish_client_pypi.sh ${{ github.ref_name }} + publish-ruby-bindings: + runs-on: "ubuntu-latest" + needs: + - "build-bindings-docs" + + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + path: "pulp_file" + + - name: "Download Ruby client" + uses: "actions/download-artifact@v3" + with: + name: "ruby-client.tar" + path: "pulp_file/" + + - name: "Untar Ruby client packages" + run: | + tar -xvf file-ruby-client.tar + + - uses: ruby/setup-ruby@v1 + with: + ruby-version: "2.6" + + - name: "Setting secrets" + run: | + python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" + env: + SECRETS_CONTEXT: "${{ toJson(secrets) }}" + + - name: "Publish client to rubygems" + run: | + bash .github/workflows/scripts/publish_client_gem.sh ${{ github.ref_name }} + publish-docs: + runs-on: "ubuntu-latest" + needs: + - "build-bindings-docs" + + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + path: "pulp_file" + + - uses: "actions/setup-python@v4" + with: + python-version: "3.8" + + - name: "Install python dependencies" + run: | + echo ::group::PYDEPS + pip install 'packaging~=21.3' requests + echo ::endgroup:: + + - name: "Setting secrets" + run: | + python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" + env: + SECRETS_CONTEXT: "${{ toJson(secrets) }}" + + - name: "Download built docs" + uses: "actions/download-artifact@v3" + with: + name: "docs.tar" + path: "pulp_file/" + + - name: "Download Python client docs" + uses: "actions/download-artifact@v3" + with: + name: "python-client-docs.tar" + path: "pulp_file/" + + - name: "Publish docs to pulpproject.org" + run: | + tar -xvf docs.tar + .github/workflows/scripts/publish_docs.sh tag ${{ github.ref_name }} + + create-gh-release: + runs-on: "ubuntu-latest" + needs: + - "build-bindings-docs" + - "publish-package" + - "publish-python-bindings" + - "publish-ruby-bindings" + - "publish-docs" + + steps: + - name: "Create release on GitHub" + uses: "actions/github-script@v7" + env: + TAG_NAME: "${{ github.ref_name }}" + with: + script: | + const { TAG_NAME } = process.env; + + await github.rest.repos.createRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + tag_name: TAG_NAME, + make_latest: "legacy", + }); diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index caca339c..74548017 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,9 +9,6 @@ name: File Release Pipeline on: workflow_dispatch: - inputs: - release: - description: "Release tag (e.g. 3.2.1)" defaults: run: @@ -29,6 +26,7 @@ jobs: with: fetch-depth: 0 path: "pulp_file" + token: ${{ secrets.RELEASE_TOKEN }} - uses: "actions/setup-python@v4" with: @@ -37,7 +35,7 @@ jobs: - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install 'packaging~=21.3' bump2version gitpython towncrier wheel requests + pip install bump2version towncrier echo ::endgroup:: - name: "Configure Git with pulpbot name and email" @@ -51,363 +49,12 @@ jobs: env: SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - name: "Create the release commit, tag it, create a post-release commit, and build plugin package" + - name: "Tag the release" run: | - python .github/workflows/scripts/release.py ${{ github.event.inputs.release }} - - - name: "Upload Package whl" - uses: "actions/upload-artifact@v3" - with: - name: "plugin_package" - path: "pulp_file/dist/" - if-no-files-found: "error" - - - name: "Tar files" - run: | - tar -cvf pulp_file.tar . - - - name: "Upload Artifact" - uses: "actions/upload-artifact@v3" - with: - name: "pulp_file.tar" - path: "pulp_file/pulp_file.tar" - - - build-bindings-docs: - needs: - - "build-artifacts" - runs-on: "ubuntu-latest" - # Install scripts expect TEST to be set, 'docs' is most appropriate even though we don't run tests - env: - TEST: "docs" - - steps: - - uses: "actions/download-artifact@v3" - with: - name: "pulp_file.tar" - path: "pulp_file/" - - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - repository: "pulp/pulp-openapi-generator" - path: "pulp-openapi-generator" - - - uses: "actions/setup-python@v4" - with: - python-version: "3.8" - - uses: ruby/setup-ruby@v1 - with: - ruby-version: "2.6" - - - name: "Untar repository" - run: | - tar -xf pulp_file.tar - - - name: "Install python dependencies" - run: | - echo ::group::PYDEPS - pip install towncrier twine wheel httpie docker netaddr boto3 ansible mkdocs - echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/pulp_file/.ci/assets/httpie/" >> $GITHUB_ENV - echo ::endgroup:: - - # Building the bindings and docs requires accessing the OpenAPI specs endpoint, so we need to - # setup the Pulp instance. - - name: "Before Install" - run: | - .github/workflows/scripts/before_install.sh + .github/workflows/scripts/release.sh shell: "bash" env: PY_COLORS: "1" ANSIBLE_FORCE_COLOR: "1" GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" - - - name: "Install" - run: | - .github/workflows/scripts/install.sh - shell: "bash" - env: - PY_COLORS: "1" - ANSIBLE_FORCE_COLOR: "1" - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" - - - name: "Install Python client" - run: | - .github/workflows/scripts/install_python_client.sh - shell: "bash" - - name: "Install Ruby client" - run: | - .github/workflows/scripts/install_ruby_client.sh - shell: "bash" - - name: "Upload python client packages" - uses: "actions/upload-artifact@v3" - with: - name: "python-client.tar" - path: "pulp_file/file-python-client.tar" - if-no-files-found: "error" - - - name: "Upload python client docs" - uses: "actions/upload-artifact@v3" - with: - name: "python-client-docs.tar" - path: "pulp_file/file-python-client-docs.tar" - if-no-files-found: "error" - - name: "Upload ruby client packages" - uses: "actions/upload-artifact@v3" - with: - name: "ruby-client.tar" - path: "pulp_file/file-ruby-client.tar" - if-no-files-found: "error" - - name: Build docs - run: | - export DJANGO_SETTINGS_MODULE=pulpcore.app.settings - export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py - make -C docs/ PULP_URL="https://pulp" diagrams html - tar -cvf docs/docs.tar docs/_build - - - name: "Upload built docs" - uses: actions/upload-artifact@v3 - with: - name: "docs.tar" - path: "pulp_file/docs/docs.tar" - - - name: "Logs" - if: always() - run: | - echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true - docker images || true - docker ps -a || true - docker logs pulp || true - docker exec pulp ls -latr /etc/yum.repos.d/ || true - docker exec pulp cat /etc/yum.repos.d/* || true - docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" - - - publish-tag: - runs-on: "ubuntu-latest" - needs: - - "build-bindings-docs" - - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - - steps: - - uses: "actions/download-artifact@v3" - with: - name: "pulp_file.tar" - path: "pulp_file/" - - - name: "Untar repository" - run: | - tar -xf pulp_file.tar - - - name: "Configure Git with pulpbot name and email" - run: | - git config --global user.name 'pulpbot' - git config --global user.email 'pulp-infra@redhat.com' - - - name: "Setting secrets" - run: | - python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - - name: "Push branch and tag to GitHub" - run: | - bash .github/workflows/scripts/push_branch_and_tag_to_github.sh ${{ github.event.inputs.release }} - publish-package: - runs-on: "ubuntu-latest" - needs: - - "build-bindings-docs" - - "publish-tag" - - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - - steps: - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - path: "pulp_file" - - - uses: "actions/download-artifact@v3" - with: - name: "plugin_package" - path: "pulp_file/dist/" - - - uses: "actions/setup-python@v4" - with: - python-version: "3.8" - - - name: "Install python dependencies" - run: | - echo ::group::PYDEPS - pip install twine - echo ::endgroup:: - - - name: "Setting secrets" - run: | - python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - - name: "Deploy plugin to pypi" - run: | - .github/workflows/scripts/publish_plugin_pypi.sh ${{ github.event.inputs.release }} - publish-python-bindings: - runs-on: "ubuntu-latest" - needs: - - "build-bindings-docs" - - "publish-tag" - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - - steps: - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - path: "pulp_file" - - - name: "Download Python client" - uses: "actions/download-artifact@v3" - with: - name: "python-client.tar" - path: "pulp_file/" - - - name: "Untar python client packages" - run: | - tar -xvf file-python-client.tar - - - uses: "actions/setup-python@v4" - with: - python-version: "3.8" - - - name: "Install python dependencies" - run: | - echo ::group::PYDEPS - pip install twine - echo ::endgroup:: - - - name: "Setting secrets" - run: | - python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - - name: "Publish client to pypi" - run: | - bash .github/workflows/scripts/publish_client_pypi.sh ${{ github.event.inputs.release }} - publish-ruby-bindings: - runs-on: "ubuntu-latest" - needs: - - "build-bindings-docs" - - "publish-tag" - - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - - steps: - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - path: "pulp_file" - - - name: "Download Ruby client" - uses: "actions/download-artifact@v3" - with: - name: "ruby-client.tar" - path: "pulp_file/" - - - name: "Untar Ruby client packages" - run: | - tar -xvf file-ruby-client.tar - - - uses: ruby/setup-ruby@v1 - with: - ruby-version: "2.6" - - - name: "Setting secrets" - run: | - python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - - name: "Publish client to rubygems" - run: | - bash .github/workflows/scripts/publish_client_gem.sh ${{ github.event.inputs.release }} - publish-docs: - runs-on: "ubuntu-latest" - needs: - - "build-bindings-docs" - - "publish-tag" - - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - - steps: - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - path: "pulp_file" - - - uses: "actions/setup-python@v4" - with: - python-version: "3.8" - - - name: "Install python dependencies" - run: | - echo ::group::PYDEPS - pip install 'packaging~=21.3' requests - echo ::endgroup:: - - - name: "Setting secrets" - run: | - python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - - name: "Download built docs" - uses: "actions/download-artifact@v3" - with: - name: "docs.tar" - path: "pulp_file/" - - - name: "Download Python client docs" - uses: "actions/download-artifact@v3" - with: - name: "python-client-docs.tar" - path: "pulp_file/" - - - name: "Publish docs to pulpproject.org" - run: | - tar -xvf docs.tar - .github/workflows/scripts/publish_docs.sh tag ${{ github.event.inputs.release }} - - create-gh-release: - runs-on: "ubuntu-latest" - needs: - - "build-bindings-docs" - - "publish-tag" - - "publish-package" - - "publish-python-bindings" - - "publish-ruby-bindings" - - "publish-docs" - - steps: - - name: "Create release on GitHub" - uses: "actions/github-script@v7" - env: - TAG_NAME: "${{ inputs.release }}" - with: - script: | - const { TAG_NAME } = process.env; - - await github.rest.repos.createRelease({ - owner: context.repo.owner, - repo: context.repo.repo, - tag_name: TAG_NAME, - make_latest: "legacy", - }); diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 011667ad..8eb24891 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -80,9 +80,7 @@ cat >> vars/main.yaml << VARSYAML pulp_env: {} pulp_settings: {"allowed_content_checksums": ["sha1", "sha224", "sha256", "sha384", "sha512"], "allowed_export_paths": ["/tmp"], "allowed_import_paths": ["/tmp"], "orphan_protection_time": 0} pulp_scheme: https - -pulp_container_tag: "latest" - +pulp_default_container: ghcr.io/pulp/pulp-ci-centos:latest VARSYAML SCENARIOS=("pulp" "performance" "azure" "gcp" "s3" "generate-bindings" "lowerbounds") diff --git a/.github/workflows/scripts/install_python_client.sh b/.github/workflows/scripts/install_python_client.sh index ab8bf6da..c6603f91 100755 --- a/.github/workflows/scripts/install_python_client.sh +++ b/.github/workflows/scripts/install_python_client.sh @@ -14,7 +14,10 @@ cd "$(dirname "$(realpath -e "$0")")"/../../.. source .github/workflows/scripts/utils.sh -export PULP_URL="${PULP_URL:-https://pulp}" +PULP_URL="${PULP_URL:-https://pulp}" +export PULP_URL +PULP_API_ROOT="${PULP_API_ROOT:-/pulp/}" +export PULP_API_ROOT REPORTED_STATUS="$(pulp status)" REPORTED_VERSION="$(echo "$REPORTED_STATUS" | jq --arg plugin "file" -r '.versions[] | select(.component == $plugin) | .version')" @@ -22,7 +25,15 @@ VERSION="$(echo "$REPORTED_VERSION" | python -c 'from packaging.version import V pushd ../pulp-openapi-generator rm -rf pulp_file-client -./generate.sh pulp_file python "$VERSION" + +if pulp debug has-plugin --name "core" --specifier ">=3.44.0.dev" +then + curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=file" + USE_LOCAL_API_JSON=1 ./generate.sh pulp_file python "$VERSION" +else + ./generate.sh pulp_file python "$VERSION" +fi + pushd pulp_file-client python setup.py sdist bdist_wheel --python-tag py3 diff --git a/.github/workflows/scripts/install_ruby_client.sh b/.github/workflows/scripts/install_ruby_client.sh index a2f980b7..73f38b06 100755 --- a/.github/workflows/scripts/install_ruby_client.sh +++ b/.github/workflows/scripts/install_ruby_client.sh @@ -14,8 +14,10 @@ cd "$(dirname "$(realpath -e "$0")")"/../../.. source .github/workflows/scripts/utils.sh -export PULP_URL="${PULP_URL:-https://pulp}" - +PULP_URL="${PULP_URL:-https://pulp}" +export PULP_URL +PULP_API_ROOT="${PULP_API_ROOT:-/pulp/}" +export PULP_API_ROOT REPORTED_STATUS="$(pulp status)" REPORTED_VERSION="$(echo "$REPORTED_STATUS" | jq --arg plugin "file" -r '.versions[] | select(.component == $plugin) | .version')" @@ -23,7 +25,15 @@ VERSION="$(echo "$REPORTED_VERSION" | python -c 'from packaging.version import V pushd ../pulp-openapi-generator rm -rf pulp_file-client -./generate.sh pulp_file ruby "$VERSION" + +if pulp debug has-plugin --name "core" --specifier ">=3.44.0.dev" +then + curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=file" + USE_LOCAL_API_JSON=1 ./generate.sh pulp_file ruby "$VERSION" +else + ./generate.sh pulp_file ruby "$VERSION" +fi + pushd pulp_file-client gem build pulp_file_client gem install --both "./pulp_file_client-$VERSION.gem" diff --git a/.github/workflows/scripts/release.sh b/.github/workflows/scripts/release.sh new file mode 100755 index 00000000..9525f229 --- /dev/null +++ b/.github/workflows/scripts/release.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -eu -o pipefail + +BRANCH=$(git branch --show-current) + +if ! [[ "${BRANCH}" =~ ^[0-9]+\.[0-9]+$ ]] +then + echo ERROR: This is not a release branch! + exit 1 +fi + +NEW_VERSION="$(bump2version --dry-run --list release | sed -ne 's/^new_version=//p')" +echo "Release ${NEW_VERSION}" + +if ! [[ "${NEW_VERSION}" == "${BRANCH}"* ]] +then + echo ERROR: Version does not match release branch + exit 1 +fi + +towncrier build --yes --version "${NEW_VERSION}" +bump2version release --commit --message "Release {new_version}" --tag --tag-name "{new_version}" --tag-message "Release {new_version}" --allow-dirty +bump2version patch --commit + +git push origin "${BRANCH}" "${NEW_VERSION}" diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index 15d1687a..4ea9c680 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -51,32 +51,68 @@ password password # Some commands like ansible-galaxy specifically require 600 cmd_prefix bash -c "chmod 600 ~pulp/.netrc" -# Infer the client name from the package name by replacing "-" with "_". -# Use the component to infer the package name on older versions of pulpcore. - -if [ "$(echo "$REPORTED_STATUS" | jq -r '.domain_enabled')" = "true" ] +# Generate and install binding +pushd ../pulp-openapi-generator +if pulp debug has-plugin --name "core" --specifier ">=3.44.0.dev" then - # Workaround: Domains are not supported by the published bindings. - # Generate new bindings for all packages. - pushd ../pulp-openapi-generator - for item in $(echo "$REPORTED_STATUS" | jq -r '.versions[]|(.package // ("pulp_" + .component)|sub("pulp_core"; "pulpcore"))|sub("-"; "_")') - do - ./generate.sh "${item}" python - cmd_prefix pip3 install "/root/pulp-openapi-generator/${item}-client" - sudo rm -rf "./${item}-client" - done - popd + # Use app_label to generate api.json and package to produce the proper package name. + + if [ "$(jq -r '.domain_enabled' <<<"$REPORTED_STATUS")" = "true" ] + then + # Workaround: Domains are not supported by the published bindings. + # Generate new bindings for all packages. + for item in $(jq -r '.versions[] | tojson' <<<"$REPORTED_STATUS") + do + echo $item + COMPONENT="$(jq -r '.component' <<<"$item")" + VERSION="$(jq -r '.version' <<<"$item")" + MODULE="$(jq -r '.module' <<<"$item")" + PACKAGE="${MODULE%%.*}" + curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=$COMPONENT" + USE_LOCAL_API_JSON=1 ./generate.sh "${PACKAGE}" python "${VERSION}" + cmd_prefix pip3 install "/root/pulp-openapi-generator/${PACKAGE}-client" + sudo rm -rf "./${PACKAGE}-client" + done + else + # Sadly: Different pulpcore-versions aren't either... + for item in $(jq -r '.versions[]| select(.component!="file")| tojson' <<<"$REPORTED_STATUS") + do + echo $item + COMPONENT="$(jq -r '.component' <<<"$item")" + VERSION="$(jq -r '.version' <<<"$item")" + MODULE="$(jq -r '.module' <<<"$item")" + PACKAGE="${MODULE%%.*}" + curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=$COMPONENT" + USE_LOCAL_API_JSON=1 ./generate.sh "${PACKAGE}" python "${VERSION}" + cmd_prefix pip3 install "/root/pulp-openapi-generator/${PACKAGE}-client" + sudo rm -rf "./${PACKAGE}-client" + done + fi else - # Sadly: Different pulpcore-versions aren't either... - pushd ../pulp-openapi-generator - for item in $(echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.component!="file")|(.package // ("pulp_" + .component)|sub("pulp_core"; "pulpcore"))|sub("-"; "_")') - do - ./generate.sh "${item}" python - cmd_prefix pip3 install "/root/pulp-openapi-generator/${item}-client" - sudo rm -rf "./${item}-client" - done - popd + # Infer the client name from the package name by replacing "-" with "_". + # Use the component to infer the package name on older versions of pulpcore. + + if [ "$(echo "$REPORTED_STATUS" | jq -r '.domain_enabled')" = "true" ] + then + # Workaround: Domains are not supported by the published bindings. + # Generate new bindings for all packages. + for item in $(echo "$REPORTED_STATUS" | jq -r '.versions[]|(.package // ("pulp_" + .component)|sub("pulp_core"; "pulpcore"))|sub("-"; "_")') + do + ./generate.sh "${item}" python + cmd_prefix pip3 install "/root/pulp-openapi-generator/${item}-client" + sudo rm -rf "./${item}-client" + done + else + # Sadly: Different pulpcore-versions aren't either... + for item in $(echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.component!="file")|(.package // ("pulp_" + .component)|sub("pulp_core"; "pulpcore"))|sub("-"; "_")') + do + ./generate.sh "${item}" python + cmd_prefix pip3 install "/root/pulp-openapi-generator/${item}-client" + sudo rm -rf "./${item}-client" + done + fi fi +popd # At this point, this is a safeguard only, so let's not make too much fuzz about the old status format. echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.package)|(.package|sub("_"; "-")) + "-client==" + .version' > bindings_requirements.txt @@ -93,14 +129,14 @@ echo "Checking for uncommitted migrations..." cmd_user_prefix bash -c "django-admin makemigrations file --check --dry-run" # Run unit tests. -cmd_user_prefix bash -c "PULP_DATABASES__default__USER=postgres pytest -v -r sx --color=yes -p no:pulpcore --pyargs pulp_file.tests.unit" +cmd_user_prefix bash -c "PULP_DATABASES__default__USER=postgres pytest -v -r sx --color=yes --suppress-no-test-exit-code -p no:pulpcore --pyargs pulp_file.tests.unit" # Run functional tests if [[ "$TEST" == "performance" ]]; then if [[ -z ${PERFORMANCE_TEST+x} ]]; then - cmd_user_prefix bash -c "pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_file.tests.performance" + cmd_user_prefix bash -c "pytest -vv -r sx --color=yes --suppress-no-test-exit-code --capture=no --durations=0 --pyargs pulp_file.tests.performance" else - cmd_user_prefix bash -c "pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_file.tests.performance.test_${PERFORMANCE_TEST}" + cmd_user_prefix bash -c "pytest -vv -r sx --color=yes --suppress-no-test-exit-code --capture=no --durations=0 --pyargs pulp_file.tests.performance.test_${PERFORMANCE_TEST}" fi exit fi @@ -108,15 +144,14 @@ fi if [ -f "$FUNC_TEST_SCRIPT" ]; then source "$FUNC_TEST_SCRIPT" else - if [[ "$GITHUB_WORKFLOW" == "File Nightly CI/CD" ]] - then - cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_file.tests.functional -m parallel -n 8 --nightly" - cmd_user_prefix bash -c "pytest -v -r sx --color=yes --pyargs pulp_file.tests.functional -m 'not parallel' --nightly" - - else - cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_file.tests.functional -m parallel -n 8" - cmd_user_prefix bash -c "pytest -v -r sx --color=yes --pyargs pulp_file.tests.functional -m 'not parallel'" - fi + if [[ "$GITHUB_WORKFLOW" =~ "Nightly" ]] + then + cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_file.tests.functional -m parallel -n 8 --nightly" + cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_file.tests.functional -m 'not parallel' --nightly" + else + cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_file.tests.functional -m parallel -n 8" + cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_file.tests.functional -m 'not parallel'" + fi fi export PULP_FIXTURES_URL="http://pulp-fixtures:8080" pushd ../pulp-cli diff --git a/.github/workflows/scripts/update_backport_labels.py b/.github/workflows/scripts/update_backport_labels.py index 854ff077..8cccd498 100755 --- a/.github/workflows/scripts/update_backport_labels.py +++ b/.github/workflows/scripts/update_backport_labels.py @@ -32,10 +32,14 @@ def random_color(): assert response.status_code == 200 old_labels = set([x["name"] for x in response.json() if x["name"].startswith("backport-")]) -# get ci_update_branches from template_config.yml +# get list of branches from template_config.yml with open("./template_config.yml", "r") as f: plugin_template = yaml.safe_load(f) -new_labels = set(["backport-" + x for x in plugin_template["ci_update_branches"]]) +branches = set(plugin_template["supported_release_branches"]) +latest_release_branch = plugin_template["latest_release_branch"] +if latest_release_branch is not None: + branches.add(latest_release_branch) +new_labels = {"backport-" + x for x in branches} # delete old labels that are not in new labels for label in old_labels.difference(new_labels): diff --git a/functest_requirements.txt b/functest_requirements.txt index b28163da..21029a3b 100644 --- a/functest_requirements.txt +++ b/functest_requirements.txt @@ -5,4 +5,4 @@ pulp-smash @ git+https://github.com/pulp/pulp-smash.git pulpcore-client pulp-file-client pulp-certguard-client -pytest +pytest<8 diff --git a/lint_requirements.txt b/lint_requirements.txt index c710c3c3..cb7bb6ed 100644 --- a/lint_requirements.txt +++ b/lint_requirements.txt @@ -6,7 +6,7 @@ # For more info visit https://github.com/pulp/plugin_template # python packages handy for developers, but not required by pulp -black +black==23.12.1 check-manifest flake8 flake8-black diff --git a/template_config.yml b/template_config.yml index dc7052ae..265ebe28 100644 --- a/template_config.yml +++ b/template_config.yml @@ -1,25 +1,18 @@ # This config represents the latest values used when running the plugin-template. Any settings that # were not present before running plugin-template have been added with their default values. -# generated with plugin_template@2021.08.26-301-g83f0607 +# generated with plugin_template@2021.08.26-315-g8ecb63d -additional_repos: -- branch: main - name: pulp-certguard api_root: /pulp/ black: true check_commit_message: true check_gettext: true check_manifest: true check_stray_pulpcore_imports: true +ci_base_image: ghcr.io/pulp/pulp-ci-centos ci_env: {} ci_trigger: '{pull_request: {branches: [''*'']}}' -ci_update_branches: -- '1.10' -- '1.11' -- '1.12' ci_update_docs: false -ci_update_release_behavior: null cli_package: pulp-cli cli_repo: https://github.com/pulp/pulp-cli.git core_import_allowed: [] @@ -37,6 +30,7 @@ flake8_ignore: [] github_org: pulp issue_tracker: github kanban: true +latest_release_branch: null lint_requirements: true noissue_marker: '[noissue]' parallel_test_workers: 8 @@ -81,6 +75,10 @@ stalebot: true stalebot_days_until_close: 30 stalebot_days_until_stale: 90 stalebot_limit_to_pulls: true +supported_release_branches: +- '1.10' +- '1.11' +- '1.12' sync_ci: true test_azure: true test_cli: true diff --git a/unittest_requirements.txt b/unittest_requirements.txt index c4380ab7..421e7ba7 100644 --- a/unittest_requirements.txt +++ b/unittest_requirements.txt @@ -1,2 +1,3 @@ mock pytest-django +pytest<8