diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index afe0b118..d57b44ed 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -9,6 +9,7 @@ ADD ./{{ item.name }} ./{{ item.name }} # S3 botocore needs to be patched to handle responses from minio during 0-byte uploads # Hacking botocore (https://github.com/boto/botocore/pull/1990) +# This MUST be the ONLY call to pip install in inside the container. RUN pip3 install --upgrade pip setuptools wheel && \ rm -rf /root/.cache/pip && \ pip3 install diff --git a/.ci/assets/release_requirements.txt b/.ci/assets/release_requirements.txt index c064e947..6635a872 100644 --- a/.ci/assets/release_requirements.txt +++ b/.ci/assets/release_requirements.txt @@ -1,3 +1,3 @@ -bump2version +bump-my-version gitpython towncrier diff --git a/.ci/scripts/calc_constraints.py b/.ci/scripts/calc_constraints.py index 2004bbdf..99050529 100755 --- a/.ci/scripts/calc_constraints.py +++ b/.ci/scripts/calc_constraints.py @@ -13,6 +13,11 @@ from packaging.version import Version import yaml +try: + import tomllib +except ImportError: + import tomli as tomllib + CORE_TEMPLATE_URL = "https://raw.githubusercontent.com/pulp/pulpcore/main/template_config.yml" @@ -59,11 +64,11 @@ def to_upper_bound(req): operator = "~=" version = Version(spec.version) if version.micro != 0: - max_version = f"{version.major}.{version.minor}.{version.micro-1}" + max_version = f"{version.major}.{version.minor}.{version.micro - 1}" elif version.minor != 0: - max_version = f"{version.major}.{version.minor-1}" + max_version = f"{version.major}.{version.minor - 1}" elif version.major != 0: - max_version = f"{version.major-1}.0" + max_version = f"{version.major - 1}.0" else: return f"# NO BETTER CONSTRAINT: {req}" return f"{requirement.name}{operator}{max_version}" @@ -100,18 +105,32 @@ def main(): parser.add_argument("filename", nargs="*") args = parser.parse_args() - with fileinput.input(files=args.filename) as req_file: - for line in req_file: - if line.strip().startswith("#"): - # Shortcut comment only lines - print(line.strip()) - else: - req, comment = split_comment(line) - if args.upper: - new_req = to_upper_bound(req) + modifier = to_upper_bound if args.upper else to_lower_bound + + req_files = [filename for filename in args.filename if not filename.endswith("pyproject.toml")] + pyp_files = [filename for filename in args.filename if filename.endswith("pyproject.toml")] + if req_files: + with fileinput.input(files=req_files) as req_file: + for line in req_file: + if line.strip().startswith("#"): + # Shortcut comment only lines + print(line.strip()) else: - new_req = to_lower_bound(req) - print(new_req + comment) + req, comment = split_comment(line) + new_req = modifier(req) + print(new_req + comment) + for filename in pyp_files: + with open(filename, "rb") as fp: + pyproject = tomllib.load(fp) + for req in pyproject["project"]["dependencies"]: + new_req = modifier(req) + print(new_req) + optional_dependencies = pyproject["project"].get("optional-dependencies") + if optional_dependencies: + for opt in optional_dependencies.values(): + for req in opt: + new_req = modifier(req) + print(new_req) if __name__ == "__main__": diff --git a/.ci/scripts/check_release.py b/.ci/scripts/check_release.py index 0872feb4..09effc4f 100755 --- a/.ci/scripts/check_release.py +++ b/.ci/scripts/check_release.py @@ -64,6 +64,7 @@ def main(): for branch in branches: if branch != DEFAULT_BRANCH: # Check if a Z release is needed + reasons = [] changes = repo.git.ls_tree("-r", "--name-only", f"origin/{branch}", "CHANGES/") z_changelog = False for change in changes.split("\n"): @@ -76,23 +77,34 @@ def main(): ) elif ext in Z_CHANGELOG_EXTS: z_changelog = True + if z_changelog: + reasons.append("Backports") last_tag = repo.git.describe("--tags", "--abbrev=0", f"origin/{branch}") req_txt_diff = repo.git.diff( f"{last_tag}", f"origin/{branch}", "--name-only", "--", "requirements.txt" ) - if z_changelog or req_txt_diff: + if req_txt_diff: + reasons.append("requirements.txt") + pyproject_diff = repo.git.diff( + f"{last_tag}", f"origin/{branch}", "--name-only", "--", "pyproject.toml" + ) + if pyproject_diff: + reasons.append( + "pyproject.toml changed (PLEASE check if dependencies are affected." + ) + + if reasons: curr_version = Version(last_tag) assert curr_version.base_version.startswith( branch ), "Current-version has to belong to the current branch!" next_version = Version(f"{branch}.{curr_version.micro + 1}") - reason = "CHANGES" if z_changelog else "requirements.txt" print( f"A Z-release is needed for {branch}, " f"Prev: {last_tag}, " f"Next: {next_version.base_version}, " - f"Reason: {reason}" + f"Reason: {','.join(reasons)}" ) releases.append(next_version) else: diff --git a/.ci/scripts/check_requirements.py b/.ci/scripts/check_requirements.py index 14506859..fc3f787d 100755 --- a/.ci/scripts/check_requirements.py +++ b/.ci/scripts/check_requirements.py @@ -5,11 +5,13 @@ # # For more info visit https://github.com/pulp/plugin_template +import tomllib import warnings -from pkg_resources import Requirement +from packaging.requirements import Requirement CHECK_MATRIX = [ + ("pyproject.toml", True, True, True), ("requirements.txt", True, True, True), ("dev_requirements.txt", False, True, False), ("ci_requirements.txt", False, True, True), @@ -20,17 +22,33 @@ ("clitest_requirements.txt", False, True, True), ] -errors = [] -for filename, check_upperbound, check_prereleases, check_r in CHECK_MATRIX: - try: +def iterate_file(filename): + if filename == "pyproject.toml": + with open(filename, "rb") as fd: + pyproject_toml = tomllib.load(fd) + if "project" in pyproject_toml: + for nr, line in enumerate(pyproject_toml["project"]["dependencies"]): + yield nr, line + else: with open(filename, "r") as fd: for nr, line in enumerate(fd.readlines()): line = line.strip() if not line or line.startswith("#"): continue + if "#" in line: + line = line.split("#", maxsplit=1)[0] + yield nr, line.strip() + + +def main(): + errors = [] + + for filename, check_upperbound, check_prereleases, check_r in CHECK_MATRIX: + try: + for nr, line in iterate_file(filename): try: - req = Requirement.parse(line) + req = Requirement(line) except ValueError: if line.startswith("git+"): # The single exception... @@ -49,18 +67,21 @@ and req.name != "pulp-file-client" ): errors.append(f"{filename}:{nr}: Prerelease versions found in {line}.") - ops = [op for op, ver in req.specs] - spec = str(req.specs) + ops = [spec.operator for spec in req.specifier] if "~=" in ops: warnings.warn(f"{filename}:{nr}: Please avoid using ~= on {req.name}!") elif "<" not in ops and "<=" not in ops and "==" not in ops: if check_upperbound: errors.append(f"{filename}:{nr}: Upper bound missing in {line}.") - except FileNotFoundError: - # skip this test for plugins that don't use this requirements.txt - pass + except FileNotFoundError: + # skip this test for plugins that don't use this requirements.txt + pass + + if errors: + print("Dependency issues found:") + print("\n".join(errors)) + exit(1) + -if errors: - print("Dependency issues found:") - print("\n".join(errors)) - exit(1) +if __name__ == "__main__": + main() diff --git a/.ci/scripts/collect_changes.py b/.ci/scripts/collect_changes.py index d39642c7..f3f03322 100755 --- a/.ci/scripts/collect_changes.py +++ b/.ci/scripts/collect_changes.py @@ -103,7 +103,7 @@ def main(): for change in main_changes: fp.write(change[1]) - repo.git.commit("-m", "Update Changelog", "-m" "[noissue]", CHANGELOG_FILE) + repo.git.commit("-m", "Update Changelog", CHANGELOG_FILE) if __name__ == "__main__": diff --git a/.ci/scripts/pr_labels.py b/.ci/scripts/pr_labels.py new file mode 100755 index 00000000..0c478a21 --- /dev/null +++ b/.ci/scripts/pr_labels.py @@ -0,0 +1,60 @@ +#!/bin/env python3 + +# This script is running with elevated privileges from the main branch against pull requests. + +import re +import sys +import tomllib +from pathlib import Path + +from git import Repo + + +def main(): + assert len(sys.argv) == 3 + + with open("pyproject.toml", "rb") as fp: + PYPROJECT_TOML = tomllib.load(fp) + BLOCKING_REGEX = re.compile(r"DRAFT|WIP|NO\s*MERGE|DO\s*NOT\s*MERGE|EXPERIMENT") + ISSUE_REGEX = re.compile(r"(?:fixes|closes)[\s:]+#(\d+)") + CHERRY_PICK_REGEX = re.compile(r"^\s*\(cherry picked from commit [0-9a-f]*\)\s*$") + try: + CHANGELOG_EXTS = { + f".{item['directory']}" for item in PYPROJECT_TOML["tool"]["towncrier"]["type"] + } + except KeyError: + CHANGELOG_EXTS = {".feature", ".bugfix", ".doc", ".removal", ".misc"} + + repo = Repo(".") + + base_commit = repo.commit(sys.argv[1]) + head_commit = repo.commit(sys.argv[2]) + + pr_commits = list(repo.iter_commits(f"{base_commit}..{head_commit}")) + + labels = { + "multi-commit": len(pr_commits) > 1, + "cherry-pick": False, + "no-issue": False, + "no-changelog": False, + "wip": False, + } + for commit in pr_commits: + labels["wip"] |= BLOCKING_REGEX.search(commit.summary) is not None + no_issue = ISSUE_REGEX.search(commit.message, re.IGNORECASE) is None + labels["no-issue"] |= no_issue + cherry_pick = CHERRY_PICK_REGEX.search(commit.message) is not None + labels["cherry-pick"] |= cherry_pick + changelog_snippets = [ + k + for k in commit.stats.files + if k.startswith("CHANGES/") and Path(k).suffix in CHANGELOG_EXTS + ] + labels["no-changelog"] |= not changelog_snippets + + print("ADD_LABELS=" + ",".join((k for k, v in labels.items() if v))) + print("REMOVE_LABELS=" + ",".join((k for k, v in labels.items() if not v))) + + +if __name__ == "__main__": + main() diff --git a/.ci/scripts/validate_commit_message.py b/.ci/scripts/validate_commit_message.py index fd6e1156..4b7b30bf 100755 --- a/.ci/scripts/validate_commit_message.py +++ b/.ci/scripts/validate_commit_message.py @@ -9,21 +9,16 @@ import sys from pathlib import Path import subprocess - - import os import warnings from github import Github - -NO_ISSUE = "[noissue]" CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc", ".deprecation"] +KEYWORDS = ["fixes", "closes"] + sha = sys.argv[1] message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8") - -KEYWORDS = ["fixes", "closes"] - g = Github(os.environ.get("GITHUB_TOKEN")) repo = g.get_repo("pulp/pulp_file") @@ -64,15 +59,5 @@ def __check_changelog(issue): for issue in pattern.findall(message): __check_status(issue) __check_changelog(issue) -else: - if NO_ISSUE in message: - print("Commit {sha} has no issues but is tagged {tag}.".format(sha=sha[0:7], tag=NO_ISSUE)) - elif "Merge" in message and "cherry picked from commit" in message: - pass - else: - sys.exit( - "Error: no attached issues found for {sha}. If this was intentional, add " - " '{tag}' to the commit message.".format(sha=sha[0:7], tag=NO_ISSUE) - ) print("Commit message for {sha} passed.".format(sha=sha[0:7])) diff --git a/.github/template_gitref b/.github/template_gitref index 9e5cdc02..d37a43c8 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-364-g6f9579c +2021.08.26-393-g0e700c1 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 40c2acbe..23f484a3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,11 +34,11 @@ jobs: - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install packaging twine wheel mkdocs jq + pip install build packaging twine wheel mkdocs jq echo ::endgroup:: - name: "Build package" run: | - python3 setup.py sdist bdist_wheel --python-tag py3 + python3 -m build twine check dist/* - name: "Install built packages" run: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a06f906d..efa974bc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,6 +43,9 @@ jobs: run: | .github/workflows/scripts/check_commit.sh + docs: + uses: "./.github/workflows/docs.yml" + lint: uses: "./.github/workflows/lint.yml" @@ -61,6 +64,7 @@ jobs: - "check-commits" - "lint" - "test" + - "docs" if: "always()" steps: - name: "Collect needed jobs results" diff --git a/.github/workflows/create-branch.yml b/.github/workflows/create-branch.yml index 39e163e5..71b39e4c 100644 --- a/.github/workflows/create-branch.yml +++ b/.github/workflows/create-branch.yml @@ -26,6 +26,12 @@ jobs: fetch-depth: 0 path: "pulp_file" + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + repository: "pulp/plugin_template" + path: "plugin_template" + - uses: "actions/setup-python@v5" with: python-version: "3.11" @@ -33,7 +39,7 @@ jobs: - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install bump2version jinja2 pyyaml packaging + pip install bump-my-version packaging -r plugin_template/requirements.txt echo ::endgroup:: - name: "Setting secrets" @@ -48,7 +54,7 @@ jobs: run: | # Just to be sure... git checkout main - NEW_BRANCH="$(bump2version --dry-run --list release | sed -Ene 's/^new_version=([[:digit:]]+\.[[:digit:]]+)\..*$/\1/p')" + NEW_BRANCH="$(bump-my-version show new_version --increment release | sed -Ene 's/^([[:digit:]]+\.[[:digit:]]+)\.[[:digit:]]+$/\1/p')" if [ -z "$NEW_BRANCH" ] then echo Could not determine the new branch name. @@ -64,20 +70,13 @@ jobs: - name: Bump version on main branch working-directory: pulp_file run: | - bump2version --no-commit minor + bump-my-version bump --no-commit minor - name: Remove entries from CHANGES directory working-directory: pulp_file run: | find CHANGES -type f -regex ".*\.\(bugfix\|doc\|feature\|misc\|deprecation\|removal\)" -exec git rm {} + - - name: Checkout plugin template - uses: actions/checkout@v4 - with: - repository: pulp/plugin_template - path: plugin_template - fetch-depth: 0 - - name: Update CI branches in template_config working-directory: plugin_template run: | @@ -94,10 +93,8 @@ jobs: branch: minor-version-bump base: main title: Bump minor version - body: '[noissue]' commit-message: | Bump minor version - [noissue] delete-branch: true - name: Push release branch diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000..887a6f75 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,54 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_file' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +--- +name: "Docs" +on: + workflow_call: + +jobs: + test: + if: "endsWith(github.base_ref, 'main')" + runs-on: "ubuntu-20.04" + defaults: + run: + working-directory: "pulp_file" + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + path: "pulp_file" + - uses: "actions/setup-python@v5" + with: + python-version: "3.11" + - name: "Setup cache key" + run: | + git ls-remote https://github.com/pulp/pulp-docs main | tee pulp-docs-main-sha + - uses: "actions/cache@v4" + with: + path: "~/.cache/pip" + key: ${{ runner.os }}-pip-${{ hashFiles('pulp-docs-main-sha') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: "Install python dependencies" + run: | + echo ::group::PYDEPS + pip install -r doc_requirements.txt + echo ::endgroup:: + - name: "Build changelog" + run: | + towncrier build --yes --version 4.0.0.ci + - name: "Build docs" + run: | + pulp-docs build + + no-test: + if: "!endsWith(github.base_ref, 'main')" + runs-on: "ubuntu-20.04" + steps: + - run: | + echo "Skip docs testing on non-main branches." diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index ebc72097..ef085081 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -16,7 +16,7 @@ defaults: jobs: lint: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" steps: - uses: "actions/checkout@v4" @@ -34,29 +34,38 @@ jobs: pip install -r lint_requirements.txt echo ::endgroup:: - - name: Lint workflow files + - name: "Lint workflow files" run: | yamllint -s -d '{extends: relaxed, rules: {line-length: disable}}' .github/workflows + - name: "Verify bump version config" + run: | + bump-my-version bump --dry-run release + bump-my-version show-bump + # run black separately from flake8 to get a diff - - name: Run black + - name: "Run black" run: | black --version black --check --diff . # Lint code. - - name: Run flake8 - run: flake8 + - name: "Run flake8" + run: | + flake8 - - name: Run extra lint checks - run: "[ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh" + - name: "Run extra lint checks" + run: | + [ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh - # check for any files unintentionally left out of MANIFEST.in - - name: Check manifest - run: check-manifest + - name: "Check for any files unintentionally left out of MANIFEST.in" + run: | + check-manifest - - name: Check for pulpcore imports outside of pulpcore.plugin - run: sh .ci/scripts/check_pulpcore_imports.sh + - name: "Check for pulpcore imports outside of pulpcore.plugin" + run: | + sh .ci/scripts/check_pulpcore_imports.sh - - name: Check for gettext problems - run: sh .ci/scripts/check_gettext.sh + - name: "Check for common gettext problems" + run: | + sh .ci/scripts/check_gettext.sh diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml index 846ab131..2a901608 100644 --- a/.github/workflows/pr_checks.yml +++ b/.github/workflows/pr_checks.yml @@ -6,57 +6,62 @@ # For more info visit https://github.com/pulp/plugin_template --- -name: File PR static checks +name: "File PR static checks" on: pull_request_target: - types: [opened, synchronize, reopened] + types: ["opened", "synchronize", "reopened"] # This workflow runs with elevated permissions. # Do not even think about running a single bit of code from the PR. # Static analysis should be fine however. concurrency: - group: ${{ github.event.pull_request.number }}-${{ github.workflow }} + group: "${{ github.event.pull_request.number }}-${{ github.workflow }}" cancel-in-progress: true jobs: - single_commit: - runs-on: ubuntu-latest - name: Label multiple commit PR + apply_labels: + runs-on: "ubuntu-latest" + name: "Label PR" permissions: - pull-requests: write + pull-requests: "write" steps: - uses: "actions/checkout@v4" with: fetch-depth: 0 - - name: Commit Count Check + - uses: "actions/setup-python@v5" + with: + python-version: "3.11" + - name: "Determine PR labels" run: | + pip install GitPython==3.1.42 git fetch origin ${{ github.event.pull_request.head.sha }} - echo "COMMIT_COUNT=$(git log --oneline --no-merges origin/${{ github.base_ref }}..${{ github.event.pull_request.head.sha }} | wc -l)" >> "$GITHUB_ENV" - - uses: actions/github-script@v7 + python .ci/scripts/pr_labels.py "origin/${{ github.base_ref }}" "${{ github.event.pull_request.head.sha }}" >> "$GITHUB_ENV" + - uses: "actions/github-script@v7" + name: "Apply PR Labels" with: script: | - const labelName = "multi-commit"; - const { COMMIT_COUNT } = process.env; + const { ADD_LABELS, REMOVE_LABELS } = process.env; - if (COMMIT_COUNT == 1) - { - try { - await github.rest.issues.removeLabel({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - name: labelName, - }); - } catch(err) { + if (REMOVE_LABELS.length) { + for await (const labelName of REMOVE_LABELS.split(",")) { + try { + await github.rest.issues.removeLabel({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + name: labelName, + }); + } catch(err) { + } } } - else - { + if (ADD_LABELS.length) { await github.rest.issues.addLabels({ issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo, - labels: [labelName], + labels: ADD_LABELS.split(","), }); } +... diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 37ab6171..7c5354a1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -35,7 +35,7 @@ jobs: - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install bump2version towncrier + pip install bump-my-version towncrier echo ::endgroup:: - name: "Configure Git with pulpbot name and email" diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index 7c0ac3f7..47a77872 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -45,11 +45,6 @@ if [ -f $PRE_BEFORE_INSTALL ]; then source $PRE_BEFORE_INSTALL fi -if [[ -n $(echo -e $COMMIT_MSG | grep -P "Required PR:.*") ]]; then - echo "The Required PR mechanism has been removed. Consider adding a scm requirement to requirements.txt." - exit 1 -fi - if [ "$GITHUB_EVENT_NAME" = "pull_request" ] || [ "${BRANCH_BUILD}" = "1" -a "${BRANCH}" != "main" ] then echo $COMMIT_MSG | sed -n -e 's/.*CI Base Image:\s*\([-_/[:alnum:]]*:[-_[:alnum:]]*\).*/ci_base: "\1"/p' >> .ci/ansible/vars/main.yaml @@ -57,7 +52,7 @@ fi for i in {1..3} do - ansible-galaxy collection install "amazon.aws:1.5.0" && s=0 && break || s=$? && sleep 3 + ansible-galaxy collection install "amazon.aws:8.1.0" && s=0 && break || s=$? && sleep 3 done if [[ $s -gt 0 ]] then diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index c8feec0d..4e2e0f2e 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -21,11 +21,8 @@ PLUGIN_SOURCE="./pulp_file/dist/pulp_file-${PLUGIN_VERSION}-py3-none-any.whl" export PULP_API_ROOT="/pulp/" PIP_REQUIREMENTS=("pulp-cli") -if [[ "$TEST" = "docs" || "$TEST" = "publish" ]] -then - PIP_REQUIREMENTS+=("-r" "doc_requirements.txt") -fi +# This must be the **only** call to "pip install" on the test runner. pip install ${PIP_REQUIREMENTS[*]} @@ -112,5 +109,5 @@ if [[ "$TEST" = "azure" ]]; then fi echo ::group::PIP_LIST -cmd_prefix bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" +cmd_prefix bash -c "pip3 list" echo ::endgroup:: diff --git a/.github/workflows/scripts/release.sh b/.github/workflows/scripts/release.sh index 9525f229..a08353cd 100755 --- a/.github/workflows/scripts/release.sh +++ b/.github/workflows/scripts/release.sh @@ -10,7 +10,8 @@ then exit 1 fi -NEW_VERSION="$(bump2version --dry-run --list release | sed -ne 's/^new_version=//p')" +# The tail is a necessary workaround to remove the warning from the output. +NEW_VERSION="$(bump-my-version show new_version --increment release | tail -n -1)" echo "Release ${NEW_VERSION}" if ! [[ "${NEW_VERSION}" == "${BRANCH}"* ]] @@ -20,7 +21,7 @@ then fi towncrier build --yes --version "${NEW_VERSION}" -bump2version release --commit --message "Release {new_version}" --tag --tag-name "{new_version}" --tag-message "Release {new_version}" --allow-dirty -bump2version patch --commit +bump-my-version bump release --commit --message "Release {new_version}" --tag --tag-name "{new_version}" --tag-message "Release {new_version}" --allow-dirty +bump-my-version bump patch --commit git push origin "${BRANCH}" "${NEW_VERSION}" diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index b03abc39..c42bce5d 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -18,7 +18,7 @@ source .github/workflows/scripts/utils.sh export POST_SCRIPT=$PWD/.github/workflows/scripts/post_script.sh export FUNC_TEST_SCRIPT=$PWD/.github/workflows/scripts/func_test_script.sh -# Needed for both starting the service and building the docs. +# Needed for starting the service # Gets set in .github/settings.yml, but doesn't seem to inherited by # this script. export DJANGO_SETTINGS_MODULE=pulpcore.app.settings @@ -26,14 +26,6 @@ export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py export PULP_URL="https://pulp" -if [[ "$TEST" = "docs" ]]; then - if [[ "$GITHUB_WORKFLOW" == "File CI" ]]; then - towncrier build --yes --version 4.0.0.ci - fi - pulp-docs build - exit -fi - REPORTED_STATUS="$(pulp status)" echo "machine pulp @@ -54,12 +46,14 @@ pushd ../pulp-openapi-generator # Workaround: Domains are not supported by the published bindings. # Sadly: Different pulpcore-versions aren't either... - # So we exclude the prebuilt ones only for domains disabled. - if [ "$(jq -r '.domain_enabled' <<<"${REPORTED_STATUS}")" = "true" ] || [ "$(jq -r '.online_workers[0].pulp_href|startswith("/pulp/api/v3/")' <<< "${REPORTED_STATUS}")" = "false" ] + # * In the 'pulp' scenario we use the published/prebuilt bindings, so we can test it. + # * In other scenarios we generate new bindings from server spec, so we have a more + # reliable client. + if [ "$TEST" = "pulp" ] then - BUILT_CLIENTS="" - else BUILT_CLIENTS=" " + else + BUILT_CLIENTS="" fi for ITEM in $(jq -r '.versions[] | tojson' <<<"${REPORTED_STATUS}") diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4fad66ff..5c43ec14 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -54,7 +54,7 @@ jobs: - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install towncrier twine wheel httpie docker netaddr boto3 ansible mkdocs jq jsonpatch + pip install towncrier twine wheel httpie docker netaddr boto3 'ansible~=10.3.0' mkdocs jq jsonpatch echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/pulp_file/.ci/assets/httpie/" >> $GITHUB_ENV echo ::endgroup:: @@ -117,5 +117,5 @@ jobs: docker logs pulp || true docker exec pulp ls -latr /etc/yum.repos.d/ || true docker exec pulp cat /etc/yum.repos.d/* || true - docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" + docker exec pulp bash -c "pip3 list" || true ... diff --git a/.github/workflows/update_ci.yml b/.github/workflows/update_ci.yml index 7aeb8151..62993e22 100644 --- a/.github/workflows/update_ci.yml +++ b/.github/workflows/update_ci.yml @@ -36,7 +36,7 @@ jobs: - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install gitpython requests packaging jinja2 pyyaml + pip install gitpython packaging -r plugin_template/requirements.txt echo ::endgroup:: - name: "Configure Git with pulpbot name and email" @@ -62,13 +62,8 @@ jobs: committer: "pulpbot " author: "pulpbot " title: "Update CI files for branch main" - body: "" branch: "update-ci/main" base: "main" - commit-message: | - Update CI files - - [noissue] delete-branch: true - uses: "actions/checkout@v4" with: @@ -89,13 +84,8 @@ jobs: committer: "pulpbot " author: "pulpbot " title: "Update CI files for branch 1.11" - body: "" branch: "update-ci/1.11" base: "1.11" - commit-message: | - Update CI files - - [noissue] delete-branch: true - uses: "actions/checkout@v4" with: @@ -116,13 +106,8 @@ jobs: committer: "pulpbot " author: "pulpbot " title: "Update CI files for branch 1.12" - body: "" branch: "update-ci/1.12" base: "1.12" - commit-message: | - Update CI files - - [noissue] delete-branch: true - uses: "actions/checkout@v4" with: @@ -143,11 +128,7 @@ jobs: committer: "pulpbot " author: "pulpbot " title: "Update CI files for branch 1.15" - body: "" branch: "update-ci/1.15" base: "1.15" - commit-message: | - Update CI files - - [noissue] delete-branch: true +... diff --git a/MANIFEST.in b/MANIFEST.in index 69515046..53db324a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -9,3 +9,4 @@ include functest_requirements.txt include test_requirements.txt include unittest_requirements.txt recursive-exclude pulp_file/tests/functional * +exclude releasing.md diff --git a/doc_requirements.txt b/doc_requirements.txt index 90ec2d51..a12e9156 100644 --- a/doc_requirements.txt +++ b/doc_requirements.txt @@ -1,9 +1,8 @@ # WARNING: DO NOT EDIT! # # This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --docs pulp_file' to update this file. +# './plugin-template --github pulp_file' to update this file. # # For more info visit https://github.com/pulp/plugin_template --r requirements.txt towncrier -# Extra requirements +pulp-docs @ git+https://github.com/pulp/pulp-docs@main diff --git a/functest_requirements.txt b/functest_requirements.txt index 7a9ed3d5..12679fc6 100644 --- a/functest_requirements.txt +++ b/functest_requirements.txt @@ -1,3 +1,4 @@ pytest<8 pytest-custom_exit_code pytest-xdist +pytest-timeout diff --git a/lint_requirements.txt b/lint_requirements.txt index 401ed981..a528dd0b 100644 --- a/lint_requirements.txt +++ b/lint_requirements.txt @@ -5,9 +5,10 @@ # # For more info visit https://github.com/pulp/plugin_template -# python packages handy for developers, but not required by pulp black==24.3.0 +bump-my-version check-manifest flake8 flake8-black +packaging yamllint diff --git a/pyproject.toml b/pyproject.toml index aef8ada7..a0383ba9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,6 @@ requires = [ build-backend = 'setuptools.build_meta' [tool.towncrier] -package = "pulp_file" filename = "CHANGES.md" directory = "CHANGES/" title_format = "## {version} ({project_date}) {{: #{version} }}" diff --git a/releasing.md b/releasing.md new file mode 100644 index 00000000..8f76a20e --- /dev/null +++ b/releasing.md @@ -0,0 +1,29 @@ +[//]: # "WARNING: DO NOT EDIT!" +[//]: # "" +[//]: # "This file was generated by plugin_template, and is managed by it. Please use" +[//]: # "'./plugin-template --github pulp_file' to update this file." +[//]: # "" +[//]: # "For more info visit https://github.com/pulp/plugin_template" +# Releasing (For Internal Use) + +This document outlines the steps to perform a release. + +### Determine if a Release is Required +- Make sure to have GitPython python package installed +- Run the release checker script: + ``` + python3 .ci/scripts/check_release.py + ``` + +### Release a New Y-version (e.g., 3.23.0) +- If a new minor version (Y) is needed, trigger a [Create New Release Branch](https://github.com/pulp/pulp_file/actions/workflows/create-branch.yml) job from the main branch via the GitHub Actions. +- Look for the "Bump minor version" pull request and merge it. +- Trigger a [Release Pipeline](https://github.com/pulp/pulp_file/actions/workflows/release.yml) job by specifying the new release branch (X.**Y**) via the GitHub Actions. + +### Release a New Z-version (Patch Release) (e.g., 3.23.1, 3.22.12) +- Trigger a [Release Pipeline](https://github.com/pulp/pulp_file/actions/workflows/release.yml) job by specifying the release branch (X.Y) via the GitHub Actions. + +## Final Steps +- Ensure the new version appears on PyPI (it should appear after [Publish Release](https://github.com/pulp/pulp_file/actions/workflows/publish.yml) workflow succeeds). +- Verify that the changelog has been updated by looking for the "Update Changelog" pull request (A new PR should be available on the next day). +- [optional] Post a brief announcement about the new release on the [Pulp Discourse](https://discourse.pulpproject.org/). diff --git a/template_config.yml b/template_config.yml index 2af39a08..d40e16a6 100644 --- a/template_config.yml +++ b/template_config.yml @@ -1,7 +1,7 @@ # This config represents the latest values used when running the plugin-template. Any settings that # were not present before running plugin-template have been added with their default values. -# generated with plugin_template@2021.08.26-364-g6f9579c +# generated with plugin_template@2021.08.26-393-g0e700c1 api_root: /pulp/ black: true @@ -12,7 +12,6 @@ check_stray_pulpcore_imports: true ci_base_image: ghcr.io/pulp/pulp-ci-centos9 ci_env: {} ci_trigger: '{pull_request: {branches: [''*'']}}' -ci_update_docs: false cli_package: pulp-cli cli_repo: https://github.com/pulp/pulp-cli.git core_import_allowed: [] @@ -21,16 +20,12 @@ deploy_client_to_rubygems: true deploy_to_pypi: true disabled_redis_runners: - s3 -doc_requirements_from_pulpcore: false docker_fixtures: false -docs_test: false flake8: true flake8_ignore: [] github_org: pulp -issue_tracker: github latest_release_branch: null lint_requirements: false -noissue_marker: '[noissue]' os_required_packages: [] parallel_test_workers: 8 plugin_app_label: file