Update CI files

.ci/ansible/Containerfile.j2

@@ -9,6 +9,7 @@ ADD ./{{ item.name }} ./{{ item.name }}
 # S3 botocore needs to be patched to handle responses from minio during 0-byte uploads
 # Hacking botocore (https://github.com/boto/botocore/pull/1990)
 
+# This MUST be the ONLY call to pip install in inside the container.
 RUN pip3 install --upgrade pip setuptools wheel && \
   rm -rf /root/.cache/pip && \
   pip3 install

.ci/assets/release_requirements.txt

@@ -1,3 +1,3 @@
-bump2version
+bump-my-version
 gitpython
 towncrier

.ci/scripts/calc_constraints.py

@@ -13,6 +13,11 @@
 from packaging.version import Version
 import yaml
 
+try:
+    import tomllib
+except ImportError:
+    import tomli as tomllib
+
 
 CORE_TEMPLATE_URL = "https://raw.githubusercontent.com/pulp/pulpcore/main/template_config.yml"
 
@@ -59,11 +64,11 @@ def to_upper_bound(req):
                 operator = "~="
                 version = Version(spec.version)
                 if version.micro != 0:
-                    max_version = f"{version.major}.{version.minor}.{version.micro-1}"
+                    max_version = f"{version.major}.{version.minor}.{version.micro - 1}"
                 elif version.minor != 0:
-                    max_version = f"{version.major}.{version.minor-1}"
+                    max_version = f"{version.major}.{version.minor - 1}"
                 elif version.major != 0:
-                    max_version = f"{version.major-1}.0"
+                    max_version = f"{version.major - 1}.0"
                 else:
                     return f"# NO BETTER CONSTRAINT: {req}"
                 return f"{requirement.name}{operator}{max_version}"
@@ -100,18 +105,32 @@ def main():
     parser.add_argument("filename", nargs="*")
     args = parser.parse_args()
 
-    with fileinput.input(files=args.filename) as req_file:
-        for line in req_file:
-            if line.strip().startswith("#"):
-                # Shortcut comment only lines
-                print(line.strip())
-            else:
-                req, comment = split_comment(line)
-                if args.upper:
-                    new_req = to_upper_bound(req)
+    modifier = to_upper_bound if args.upper else to_lower_bound
+
+    req_files = [filename for filename in args.filename if not filename.endswith("pyproject.toml")]
+    pyp_files = [filename for filename in args.filename if filename.endswith("pyproject.toml")]
+    if req_files:
+        with fileinput.input(files=req_files) as req_file:
+            for line in req_file:
+                if line.strip().startswith("#"):
+                    # Shortcut comment only lines
+                    print(line.strip())
                 else:
-                    new_req = to_lower_bound(req)
-                print(new_req + comment)
+                    req, comment = split_comment(line)
+                    new_req = modifier(req)
+                    print(new_req + comment)
+    for filename in pyp_files:
+        with open(filename, "rb") as fp:
+            pyproject = tomllib.load(fp)
+            for req in pyproject["project"]["dependencies"]:
+                new_req = modifier(req)
+                print(new_req)
+            optional_dependencies = pyproject["project"].get("optional-dependencies")
+            if optional_dependencies:
+                for opt in optional_dependencies.values():
+                    for req in opt:
+                        new_req = modifier(req)
+                        print(new_req)
 
 
 if __name__ == "__main__":

.ci/scripts/check_release.py

@@ -64,6 +64,7 @@ def main():
         for branch in branches:
             if branch != DEFAULT_BRANCH:
                 # Check if a Z release is needed
+                reasons = []
                 changes = repo.git.ls_tree("-r", "--name-only", f"origin/{branch}", "CHANGES/")
                 z_changelog = False
                 for change in changes.split("\n"):
@@ -76,23 +77,34 @@ def main():
                         )
                     elif ext in Z_CHANGELOG_EXTS:
                         z_changelog = True
+                if z_changelog:
+                    reasons.append("Backports")
 
                 last_tag = repo.git.describe("--tags", "--abbrev=0", f"origin/{branch}")
                 req_txt_diff = repo.git.diff(
                     f"{last_tag}", f"origin/{branch}", "--name-only", "--", "requirements.txt"
                 )
-                if z_changelog or req_txt_diff:
+                if req_txt_diff:
+                    reasons.append("requirements.txt")
+                pyproject_diff = repo.git.diff(
+                    f"{last_tag}", f"origin/{branch}", "--name-only", "--", "pyproject.toml"
+                )
+                if pyproject_diff:
+                    reasons.append(
+                        "pyproject.toml changed (PLEASE check if dependencies are affected."
+                    )
+
+                if reasons:
                     curr_version = Version(last_tag)
                     assert curr_version.base_version.startswith(
                         branch
                     ), "Current-version has to belong to the current branch!"
                     next_version = Version(f"{branch}.{curr_version.micro + 1}")
-                    reason = "CHANGES" if z_changelog else "requirements.txt"
                     print(
                         f"A Z-release is needed for {branch}, "
                         f"Prev: {last_tag}, "
                         f"Next: {next_version.base_version}, "
-                        f"Reason: {reason}"
+                        f"Reason: {','.join(reasons)}"
                     )
                     releases.append(next_version)
             else:

.ci/scripts/check_requirements.py

@@ -5,11 +5,13 @@
 #
 # For more info visit https://github.com/pulp/plugin_template
 
+import tomllib
 import warnings
-from pkg_resources import Requirement
+from packaging.requirements import Requirement
 
 
 CHECK_MATRIX = [
+    ("pyproject.toml", True, True, True),
     ("requirements.txt", True, True, True),
     ("dev_requirements.txt", False, True, False),
     ("ci_requirements.txt", False, True, True),
@@ -20,17 +22,33 @@
     ("clitest_requirements.txt", False, True, True),
 ]
 
-errors = []
 
-for filename, check_upperbound, check_prereleases, check_r in CHECK_MATRIX:
-    try:
+def iterate_file(filename):
+    if filename == "pyproject.toml":
+        with open(filename, "rb") as fd:
+            pyproject_toml = tomllib.load(fd)
+            if "project" in pyproject_toml:
+                for nr, line in enumerate(pyproject_toml["project"]["dependencies"]):
+                    yield nr, line
+    else:
         with open(filename, "r") as fd:
             for nr, line in enumerate(fd.readlines()):
                 line = line.strip()
                 if not line or line.startswith("#"):
                     continue
+                if "#" in line:
+                    line = line.split("#", maxsplit=1)[0]
+                yield nr, line.strip()
+
+
+def main():
+    errors = []
+
+    for filename, check_upperbound, check_prereleases, check_r in CHECK_MATRIX:
+        try:
+            for nr, line in iterate_file(filename):
                 try:
-                    req = Requirement.parse(line)
+                    req = Requirement(line)
                 except ValueError:
                     if line.startswith("git+"):
                         # The single exception...
@@ -49,18 +67,21 @@
                             and req.name != "pulp-file-client"
                         ):
                             errors.append(f"{filename}:{nr}: Prerelease versions found in {line}.")
-                    ops = [op for op, ver in req.specs]
-                    spec = str(req.specs)
+                    ops = [spec.operator for spec in req.specifier]
                     if "~=" in ops:
                         warnings.warn(f"{filename}:{nr}: Please avoid using ~= on {req.name}!")
                     elif "<" not in ops and "<=" not in ops and "==" not in ops:
                         if check_upperbound:
                             errors.append(f"{filename}:{nr}: Upper bound missing in {line}.")
-    except FileNotFoundError:
-        # skip this test for plugins that don't use this requirements.txt
-        pass
+        except FileNotFoundError:
+            # skip this test for plugins that don't use this requirements.txt
+            pass
+
+    if errors:
+        print("Dependency issues found:")
+        print("\n".join(errors))
+        exit(1)
+
 
-if errors:
-    print("Dependency issues found:")
-    print("\n".join(errors))
-    exit(1)
+if __name__ == "__main__":
+    main()

.ci/scripts/collect_changes.py

@@ -103,7 +103,7 @@ def main():
             for change in main_changes:
                 fp.write(change[1])
 
-        repo.git.commit("-m", "Update Changelog", "-m" "[noissue]", CHANGELOG_FILE)
+        repo.git.commit("-m", "Update Changelog", CHANGELOG_FILE)
 
 
 if __name__ == "__main__":

.ci/scripts/pr_labels.py

@@ -0,0 +1,60 @@
+#!/bin/env python3
+
+# This script is running with elevated privileges from the main branch against pull requests.
+
+import re
+import sys
+import tomllib
+from pathlib import Path
+
+from git import Repo
+
+
+def main():
+    assert len(sys.argv) == 3
+
+    with open("pyproject.toml", "rb") as fp:
+        PYPROJECT_TOML = tomllib.load(fp)
+    BLOCKING_REGEX = re.compile(r"DRAFT|WIP|NO\s*MERGE|DO\s*NOT\s*MERGE|EXPERIMENT")
+    ISSUE_REGEX = re.compile(r"(?:fixes|closes)[\s:]+#(\d+)")
+    CHERRY_PICK_REGEX = re.compile(r"^\s*\(cherry picked from commit [0-9a-f]*\)\s*$")
+    try:
+        CHANGELOG_EXTS = {
+            f".{item['directory']}" for item in PYPROJECT_TOML["tool"]["towncrier"]["type"]
+        }
+    except KeyError:
+        CHANGELOG_EXTS = {".feature", ".bugfix", ".doc", ".removal", ".misc"}
+
+    repo = Repo(".")
+
+    base_commit = repo.commit(sys.argv[1])
+    head_commit = repo.commit(sys.argv[2])
+
+    pr_commits = list(repo.iter_commits(f"{base_commit}..{head_commit}"))
+
+    labels = {
+        "multi-commit": len(pr_commits) > 1,
+        "cherry-pick": False,
+        "no-issue": False,
+        "no-changelog": False,
+        "wip": False,
+    }
+    for commit in pr_commits:
+        labels["wip"] |= BLOCKING_REGEX.search(commit.summary) is not None
+        no_issue = ISSUE_REGEX.search(commit.message, re.IGNORECASE) is None
+        labels["no-issue"] |= no_issue
+        cherry_pick = CHERRY_PICK_REGEX.search(commit.message) is not None
+        labels["cherry-pick"] |= cherry_pick
+        changelog_snippets = [
+            k
+            for k in commit.stats.files
+            if k.startswith("CHANGES/") and Path(k).suffix in CHANGELOG_EXTS
+        ]
+        labels["no-changelog"] |= not changelog_snippets
+
+    print("ADD_LABELS=" + ",".join((k for k, v in labels.items() if v)))
+    print("REMOVE_LABELS=" + ",".join((k for k, v in labels.items() if not v)))
+
+
+if __name__ == "__main__":
+    main()

.ci/scripts/validate_commit_message.py

@@ -9,21 +9,16 @@
 import sys
 from pathlib import Path
 import subprocess
-
-
 import os
 import warnings
 from github import Github
 
-
-NO_ISSUE = "[noissue]"
 CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc", ".deprecation"]
+KEYWORDS = ["fixes", "closes"]
+
 sha = sys.argv[1]
 message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8")
 
-
-KEYWORDS = ["fixes", "closes"]
-
 g = Github(os.environ.get("GITHUB_TOKEN"))
 repo = g.get_repo("pulp/pulp_file")
 
@@ -64,15 +59,5 @@ def __check_changelog(issue):
     for issue in pattern.findall(message):
         __check_status(issue)
         __check_changelog(issue)
-else:
-    if NO_ISSUE in message:
-        print("Commit {sha} has no issues but is tagged {tag}.".format(sha=sha[0:7], tag=NO_ISSUE))
-    elif "Merge" in message and "cherry picked from commit" in message:
-        pass
-    else:
-        sys.exit(
-            "Error: no attached issues found for {sha}. If this was intentional, add "
-            " '{tag}' to the commit message.".format(sha=sha[0:7], tag=NO_ISSUE)
-        )
 
 print("Commit message for {sha} passed.".format(sha=sha[0:7]))

.github/template_gitref

@@ -1 +1 @@
-2021.08.26-364-g6f9579c
+2021.08.26-393-g0e700c1

.github/workflows/build.yml

@@ -34,11 +34,11 @@ jobs:
       - name: "Install python dependencies"
         run: |
           echo ::group::PYDEPS
-          pip install packaging twine wheel mkdocs jq
+          pip install build packaging twine wheel mkdocs jq
           echo ::endgroup::
       - name: "Build package"
         run: |
-          python3 setup.py sdist bdist_wheel --python-tag py3
+          python3 -m build
           twine check dist/*
       - name: "Install built packages"
         run: |

.github/workflows/ci.yml

@@ -43,6 +43,9 @@ jobs:
         run: |
           .github/workflows/scripts/check_commit.sh
 
+  docs:
+    uses: "./.github/workflows/docs.yml"
+
   lint:
     uses: "./.github/workflows/lint.yml"
 
@@ -61,6 +64,7 @@ jobs:
       - "check-commits"
       - "lint"
       - "test"
+      - "docs"
     if: "always()"
     steps:
       - name: "Collect needed jobs results"