Update CI files

[noissue]
pulp · Jun 16, 2024 · 4b7d7c3 · 4b7d7c3
1 parent 4e40770
commit 4b7d7c3
Show file tree

Hide file tree

Showing 23 changed files with 633 additions and 827 deletions.
diff --git a/.ci/ansible/settings.py.j2 b/.ci/ansible/settings.py.j2
@@ -49,7 +49,7 @@ AZURE_CONTAINER =  "pulp-test"
 AZURE_LOCATION = "pulp3"
 AZURE_OVERWRITE_FILES = True
 AZURE_URL_EXPIRATION_SECS = 120
-AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol={{ pulp_scheme }};AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint={{ pulp_scheme }}://ci-azurite:10000/devstoreaccount1;'
+AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;'
 {% endif %}
 
 {% if gcp_test | default(false) %}

diff --git a/.ci/scripts/collect_changes.py b/.ci/scripts/collect_changes.py
@@ -1,3 +1,4 @@
+#!/bin/env python3
 # WARNING: DO NOT EDIT!
 #
 # This file was generated by plugin_template, and is managed by it. Please use
@@ -8,13 +9,14 @@
 import itertools
 import os
 import re
+import tomllib
 
-import toml
 from git import GitCommandError, Repo
 from packaging.version import parse as parse_version
 
 # Read Towncrier settings
-tc_settings = toml.load("pyproject.toml")["tool"]["towncrier"]
+with open("pyproject.toml", "rb") as fp:
+    tc_settings = tomllib.load(fp)["tool"]["towncrier"]
 
 CHANGELOG_FILE = tc_settings.get("filename", "NEWS.rst")
 START_STRING = tc_settings.get(
@@ -79,7 +81,7 @@ def main():
     old_length = len(main_changes)
 
     for branch in branches:
-        print(f"Looking at branch {branch}")
+        print(f"Looking for './{CHANGELOG_FILE}' at branch {branch}")
         try:
             changelog = get_changelog(repo, branch)
         except GitCommandError:

diff --git a/.github/template_gitref b/.github/template_gitref
@@ -1 +1 @@
-2021.08.26-327-g6a0d984
+2021.08.26-339-gf0d923e
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -23,7 +23,7 @@ jobs:
         with:
           fetch-depth: 1
           path: "pulp_file"
-      - uses: "actions/setup-python@v4"
+      - uses: "actions/setup-python@v5"
         with:
           python-version: "3.11"
       - name: "Install python dependencies"
@@ -36,9 +36,10 @@ jobs:
           python3 setup.py sdist bdist_wheel --python-tag py3
           twine check dist/*
       - name: "Upload Package whl"
-        uses: "actions/upload-artifact@v3"
+        uses: "actions/upload-artifact@v4"
         with:
           name: "plugin_package"
           path: "pulp_file/dist/"
           if-no-files-found: "error"
           retention-days: 5
+          overwrite: true
diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml
@@ -28,7 +28,7 @@ jobs:
         with:
           fetch-depth: 1
 
-      - uses: "actions/setup-python@v4"
+      - uses: "actions/setup-python@v5"
         with:
           python-version: "3.11"
 

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -25,7 +25,7 @@ jobs:
         with:
           fetch-depth: 0
           path: "pulp_file"
-      - uses: "actions/setup-python@v4"
+      - uses: "actions/setup-python@v5"
         with:
           python-version: "3.11"
       - name: "Install python dependencies"
@@ -67,10 +67,11 @@ jobs:
           mkdir -p "pulp_file"
         working-directory: "."
       - name: "Download Deprecations"
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
-          name: "deprecations"
+          pattern: "deprecations-*"
           path: "pulp_file"
+          merge-multiple: true
       - name: "Print deprecations"
         run: |
           cat deprecations-*.txt | sort -u

diff --git a/.github/workflows/create-branch.yml b/.github/workflows/create-branch.yml
@@ -26,7 +26,7 @@ jobs:
           fetch-depth: 0
           path: "pulp_file"
 
-      - uses: "actions/setup-python@v4"
+      - uses: "actions/setup-python@v5"
         with:
           python-version: "3.11"
 

diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -24,7 +24,7 @@ jobs:
           fetch-depth: 1
           path: "pulp_file"
 
-      - uses: "actions/setup-python@v4"
+      - uses: "actions/setup-python@v5"
         with:
           python-version: "3.11"
 

diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -38,7 +38,7 @@ jobs:
           fetch-depth: 0
           path: "pulp_file"
 
-      - uses: "actions/setup-python@v4"
+      - uses: "actions/setup-python@v5"
         with:
           python-version: "3.11"
 
@@ -76,12 +76,12 @@ jobs:
           fetch-depth: 1
           path: "pulp_file"
 
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: "plugin_package"
           path: "pulp_file/dist/"
 
-      - uses: "actions/setup-python@v4"
+      - uses: "actions/setup-python@v5"
         with:
           python-version: "3.11"
 
@@ -96,13 +96,13 @@ jobs:
           echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV
 
       - name: Download built docs
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: "docs.tar"
           path: "pulp_file"
 
       - name: Download Python client docs
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: "python-client-docs.tar"
           path: "pulp_file"
@@ -140,12 +140,12 @@ jobs:
           repository: "pulp/pulp-openapi-generator"
           path: "pulp-openapi-generator"
 
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: "plugin_package"
           path: "pulp_file/dist/"
 
-      - uses: "actions/setup-python@v4"
+      - uses: "actions/setup-python@v5"
         with:
           python-version: "3.11"
 

diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -39,11 +39,11 @@ jobs:
           repository: "pulp/pulp-openapi-generator"
           path: "pulp-openapi-generator"
 
-      - uses: "actions/setup-python@v4"
+      - uses: "actions/setup-python@v5"
         with:
           python-version: "3.11"
 
-      - uses: "actions/download-artifact@v3"
+      - uses: "actions/download-artifact@v4"
         with:
           name: "plugin_package"
           path: "pulp_file/dist/"
@@ -88,25 +88,32 @@ jobs:
         run: |
           .github/workflows/scripts/install_ruby_client.sh
         shell: "bash"
+
       - name: "Upload python client packages"
-        uses: "actions/upload-artifact@v3"
+        uses: "actions/upload-artifact@v4"
         with:
           name: "python-client.tar"
-          path: "pulp_file/file-python-client.tar"
+          path: |
+            pulp_file/file-python-client.tar
           if-no-files-found: "error"
+          overwrite: true
 
       - name: "Upload python client docs"
-        uses: "actions/upload-artifact@v3"
+        uses: "actions/upload-artifact@v4"
         with:
           name: "python-client-docs.tar"
-          path: "pulp_file/file-python-client-docs.tar"
+          path: |
+            pulp_file/file-python-client-docs.tar
           if-no-files-found: "error"
+          overwrite: true
       - name: "Upload ruby client packages"
-        uses: "actions/upload-artifact@v3"
+        uses: "actions/upload-artifact@v4"
         with:
           name: "ruby-client.tar"
-          path: "pulp_file/file-ruby-client.tar"
+          path: |
+            pulp_file/file-ruby-client.tar
           if-no-files-found: "error"
+          overwrite: true
       - name: Build docs
         run: |
           export DJANGO_SETTINGS_MODULE=pulpcore.app.settings
@@ -115,10 +122,12 @@ jobs:
           tar -cvf docs/docs.tar docs/_build
 
       - name: "Upload built docs"
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: "docs.tar"
           path: "pulp_file/docs/docs.tar"
+          if-no-files-found: "error"
+          overwrite: true
 
       - name: "Logs"
         if: always()
@@ -145,12 +154,12 @@ jobs:
           fetch-depth: 1
           path: "pulp_file"
 
-      - uses: "actions/download-artifact@v3"
+      - uses: "actions/download-artifact@v4"
         with:
           name: "plugin_package"
           path: "pulp_file/dist/"
 
-      - uses: "actions/setup-python@v4"
+      - uses: "actions/setup-python@v5"
         with:
           python-version: "3.11"
 
@@ -184,7 +193,7 @@ jobs:
           path: "pulp_file"
 
       - name: "Download Python client"
-        uses: "actions/download-artifact@v3"
+        uses: "actions/download-artifact@v4"
         with:
           name: "python-client.tar"
           path: "pulp_file/"
@@ -193,7 +202,7 @@ jobs:
         run: |
           tar -xvf file-python-client.tar
 
-      - uses: "actions/setup-python@v4"
+      - uses: "actions/setup-python@v5"
         with:
           python-version: "3.11"
 
@@ -227,7 +236,7 @@ jobs:
           path: "pulp_file"
 
       - name: "Download Ruby client"
-        uses: "actions/download-artifact@v3"
+        uses: "actions/download-artifact@v4"
         with:
           name: "ruby-client.tar"
           path: "pulp_file/"
@@ -263,7 +272,7 @@ jobs:
           fetch-depth: 1
           path: "pulp_file"
 
-      - uses: "actions/setup-python@v4"
+      - uses: "actions/setup-python@v5"
         with:
           python-version: "3.11"
 
@@ -280,13 +289,13 @@ jobs:
           SECRETS_CONTEXT: "${{ toJson(secrets) }}"
 
       - name: "Download built docs"
-        uses: "actions/download-artifact@v3"
+        uses: "actions/download-artifact@v4"
         with:
           name: "docs.tar"
           path: "pulp_file/"
 
       - name: "Download Python client docs"
-        uses: "actions/download-artifact@v3"
+        uses: "actions/download-artifact@v4"
         with:
           name: "python-client-docs.tar"
           path: "pulp_file/"

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -28,7 +28,7 @@ jobs:
           path: "pulp_file"
           token: ${{ secrets.RELEASE_TOKEN }}
 
-      - uses: "actions/setup-python@v4"
+      - uses: "actions/setup-python@v5"
         with:
           python-version: "3.11"
 

diff --git a/.github/workflows/scripts/before_script.sh b/.github/workflows/scripts/before_script.sh
@@ -36,12 +36,6 @@ tail -v -n +1 .ci/ansible/Containerfile
 cmd_prefix bash -c "echo '%wheel        ALL=(ALL)       NOPASSWD: ALL' > /etc/sudoers.d/nopasswd"
 cmd_prefix bash -c "usermod -a -G wheel pulp"
 
-SCENARIOS=("pulp" "performance" "azure" "gcp" "s3" "generate-bindings" "lowerbounds")
-if [[ " ${SCENARIOS[*]} " =~ " ${TEST} " ]]; then
-  # Many functional tests require these
-  cmd_prefix dnf install -yq lsof which
-fi
-
 if [[ "${REDIS_DISABLED:-false}" == true ]]; then
   cmd_prefix bash -c "s6-rc -d change redis"
   echo "The Redis service was disabled for $TEST"

diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh
@@ -114,18 +114,12 @@ pulp_scenario_env: {}\
 fi
 
 if [ "$TEST" = "azure" ]; then
-  mkdir -p azurite
-  cd azurite
-  openssl req -newkey rsa:2048 -x509 -nodes -keyout azkey.pem -new -out azcert.pem -sha256 -days 365 -addext "subjectAltName=DNS:ci-azurite" -subj "/C=CO/ST=ST/L=LO/O=OR/OU=OU/CN=CN"
-  sudo cp azcert.pem /usr/local/share/ca-certificates/azcert.crt
-  sudo dpkg-reconfigure ca-certificates
-  cd ..
   sed -i -e '/^services:/a \
   - name: ci-azurite\
     image: mcr.microsoft.com/azure-storage/azurite\
     volumes:\
       - ./azurite:/etc/pulp\
-    command: "azurite-blob --blobHost 0.0.0.0 --cert /etc/pulp/azcert.pem --key /etc/pulp/azkey.pem"' vars/main.yaml
+    command: "azurite-blob --blobHost 0.0.0.0"' vars/main.yaml
   sed -i -e '$a azure_test: true\
 pulp_scenario_settings: null\
 pulp_scenario_env: {}\
@@ -161,31 +155,17 @@ sudo docker cp pulp:/etc/pulp/certs/pulp_webserver.crt /usr/local/share/ca-certi
 # Hack: adding pulp CA to certifi.where()
 CERTIFI=$(python -c 'import certifi; print(certifi.where())')
 cat /usr/local/share/ca-certificates/pulp_webserver.crt | sudo tee -a "$CERTIFI" > /dev/null
-if [[ "$TEST" = "azure" ]]; then
-  cat /usr/local/share/ca-certificates/azcert.crt | sudo tee -a "$CERTIFI" > /dev/null
-fi
 
 # Hack: adding pulp CA to default CA file
 CERT=$(python -c 'import ssl; print(ssl.get_default_verify_paths().openssl_cafile)')
-cat "$CERTIFI" | sudo tee -a "$CERT" > /dev/null
+cat /usr/local/share/ca-certificates/pulp_webserver.crt | sudo tee -a "$CERT" > /dev/null
 
 # Updating certs
 sudo update-ca-certificates
 echo ::endgroup::
 
-# Add our azcert.crt certificate to the container image along with the certificates from certifi
-# so that we can use HTTPS with our fake Azure CI. certifi is self-contained and doesn't allow
-# extension or modification of the trust store, so we do a weird and hacky thing (above) where we just
-# overwrite or append to certifi's trust store behind it's back.
-#
-# We do this for both the CI host and the CI image.
 if [[ "$TEST" = "azure" ]]; then
-  AZCERTIFI=$(/opt/az/bin/python3 -c 'import certifi; print(certifi.where())')
-  PULPCERTIFI=$(cmd_prefix python3 -c 'import certifi; print(certifi.where())')
-  cat /usr/local/share/ca-certificates/azcert.crt >> $AZCERTIFI
-  cat /usr/local/share/ca-certificates/azcert.crt | cmd_stdin_prefix tee -a "$PULPCERTIFI" > /dev/null
-  cat /usr/local/share/ca-certificates/azcert.crt | cmd_stdin_prefix tee -a /etc/pki/tls/cert.pem > /dev/null
-  AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=https;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=https://ci-azurite:10000/devstoreaccount1;'
+  AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;'
   az storage container create --name pulp-test --connection-string $AZURE_STORAGE_CONNECTION_STRING
 fi
 

diff --git a/.github/workflows/scripts/publish_client_pypi.sh b/.github/workflows/scripts/publish_client_pypi.sh
@@ -24,10 +24,8 @@ RESPONSE="$(curl --write-out '%{http_code}' --silent --output /dev/null "https:/
 if [ "$RESPONSE" == "200" ];
 then
   echo "pulp_file client $VERSION has already been released. Skipping."
-  exit
+else
+  twine upload -u __token__ -p "$PYPI_API_TOKEN" \
+  "dist/pulp_file_client-$VERSION-py3-none-any.whl" \
+  "dist/pulp_file-client-$VERSION.tar.gz"
 fi
-
-twine upload -u __token__ -p "$PYPI_API_TOKEN" \
-"dist/pulp_file_client-$VERSION-py3-none-any.whl" \
-"dist/pulp_file-client-$VERSION.tar.gz" \
-;