From 63657a4b26a916e11277eaa3bfb9c69bd1fc93fa Mon Sep 17 00:00:00 2001 From: Sorin Sbarnea Date: Wed, 4 Sep 2024 16:01:35 +0100 Subject: [PATCH] Ensure we use pre-release versions with untagged commits (#303) - Ensure that the changelog generated version matches the galaxy.yml version - Ensure that we generate semver pre-release versions for untagged commits --- .config/constraints.txt | 26 +++++++++++++++++- .config/manifest.txt | 48 +++++++++++++++------------------ .config/requirements-docs.txt | 3 +++ .github/workflows/tox.yml | 7 ++--- docs/build.sh | 50 ++++++++++++++++++++++++++++++++--- tests/run-galaxy-importer | 25 ------------------ tox.ini | 16 +---------- 7 files changed, 101 insertions(+), 74 deletions(-) delete mode 100755 tests/run-galaxy-importer diff --git a/.config/constraints.txt b/.config/constraints.txt index 65151517..be311d15 100644 --- a/.config/constraints.txt +++ b/.config/constraints.txt @@ -13,6 +13,9 @@ aiokafka==0.11.0 aiosignal==1.3.1 alabaster==0.7.16 annotated-types==0.7.0 +ansible-builder==3.1.0 +ansible-compat==24.8.0 +ansible-lint==6.22.2 ansible-pygments==0.1.1 ansible-rulebook==1.1.0 ansible-runner==2.4.0 @@ -23,12 +26,17 @@ antsibull-docs-parser==1.1.0 async-timeout==4.0.3 asyncio-pool==0.6.0 asyncmock==0.4.2 -attrs==24.2.0 +attrs==22.2.0 azure-core==1.30.2 azure-servicebus==7.12.2 babel==2.16.0 backports-tarfile==1.2.0 +bindep==2.11.0 +black==24.8.0 +bleach==3.3.1 +bleach-allowlist==1.0.3 botocore==1.35.7 +bracex==2.5 build==1.2.1 cachetools==5.5.0 certifi==2024.8.30 @@ -42,12 +50,15 @@ coverage-enable-subprocess==1.0 cryptography==43.0.1 diskcache==5.6.3 distlib==0.3.8 +distro==1.9.0 docutils==0.20.1 dpath==2.2.0 drools-jpy==0.3.9 exceptiongroup==1.2.2 filelock==3.15.4 +flake8==6.1.0 frozenlist==1.4.1 +galaxy-importer==0.4.24 gitdb==4.0.11 gitpython==3.1.43 idna==3.8 @@ -68,17 +79,23 @@ jsonschema-specifications==2023.12.1 kafka-python==2.0.2 ; python_version < "3.12" keyring==25.3.0 lockfile==0.12.2 +markdown==3.7 markdown-it-py==3.0.0 markupsafe==2.1.5 +mccabe==0.7.0 mdit-py-plugins==0.4.1 mdurl==0.1.2 mk==2.6.1 mock==5.1.0 more-itertools==10.4.0 multidict==6.0.5 +mypy-extensions==1.0.0 myst-parser==3.0.1 nh3==0.2.18 packaging==24.1 +parsley==1.3 +pathspec==0.12.1 +pbr==6.1.0 perky==0.9.2 pexpect==4.9.0 pip==24.2 @@ -90,9 +107,11 @@ psycopg==3.1.18 psycopg-binary==3.1.18 psycopg-pool==3.2.2 ptyprocess==0.7.0 +pycodestyle==2.11.1 pycparser==2.22 pydantic==2.8.2 pydantic-core==2.20.1 +pyflakes==3.1.0 pygments==2.18.0 pyparsing==3.1.4 pyproject-api==1.7.1 @@ -114,6 +133,8 @@ rich==13.8.0 rpds-py==0.20.0 rstcheck==6.2.4 rstcheck-core==1.2.1 +ruamel-yaml==0.18.6 +ruamel-yaml-clib==0.2.8 semantic-version==2.10.0 shellingham==1.5.4 six==1.16.0 @@ -140,9 +161,12 @@ typing-extensions==4.12.2 urllib3==1.26.20 virtualenv==20.26.3 watchdog==5.0.2 +wcmatch==9.0 +webencodings==0.5.1 websockets==13.0.1 wrapt==1.16.0 xxhash==3.5.0 +yamllint==1.35.1 yarl==1.9.8 zipp==3.20.1 diff --git a/.config/manifest.txt b/.config/manifest.txt index 4fc8e97b..c3129930 100644 --- a/.config/manifest.txt +++ b/.config/manifest.txt @@ -1,21 +1,14 @@ .ansible-lint .flake8 .gitignore +bindep.txt CHANGELOG.md CONTRIBUTING.md -FILES.json -LICENSE -MANIFEST.json -README.md -bindep.txt docs/ docs/.changelog.yml docs/.gitignore docs/antsibull-docs.cfg docs/build.sh -docs/collections/ -docs/collections/ansible/ -docs/collections/ansible/eda/ docs/conf.py docs/docsite/ docs/docsite/links.yml @@ -23,15 +16,13 @@ extensions/ extensions/eda/ extensions/eda/plugins/ extensions/eda/plugins/event_filter/ -extensions/eda/plugins/event_filter/README.md extensions/eda/plugins/event_filter/dashes_to_underscores.py extensions/eda/plugins/event_filter/insert_hosts_to_meta.py extensions/eda/plugins/event_filter/json_filter.py extensions/eda/plugins/event_filter/noop.py extensions/eda/plugins/event_filter/normalize_keys.py +extensions/eda/plugins/event_filter/README.md extensions/eda/plugins/event_source/ -extensions/eda/plugins/event_source/README.md -extensions/eda/plugins/event_source/__init__.py extensions/eda/plugins/event_source/alertmanager.py extensions/eda/plugins/event_source/aws_cloudtrail.py extensions/eda/plugins/event_source/aws_sqs_queue.py @@ -43,6 +34,7 @@ extensions/eda/plugins/event_source/journald.py extensions/eda/plugins/event_source/kafka.py extensions/eda/plugins/event_source/pg_listener.py extensions/eda/plugins/event_source/range.py +extensions/eda/plugins/event_source/README.md extensions/eda/plugins/event_source/schemas/ extensions/eda/plugins/event_source/schemas/alertmanager.json extensions/eda/plugins/event_source/schemas/aws_cloudtrail.json @@ -60,6 +52,7 @@ extensions/eda/plugins/event_source/schemas/webhook.json extensions/eda/plugins/event_source/tick.py extensions/eda/plugins/event_source/url_check.py extensions/eda/plugins/event_source/webhook.py +extensions/eda/plugins/event_source/__init__.py extensions/eda/rulebooks/ extensions/eda/rulebooks/demo_controller_rulebook.yml extensions/eda/rulebooks/demo_rulebook.yml @@ -72,25 +65,19 @@ extensions/eda/rulebooks/journald_events.yml extensions/eda/rulebooks/kafka-test-rules.yml extensions/eda/rulebooks/local-test-rules.yml extensions/eda/rulebooks/process_down.yml +FILES.json +LICENSE +MANIFEST.json meta/ meta/extensions.yml meta/runtime.yml playbooks/ playbooks/hello.yml plugins/ -plugins/README.md plugins/doc_fragments/ -plugins/doc_fragments/__init__.py plugins/doc_fragments/eda_controller.py -plugins/module_utils/ -plugins/module_utils/__init__.py -plugins/module_utils/arguments.py -plugins/module_utils/client.py -plugins/module_utils/common.py -plugins/module_utils/controller.py -plugins/module_utils/errors.py +plugins/doc_fragments/__init__.py plugins/modules/ -plugins/modules/__init__.py plugins/modules/activation.py plugins/modules/activation_info.py plugins/modules/controller_token.py @@ -105,14 +92,23 @@ plugins/modules/event_stream_info.py plugins/modules/project.py plugins/modules/project_info.py plugins/modules/user.py +plugins/modules/__init__.py +plugins/module_utils/ +plugins/module_utils/arguments.py +plugins/module_utils/client.py +plugins/module_utils/common.py +plugins/module_utils/controller.py +plugins/module_utils/errors.py +plugins/module_utils/__init__.py +plugins/README.md +README.md requirements.txt schemas/ -schemas/README.md schemas/event/ -schemas/event/README.md schemas/event/range.json -test_requirements.txt +schemas/event/README.md +schemas/README.md tests/ -tests/__init__.py tests/integration/ -tests/run-galaxy-importer +tests/__init__.py +test_requirements.txt diff --git a/.config/requirements-docs.txt b/.config/requirements-docs.txt index 4c7f9f3f..a2463cb5 100644 --- a/.config/requirements-docs.txt +++ b/.config/requirements-docs.txt @@ -1,6 +1,9 @@ ansible-pygments antsibull-docs>=2.13.1,<3.0.0 +attrs +galaxy-importer>=0.4.24 mk>=2.6.1 myst-parser +setuptools # https://issues.redhat.com/browse/AAH-3375 sphinx sphinx-ansible-theme>=0.10.3 diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index f7e464d5..7f1466aa 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -47,12 +47,13 @@ jobs: py39:tox -e py39-unit;tox -e py39-integration;tox -e coverage py310:tox -e py310-unit; tox -e py310-integration;tox -e coverage py311:tox -e py311-unit; tox -e py311-integration;tox -e coverage - py312:tox -e py312-unit; tox -e py312-integration;tox -e coverage - py39-macos:tox -e py39-unit;tox -e py39-integration;tox -e coverage - py312-macos:tox -e py312-unit;tox -e py312-integration;tox -e coverage + py312:tox -e py312-unit;tox -e py312-unit; tox -e py312-integration;tox -e coverage + py39-macos:tox -e py39-docs;tox -e py39-unit;tox -e py39-integration;tox -e coverage + py312-macos:tox -e py312-docs;tox -e py312-unit;tox -e py312-integration;tox -e coverage py311-linux-arm64:tox -e py312-unit;tox -e py312-integration;tox -e coverage staging: tox -e py311-staging # ^ arm64 runner is using py311 for matching python version used in AAP 2.5 + # building docs on macos is needed to avoid diverging behavior platforms: linux,macos,linux-arm64:ubuntu-24.04-arm64-2core skip_explode: "1" build: diff --git a/docs/build.sh b/docs/build.sh index d5707b1f..3ebe870c 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -4,10 +4,8 @@ set -eu pushd "$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" trap "{ popd; }" EXIT +GALAXY_VERSION=$(python -c "import yaml; print(yaml.safe_load(open('../galaxy.yml'))['version'])") # Determine the collection version, if current commit is tagged, use it. Otherwise, generate pre-release version. -COLLECTION_VERSION_WITH_PREFIX=$(git describe --dirty --tags --long --match "v*.*" | cut -f1,2 -d'-') -COLLECTION_VERSION=${COLLECTION_VERSION_WITH_PREFIX/v/} -echo "INFO: Determined collection version to be ${COLLECTION_VERSION}" # Create collection documentation mkdir -p rst @@ -28,6 +26,12 @@ mk -v changelog cp CHANGELOG.md ./docs/rst/changelog.md popd +CHANGELOG_VERSION=$(grep -m 1 '^#' rst/changelog.md | awk '/^#/ {print $2; exit}' | sed 's/v//') + +if [[ "${GALAXY_VERSION}" != "${CHANGELOG_VERSION}" ]]; then + echo "WARN: galaxy.yaml collection version ${GALAXY_VERSION} does not match the changelog version ${CHANGELOG_VERSION}. Please update the galaxy.yaml version to match the changelog version." +fi + cat >rst/_.rst < .config/manifest.txt +sync + +echo "INFO: Restore temporary modified galaxy.yml file." +git checkout HEAD -- galaxy.yml >/dev/null + +git --no-pager diff -U0 --minimal || { + echo "ERROR: Manifest at .config/manifest.txt changed, please update it." + exit 3 +} +echo "INFO: Galaxy importer check passed." + +echo "INFO: Determined collection version ${GALAXY_VERSION}" diff --git a/tests/run-galaxy-importer b/tests/run-galaxy-importer deleted file mode 100755 index 790e1fe7..00000000 --- a/tests/run-galaxy-importer +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -eu -set -ex - -echo "INFO: Cleaning up old files" -rm -f ./*.tar.gz importer_result.json - -# we produce the changelog as that is not tracked in git -echo "INFO: Generating changelog" -# python3 -m antsibull_changelog generate -vvv - -echo "INFO: Run galaxy-importer" -# produces importer_result.json -GALAXY_IMPORTER_CONFIG=.config/galaxy-importer.cfg python3 -m galaxy_importer.main --git-clone-path=. --output-path=. - -echo "INFO: Check if importer_result.json is not empty" -ARCHIVE=$(python3 -c "import json; print(json.load(open('importer_result.json'))[-1])") - -echo "INFO: Check that list of files (manifest) inside the collection archive is the expected one." -tar -tf "$ARCHIVE" | sort > .config/manifest.txt -sync -git --no-pager diff -U0 --minimal || { - echo "ERROR: Manifest at .config/manifest.txt changed, please update it." - exit 3 -} -echo "INFO: Galaxy importer check passed." diff --git a/tox.ini b/tox.ini index a17c71b3..f7573e66 100644 --- a/tox.ini +++ b/tox.ini @@ -171,7 +171,7 @@ commands = # produces tests/output/reports/coverage.xml ansible-test coverage xml --requirements -[testenv:docs] +[testenv:docs,py{39,310,311,312,313}-docs] description = Generate plugins documentation under /docs directory using antsibull-docs # When run under RTD it will reuse their pre-created environment and save time. env_dir = {env:READTHEDOCS_VIRTUALENV_PATH:{work_dir}/docs}} @@ -186,17 +186,3 @@ commands = allowlist_externals = ./docs/build.sh bash - -[testenv:build] -description = Build and validates the collection via galaxy-importer (requires building docs first) -depends = docs -deps = - attrs - antsibull-changelog>=0.29.0 - galaxy-importer>=0.4.24 - setuptools # https://issues.redhat.com/browse/AAH-3375 - tox -commands_pre = -commands = - tox -e docs - bash ./tests/run-galaxy-importer