From 07251981c5d91d8763d11370a04395bafdda3b55 Mon Sep 17 00:00:00 2001 From: "laysa.uchoa" Date: Sat, 10 Dec 2022 09:10:47 +0100 Subject: [PATCH 1/5] Add script to check renamed files If a file is renamed, a redirect should be added. This script helps to check and report whether a renamed file is missing redirects. More info about the redirects process: https://docs.aiven.io/docs/community/documentation/tips-tricks/renaming-files.html --- scripts/check_renamed_files.py | 123 +++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 scripts/check_renamed_files.py diff --git a/scripts/check_renamed_files.py b/scripts/check_renamed_files.py new file mode 100644 index 0000000000..3506ed4ab9 --- /dev/null +++ b/scripts/check_renamed_files.py @@ -0,0 +1,123 @@ +""" +This script checks if a renamed file is added for proper redirection on redirects_ file. +""" +import os, sys, argparse +import coloredlogs, logging +from typing import Dict, List + +p = os.path.abspath(os.path.join(__file__, "../../..")) +os.chdir(p) + +REDIRECTED_FILE = "_redirects" + +logger = logging.getLogger("check_renamed_files") +logging.basicConfig(format="%(asctime)s - [%(levelname)s] - %(message)s") + + +FIELD_STYLES = dict(asctime=dict(color="yellow"), levelname=dict(color="magenta")) +LEVEL_STYLES = dict( + debug=dict(color="green"), + info=dict(color="cyan"), + verbose=dict(color="blue"), + warning=dict(color="yellow"), + error=dict(color="red"), + critical=dict(color="red"), +) + +coloredlogs.install( + level="DEBUG", + logger=logger, + isatty=True, + level_styles=LEVEL_STYLES, + field_styles=FIELD_STYLES, +) + + +def find_redirected() -> Dict: + """Creates a dict with the redirected files + on the _redirects file. + + { previous : redirected} + + :returns: set with redirected files + :rtype: set + """ + all_redirected_links = {} + with open(REDIRECTED_FILE) as f: + lines = f.readlines() + for line in lines: + if line.startswith("/docs/") and not line.startswith("/docs/:"): + previous, redirected = [x.lstrip("/") for x in line.split()] + # avoid repetitions of `html` + if not previous.endswith(".html"): + all_redirected_links[previous] = redirected + + return all_redirected_links + + +def check_missing_redirects(renamed_files: List[str]): + """ + Creates a dict with all missing redirects. + + { previous_link: current_link } + + :returns: missing redirects + :rtype: dict + """ + + INPUT = os.environ["ALL_OLD_AND_NEW_RENAMED_FILES"] + all_new_and_renamed_files = dict([x.split(",")[::-1] for x in INPUT.split(" ")]) + + missing_redirects = {} + all_redirected_links = find_redirected() + for renamed in renamed_files: + try: + previous_link = all_new_and_renamed_files[renamed].rstrip(".rst") + logger.debug(previous_link) + except KeyError: + logger.error("Missing renamed files on all new and renamed files.") + exit(0) + current_link = renamed.rstrip(".rst") + if all_redirected_links.get(previous_link) != current_link: + missing_redirects[previous_link] = current_link + return missing_redirects + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Script to find missing redirects.") + parser.add_argument( + "--renamed_files", + help="delimited list input", + type=str, + nargs="*", + default=" ", + ) + parser.add_argument( + "--path_file", help="path to a file", type=str, default=os.getenv("GITHUB_ENV") + ) + + renamed_files = parser.parse_args().renamed_files + env_file = parser.parse_args().path_file + + if not renamed_files: + logger.info("No files are renamed.") + sys.exit() + + missing_redirects = check_missing_redirects(renamed_files) + if not missing_redirects: + logger.info("Files renamed and redirects are added to _redirects.") + sys.exit() + + res = "" + h = ["Previous Name", "Current Name"] + logger.error("{:<40s} {:<40s}".format(*h)) + for k, v in missing_redirects.items(): + logger.error("/{:<40s} /{:<40s}".format(k, v)) + + logger.info("🚨 Seems like you forgot to add redirects for the renamed files. 🚨") + logger.info( + "ℹī¸ More info: https://docs.aiven.io/docs/community/documentation/tips-tricks/renaming-files.html" + ) + + with open(env_file, "a") as myfile: + myfile.write(f"MISSING_REDIRECTION={missing_redirects}") From 43aa5b9a0cb9cf561045214c9baf7494bbe4b917 Mon Sep 17 00:00:00 2001 From: "laysa.uchoa" Date: Sat, 10 Dec 2022 09:14:51 +0100 Subject: [PATCH 2/5] Add redirect checks workflow This checks in a pull request if a file was renamed without adding redirects. Shows the files, and info how to fix it. --- .github/workflows/redirect_checks.yaml | 57 ++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 .github/workflows/redirect_checks.yaml diff --git a/.github/workflows/redirect_checks.yaml b/.github/workflows/redirect_checks.yaml new file mode 100644 index 0000000000..c0b9d60d0c --- /dev/null +++ b/.github/workflows/redirect_checks.yaml @@ -0,0 +1,57 @@ +name: "Redirects in renamed files" +"on": + pull_request: + branches: + - main + +jobs: + prose: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Get changed files in the docs folder + id: changed-files + uses: tj-actions/changed-files@v34 + with: + include_all_old_new_renamed_files: true + files: | + docs/** + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.8 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements-dev.txt + + - name: Check renamed files + id: missing_redirection + env: + ALL_OLD_AND_NEW_RENAMED_FILES: ${{ steps.changed-files.outputs.all_old_new_renamed_files }} + run: | + for file in ${{ steps.changed-files.outputs.renamed_files }}; + do + if [ "${file: -4}" == ".rst" ] + file="$file" | sed -e 's/\.[^.]*$//' + then + var="$var $file" + fi + done + echo "$var" + if ! [ -z "$var" ];then + python "scripts/aiven/check_renamed_files.py" --renamed_files ${{ steps.changed-files.outputs.renamed_files }} + fi + + - name: Check env + env: + MISSING_REDIRECTION: ${{ env.MISSING_REDIRECTION }} + if: "${{ env.MISSING_REDIRECTION != '' }}" + run: | + echo "::error::Files were renamed with missing _redirects. Add redirect to renamed files to fix it. Renamed: ${{env.MISSING_REDIRECTION}}" + echo "::error::For full report, check logs on the step above." + exit 1 From ae381be0e5aeef9fd1bc2b5768682876206994ff Mon Sep 17 00:00:00 2001 From: "laysa.uchoa" Date: Sat, 10 Dec 2022 09:46:57 +0100 Subject: [PATCH 3/5] Fix formatting --- .github/workflows/redirect_checks.yaml | 2 +- scripts/check_renamed_files.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/redirect_checks.yaml b/.github/workflows/redirect_checks.yaml index c0b9d60d0c..f7894e9456 100644 --- a/.github/workflows/redirect_checks.yaml +++ b/.github/workflows/redirect_checks.yaml @@ -44,7 +44,7 @@ jobs: done echo "$var" if ! [ -z "$var" ];then - python "scripts/aiven/check_renamed_files.py" --renamed_files ${{ steps.changed-files.outputs.renamed_files }} + python "scripts/check_renamed_files.py" --renamed_files ${{ steps.changed-files.outputs.renamed_files }} fi - name: Check env diff --git a/scripts/check_renamed_files.py b/scripts/check_renamed_files.py index 3506ed4ab9..9b2412ea33 100644 --- a/scripts/check_renamed_files.py +++ b/scripts/check_renamed_files.py @@ -5,10 +5,9 @@ import coloredlogs, logging from typing import Dict, List -p = os.path.abspath(os.path.join(__file__, "../../..")) -os.chdir(p) +myPath = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, myPath + "/../../") -REDIRECTED_FILE = "_redirects" logger = logging.getLogger("check_renamed_files") logging.basicConfig(format="%(asctime)s - [%(levelname)s] - %(message)s") @@ -42,6 +41,7 @@ def find_redirected() -> Dict: :returns: set with redirected files :rtype: set """ + REDIRECTED_FILE = "_redirects" all_redirected_links = {} with open(REDIRECTED_FILE) as f: lines = f.readlines() @@ -109,10 +109,10 @@ def check_missing_redirects(renamed_files: List[str]): sys.exit() res = "" - h = ["Previous Name", "Current Name"] - logger.error("{:<40s} {:<40s}".format(*h)) + h = ["Previous Name", " Current Name"] + logger.error("{:<45s} {:<45s}".format(*h)) for k, v in missing_redirects.items(): - logger.error("/{:<40s} /{:<40s}".format(k, v)) + logger.error("/{:<45s} /{:<45s}".format(k, v)) logger.info("🚨 Seems like you forgot to add redirects for the renamed files. 🚨") logger.info( From 167f3938f4e1980ebbb7f0f6abb02e7f30859b79 Mon Sep 17 00:00:00 2001 From: "laysa.uchoa" Date: Sat, 10 Dec 2022 09:47:31 +0100 Subject: [PATCH 4/5] Add tests --- scripts/__init__.py | 0 scripts/tests/test_renamed_files.py | 58 +++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 scripts/__init__.py create mode 100644 scripts/tests/test_renamed_files.py diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/tests/test_renamed_files.py b/scripts/tests/test_renamed_files.py new file mode 100644 index 0000000000..8303e2c8ee --- /dev/null +++ b/scripts/tests/test_renamed_files.py @@ -0,0 +1,58 @@ +import sys, os +import mock, pytest + +myPath = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, myPath + "/../../") +from scripts.check_renamed_files import check_missing_redirects + + +@mock.patch.dict( + os.environ, + { + "ALL_OLD_AND_NEW_RENAMED_FILES": "docs/tools.rst,docs/renamed_tools_file.rst docs/community.rst,docs/renamed_community_file.rst docs/integrations.rst,docs/renamed_integration_file.rst docs/platform.rst,docs/platform2.rst docs/products/opensearch/howto/connect-with-python.rst,docs/products/opensearch/howto/renamed_opensearch_file.rst" + }, +) +def test_check_missing_redirects_with_missing_links(): + renamed_files = [ + "docs/renamed_community_file.rst", + "docs/renamed_integration_file.rst", + "docs/renamed_tools_file.rst", + "docs/products/opensearch/howto/renamed_opensearch_file.rst", + ] + missing_redirects = check_missing_redirects(renamed_files) + assert missing_redirects == { + "docs/community": "docs/renamed_community_file", + "docs/integration": "docs/renamed_integration_file", + "docs/tool": "docs/renamed_tools_file", + "docs/products/opensearch/howto/connect-with-python": "docs/products/opensearch/howto/renamed_opensearch_file", + } + + +@mock.patch.dict( + os.environ, + { + "ALL_OLD_AND_NEW_RENAMED_FILES": "docs/tools.rst,docs/renamed_tools_file.rst docs/community.rst,docs/renamed_community_file.rst docs/integrations.rst,docs/renamed_integration_file.rst docs/platform.rst,docs/platform2.rst docs/products/opensearch/howto/connect-with-python.rst,docs/products/opensearch/howto/renamed_opensearch_file.rst" + }, +) +def test_check_missing_redirects_with_no_renamed_files(): + """ + Test when there were previous changes in all old and new renamed file, + without renamed files in current commit. No missind redirects should be + reported + """ + renamed_files = [] + missing_redirects = check_missing_redirects(renamed_files) + assert missing_redirects == {} + + +@mock.patch.dict( + os.environ, + {"ALL_OLD_AND_NEW_RENAMED_FILES": ""}, +) +def test_check_missing_redirects_no_files_changed_and_no_renamed_files(): + """ + Function should raise an error if no files changed and this function is called + """ + with pytest.raises(ValueError): + renamed_files = [] + check_missing_redirects(renamed_files) From d3bf7a6194ca8260a9cd9e18aeb68d6c29385562 Mon Sep 17 00:00:00 2001 From: "laysa.uchoa" Date: Sat, 10 Dec 2022 09:54:42 +0100 Subject: [PATCH 5/5] Add coloredlogs in dev requirements --- requirements-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index 191c2ec315..505d123b20 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1 +1,2 @@ ultimate-sitemap-parser==0.5 +coloredlogs==15.0.1