From a67459ea5462fc02fa5e32d87893fcc8e7fcf0d7 Mon Sep 17 00:00:00 2001 From: Catalin Date: Thu, 1 Aug 2024 15:23:06 +0300 Subject: [PATCH 1/4] HDX-9928 update config and add actions for linting & type checking --- .github/workflows/run-linter.yml | 72 ++++++++++++++++++++ .github/workflows/run-types-checker.yml | 88 +++++++++++++++++++++++++ pyproject.toml | 58 ++++++++++++---- 3 files changed, 206 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/run-linter.yml create mode 100644 .github/workflows/run-types-checker.yml diff --git a/.github/workflows/run-linter.yml b/.github/workflows/run-linter.yml new file mode 100644 index 0000000000..0c2492318f --- /dev/null +++ b/.github/workflows/run-linter.yml @@ -0,0 +1,72 @@ +name: HDX/CKAN linter + +on: + push: + branches: [ '**' ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ dev ] + +jobs: + linter: + runs-on: ubuntu-latest + + steps: + + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install ruff linter + run: pip install ruff + + - name: Run ruff linter for ckanext-hdx_dataviz + run: ruff check ckanext-hdx_dataviz + continue-on-error: true + + - name: Run ruff linter for ckanext-hdx_hxl_preview + run: ruff check ckanext-hdx_hxl_preview + continue-on-error: true + + - name: Run ruff linter for ckanext-hdx_office_preview + run: ruff check ckanext-hdx_office_preview + continue-on-error: true + + - name: Run ruff linter for ckanext-hdx_org_group + run: ruff check ckanext-hdx_org_group + continue-on-error: true + + - name: Run ruff linter for ckanext-hdx_package + run: ruff check ckanext-hdx_package + continue-on-error: true + + - name: Run ruff linter for ckanext-hdx_pages + run: ruff check ckanext-hdx_pages + continue-on-error: true + + - name: Run ruff linter for ckanext-hdx_search + run: ruff check ckanext-hdx_search + continue-on-error: true + + - name: Run ruff linter for ckanext-hdx_service_checker + run: ruff check ckanext-hdx_service_checker + continue-on-error: true + + - name: Run ruff linter for ckanext-hdx_theme + run: ruff check ckanext-hdx_theme + continue-on-error: true + + - name: Run ruff linter for ckanext-hdx_user_extra + run: ruff check ckanext-hdx_user_extra + continue-on-error: true + + - name: Run ruff linter for ckanext-hdx_users + run: ruff check ckanext-hdx_users + continue-on-error: true + + - name: Run ruff linter for ckanext-sitemap + run: ruff check ckanext-sitemap + continue-on-error: true + + - name: Run ruff linter for ckanext-ytp-request + run: ruff check ckanext-ytp-request + continue-on-error: true diff --git a/.github/workflows/run-types-checker.yml b/.github/workflows/run-types-checker.yml new file mode 100644 index 0000000000..27d2eca10f --- /dev/null +++ b/.github/workflows/run-types-checker.yml @@ -0,0 +1,88 @@ +name: HDX/CKAN types checker + +on: + push: + branches: [ '**' ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ dev ] + +jobs: + typecheck: + runs-on: ubuntu-latest + + steps: + + - name: Checkout code + uses: actions/checkout@v4 + + - name: Building CKAN docker image + run: docker-compose build ckan + + - name: Spinning up docker-compose stack + run: | + docker-compose up -d + sleep 10 + + - name: Installing HDX/CKAN PY3 dev-requirements + run: docker-compose exec -T ckan pip install -r /srv/ckan/dev-requirements.txt + + - name: Install Node.js and npm + run: | + docker-compose exec -T ckan bash -c " + curl -sL https://deb.nodesource.com/setup_22.x | bash - + apt-get install -y nodejs + npm install -g pyright + " + + - name: Check types for ckanext-hdx_dataviz + run: docker-compose exec -T ckan pyright /srv/ckan/ckanext-hdx_dataviz + continue-on-error: true + + - name: Check types for ckanext-hdx_hxl_preview + run: docker-compose exec -T ckan pyright /srv/ckan/ckanext-hdx_hxl_preview + continue-on-error: true + + - name: Check types for ckanext-hdx_office_preview + run: docker-compose exec -T ckan pyright /srv/ckan/ckanext-hdx_office_preview + continue-on-error: true + + - name: Check types for ckanext-hdx_org_group + run: docker-compose exec -T ckan pyright /srv/ckan/ckanext-hdx_org_group + continue-on-error: true + + - name: Check types for ckanext-hdx_package + run: docker-compose exec -T ckan pyright /srv/ckan/ckanext-hdx_package + continue-on-error: true + + - name: Check types for ckanext-hdx_pages + run: docker-compose exec -T ckan pyright /srv/ckan/ckanext-hdx_pages + continue-on-error: true + + - name: Check types for ckanext-hdx_search + run: docker-compose exec -T ckan pyright /srv/ckan/ckanext-hdx_search + continue-on-error: true + + - name: Check types for ckanext-hdx_service_checker + run: docker-compose exec -T ckan pyright /srv/ckan/ckanext-hdx_service_checker + continue-on-error: true + + - name: Check types for ckanext-hdx_theme + run: docker-compose exec -T ckan pyright /srv/ckan/ckanext-hdx_theme + continue-on-error: true + + - name: Check types for ckanext-hdx_user_extra + run: docker-compose exec -T ckan pyright /srv/ckan/ckanext-hdx_user_extra + continue-on-error: true + + - name: Check types for ckanext-hdx_users + run: docker-compose exec -T ckan pyright /srv/ckan/ckanext-hdx_users + continue-on-error: true + + - name: Check types for ckanext-sitemap + run: docker-compose exec -T ckan pyright /srv/ckan/ckanext-sitemap + continue-on-error: true + + - name: Check types for ckanext-ytp-request + run: docker-compose exec -T ckan pyright /srv/ckan/ckanext-ytp-request + continue-on-error: true diff --git a/pyproject.toml b/pyproject.toml index c59037bc97..3095e7a5c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,40 +69,41 @@ reportMissingImports = true reportMissingModuleSource = true reportMissingTypeStubs = false reportImportCycles = false -reportUnusedImport = true +reportUnusedImport = false reportUnusedClass = true reportUnusedFunction = true reportUnusedVariable = true reportDuplicateImport = true reportOptionalSubscript = true -reportOptionalMemberAccess = true +reportOptionalMemberAccess = false reportOptionalCall = true -reportOptionalIterable = true +reportOptionalIterable = false reportOptionalContextManager = true reportOptionalOperand = true -reportTypedDictNotRequiredAccess = false # We are using Context in a way that conflicts with this check +# We are using Context in a way that conflicts with this check +reportTypedDictNotRequiredAccess = false reportConstantRedefinition = true reportIncompatibleMethodOverride = false -reportIncompatibleVariableOverride = true +reportIncompatibleVariableOverride = false reportOverlappingOverload = true reportUntypedFunctionDecorator = false -reportUnknownParameterType = true +reportUnknownParameterType = false reportUnknownArgumentType = false reportUnknownLambdaType = false reportUnknownMemberType = false -reportMissingTypeArgument = true +reportMissingTypeArgument = false reportInvalidTypeVarUse = true reportCallInDefaultInitializer = true reportUnknownVariableType = false reportUntypedBaseClass = true -reportUnnecessaryIsInstance = true -reportUnnecessaryCast = true -reportUnnecessaryComparison = true +reportUnnecessaryIsInstance = false +reportUnnecessaryCast = false +reportUnnecessaryComparison = false reportAssertAlwaysTrue = true reportSelfClsParameterName = true reportUnusedCallResult = false # allow function calls for side-effect only (like logic.check_acces) useLibraryCodeForTypes = true -reportGeneralTypeIssues = true +reportGeneralTypeIssues = false reportPropertyTypeMismatch = true reportWildcardImportFromLibrary = true reportUntypedClassDecorator = false # authenticator relies on repoze.who class-decorator @@ -121,8 +122,13 @@ reportInvalidStubStatement = true reportIncompleteStub = true reportUnsupportedDunderAll = true reportUnusedCoroutine = true -reportUnnecessaryTypeIgnoreComment = true +reportUnnecessaryTypeIgnoreComment = false reportMatchNotExhaustive = true +reportArgumentType = false +reportAssignmentType = false +reportAttributeAccessIssue = false +reportInvalidTypeForm = false +reportReturnType = false [tool.mypy] plugins = "sqlalchemy.ext.mypy.plugin" @@ -149,3 +155,31 @@ module = [ "webassets.*", "zope.*", ] + +[tool.ruff] +extend-exclude = [ + "./src", +] + +# Allow lines to be as long as 120 characters. +line-length = 120 + +[tool.ruff.lint] +select = [ + "E", + "F", + "Q", + "INP001", # Checks for packages that are missing an __init__.py file. +] +extend-ignore = [ + "Q003", # avoidable-escaped-quote + "E712", # true-false-comparison (interferes with sqlAlchemy queries) +] + +[tool.ruff.lint.flake8-quotes] +inline-quotes = "single" +multiline-quotes = "single" + +[tool.ruff.format] +# Prefer single quotes over double quotes +quote-style = "single" From 0a5ec23382f85e3f83ad78cd09a46ac0ed3627c8 Mon Sep 17 00:00:00 2001 From: Catalin Date: Thu, 1 Aug 2024 19:58:12 +0300 Subject: [PATCH 2/4] HDX-9928 add documentation and ruff dev dependency --- ckanext-hdx_theme/docs/README.rst | 6 ++ .../docs/code_quality/linter.rst | 60 +++++++++++++++++++ .../docs/code_quality/type_checker.rst | 28 +++++++++ dev-requirements.in | 1 + 4 files changed, 95 insertions(+) create mode 100644 ckanext-hdx_theme/docs/code_quality/linter.rst create mode 100644 ckanext-hdx_theme/docs/code_quality/type_checker.rst diff --git a/ckanext-hdx_theme/docs/README.rst b/ckanext-hdx_theme/docs/README.rst index 927b885e88..82062d1098 100644 --- a/ckanext-hdx_theme/docs/README.rst +++ b/ckanext-hdx_theme/docs/README.rst @@ -40,3 +40,9 @@ Adding a new field to dataset. Please note that doesn't necessarily mean storing HOW TO DOWNLOAD FILES/RESOURCES FROM HDX -------------- Information about how to download the files from HDX can be found here `Download Files from HDX `_ + +CODE QUALITY +++++++++++++ +Information about configuring the linter (Ruff) can be found here: `Linter Configuration `_. + +Information about configuring the type(s) checker can be found here: `Type Checker Configuration `_. diff --git a/ckanext-hdx_theme/docs/code_quality/linter.rst b/ckanext-hdx_theme/docs/code_quality/linter.rst new file mode 100644 index 0000000000..7f3d512b82 --- /dev/null +++ b/ckanext-hdx_theme/docs/code_quality/linter.rst @@ -0,0 +1,60 @@ +Code Quality +============ + +Linter Configuration +++++++++++++++++++++ +This section outlines how to configure the linter, Ruff, for your local development environment in PyCharm. + +Installing Ruff Linter +---------------------- + +Since we utilize CKAN within a Docker container, you'll need Ruff installed locally on your development machine. + +Ruff is available as ``ruff`` on PyPI: + +:: + + pip install ruff + +Starting with version ``0.5.0``, Ruff can be installed using their standalone installers: + +:: + + # On macOS and Linux. + curl -LsSf https://astral.sh/ruff/install.sh | sh + + # On Windows. + powershell -c "irm https://astral.sh/ruff/install.ps1 | iex" + + # For a specific version. + curl -LsSf https://astral.sh/ruff/0.5.0/install.sh | sh + powershell -c "irm https://astral.sh/ruff/0.5.0/install.ps1 | iex" + +For **macOS Homebrew** and **Linuxbrew** users, Ruff is also available as ``ruff`` on Homebrew: + +:: + + brew install ruff + +Enabling Ruff Extension in PyCharm +---------------------------------- + +1. Open `File` > `Settings` (or `Preferences` on macOS) > `Plugins`. +2. Ensure the ``Marketplace`` tab is active. +3. Search for and install the ``Ruff`` extension. + +Configuring Ruff Extension in PyCharm +------------------------------------- + +1. Go to `File` > `Settings` (or `Preferences` on macOS) > `Tools` > `Ruff`. +2. Verify that only the following options are checked: + + * ``Run ruff when Reformat Code`` + * ``Show Rule Code on inspection message`` + +3. Set the Ruff executable path in the ``Global`` section if it is not already set. + +Ruff Configuration File +----------------------- + +The configuration for Ruff can be found in the project root directory in the ``pyproject.toml`` file. diff --git a/ckanext-hdx_theme/docs/code_quality/type_checker.rst b/ckanext-hdx_theme/docs/code_quality/type_checker.rst new file mode 100644 index 0000000000..6a7fd2abf3 --- /dev/null +++ b/ckanext-hdx_theme/docs/code_quality/type_checker.rst @@ -0,0 +1,28 @@ +Code Quality +============ + +Type Checker Configuration +++++++++++++++++++++++++++ +This section details enabling and configuring type checking within PyCharm. + +Enabling Type Checking in PyCharm +--------------------------------- + +1. Go to `File` > `Settings` (or `Preferences` on macOS) > `Editor` > `Inspections`. +2. Enable the following inspections: + + * Python > ``Incorrect type`` + * Python > ``Missing type hinting for function definition`` + +3. For the ``Missing type hinting for function definition`` inspection, click on it and uncheck the ``Only when types are known`` option. + +Optional Configuration +---------------------- + +1. You can adjust the severity level to ``Warning`` or ``Error`` for both inspections by clicking on them and selecting the desired level under the ``Severity`` option. + +Manually Running Type Checking Inspections +------------------------------------------ + +1. Go to `Code` > `Inspect Code`. +2. Choose the scope (e.g., `Current File` or `Whole Project`). diff --git a/dev-requirements.in b/dev-requirements.in index a3edef4ef6..78bc41c3af 100644 --- a/dev-requirements.in +++ b/dev-requirements.in @@ -26,3 +26,4 @@ pytest-rerunfailures==10.2 towncrier==22.8.0 moto==2.3.2 +ruff==0.5.5 From 5360c87cafb20efb6b38b891e9c095659d81a1e3 Mon Sep 17 00:00:00 2001 From: Catalin Date: Mon, 5 Aug 2024 18:07:47 +0300 Subject: [PATCH 3/4] HDX-9928 document running pyright in ckan container & ruff executables --- .../docs/code_quality/linter.rst | 13 +++++++++ .../docs/code_quality/type_checker.rst | 27 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/ckanext-hdx_theme/docs/code_quality/linter.rst b/ckanext-hdx_theme/docs/code_quality/linter.rst index 7f3d512b82..51e8b2b80c 100644 --- a/ckanext-hdx_theme/docs/code_quality/linter.rst +++ b/ckanext-hdx_theme/docs/code_quality/linter.rst @@ -10,6 +10,10 @@ Installing Ruff Linter Since we utilize CKAN within a Docker container, you'll need Ruff installed locally on your development machine. +The PyCharm Ruff plugin doesn't know how to use the Ruff installation from the CKAN container. It needs a Ruff executable on the Docker host (the real machine). It also cannot execute Ruff from a Python virtual environment as there doesn't seem to be a way to activate a specific virtual env before running Ruff. + +Please note that since Ruff is written in the Rust programming language, you don't necessarily need to install it as a Python package, although you can. Please see below the various options. + Ruff is available as ``ruff`` on PyPI: :: @@ -36,6 +40,15 @@ For **macOS Homebrew** and **Linuxbrew** users, Ruff is also available as ``ruff brew install ruff +Direct downloads of Ruff executables: + +- `Linux x86_64 `_ +- `Linux ARM64 `_ +- `macOS x86_64 `_ +- `macOS ARM64 `_ +- `Windows x86_64 `_ + + Enabling Ruff Extension in PyCharm ---------------------------------- diff --git a/ckanext-hdx_theme/docs/code_quality/type_checker.rst b/ckanext-hdx_theme/docs/code_quality/type_checker.rst index 6a7fd2abf3..36e802a6de 100644 --- a/ckanext-hdx_theme/docs/code_quality/type_checker.rst +++ b/ckanext-hdx_theme/docs/code_quality/type_checker.rst @@ -26,3 +26,30 @@ Manually Running Type Checking Inspections 1. Go to `Code` > `Inspect Code`. 2. Choose the scope (e.g., `Current File` or `Whole Project`). + +Running Pyright in the CKAN Container +----------------------------------------------------------- +To run Pyright manually within the CKAN Docker container, follow these steps: + +1. Install Node.js and npm on the CKAN Docker container: + +:: + + # Enter the CKAN Docker container + docker-compose exec -it ckan /bin/bash + + # Install Node + curl -sL https://deb.nodesource.com/setup_22.x | bash - + apt-get install -y nodejs + +2. Install Pyright globally using ``npm``: + +:: + + npm install -g pyright + +3. To run Pyright within the CKAN Docker container, execute the following command: + +:: + + docker-compose exec -T ckan pyright /srv/ckan/ From cd1777d6ea9df8b6176095947132fd7e61e7e9ec Mon Sep 17 00:00:00 2001 From: Alexandru Gartner Date: Mon, 5 Aug 2024 21:09:35 +0000 Subject: [PATCH 4/4] HDX-10066 replace docker-compose v1 with v2 for type checker --- .github/workflows/run-types-checker.yml | 36 ++++++++++++------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/run-types-checker.yml b/.github/workflows/run-types-checker.yml index 27d2eca10f..b4d7f4ffaa 100644 --- a/.github/workflows/run-types-checker.yml +++ b/.github/workflows/run-types-checker.yml @@ -17,72 +17,72 @@ jobs: uses: actions/checkout@v4 - name: Building CKAN docker image - run: docker-compose build ckan + run: docker compose build ckan - - name: Spinning up docker-compose stack + - name: Spinning up docker compose stack run: | - docker-compose up -d + docker compose up -d sleep 10 - name: Installing HDX/CKAN PY3 dev-requirements - run: docker-compose exec -T ckan pip install -r /srv/ckan/dev-requirements.txt + run: docker compose exec -T ckan pip install -r /srv/ckan/dev-requirements.txt - name: Install Node.js and npm run: | - docker-compose exec -T ckan bash -c " + docker compose exec -T ckan bash -c " curl -sL https://deb.nodesource.com/setup_22.x | bash - apt-get install -y nodejs npm install -g pyright " - name: Check types for ckanext-hdx_dataviz - run: docker-compose exec -T ckan pyright /srv/ckan/ckanext-hdx_dataviz + run: docker compose exec -T ckan pyright /srv/ckan/ckanext-hdx_dataviz continue-on-error: true - name: Check types for ckanext-hdx_hxl_preview - run: docker-compose exec -T ckan pyright /srv/ckan/ckanext-hdx_hxl_preview + run: docker compose exec -T ckan pyright /srv/ckan/ckanext-hdx_hxl_preview continue-on-error: true - name: Check types for ckanext-hdx_office_preview - run: docker-compose exec -T ckan pyright /srv/ckan/ckanext-hdx_office_preview + run: docker compose exec -T ckan pyright /srv/ckan/ckanext-hdx_office_preview continue-on-error: true - name: Check types for ckanext-hdx_org_group - run: docker-compose exec -T ckan pyright /srv/ckan/ckanext-hdx_org_group + run: docker compose exec -T ckan pyright /srv/ckan/ckanext-hdx_org_group continue-on-error: true - name: Check types for ckanext-hdx_package - run: docker-compose exec -T ckan pyright /srv/ckan/ckanext-hdx_package + run: docker compose exec -T ckan pyright /srv/ckan/ckanext-hdx_package continue-on-error: true - name: Check types for ckanext-hdx_pages - run: docker-compose exec -T ckan pyright /srv/ckan/ckanext-hdx_pages + run: docker compose exec -T ckan pyright /srv/ckan/ckanext-hdx_pages continue-on-error: true - name: Check types for ckanext-hdx_search - run: docker-compose exec -T ckan pyright /srv/ckan/ckanext-hdx_search + run: docker compose exec -T ckan pyright /srv/ckan/ckanext-hdx_search continue-on-error: true - name: Check types for ckanext-hdx_service_checker - run: docker-compose exec -T ckan pyright /srv/ckan/ckanext-hdx_service_checker + run: docker compose exec -T ckan pyright /srv/ckan/ckanext-hdx_service_checker continue-on-error: true - name: Check types for ckanext-hdx_theme - run: docker-compose exec -T ckan pyright /srv/ckan/ckanext-hdx_theme + run: docker compose exec -T ckan pyright /srv/ckan/ckanext-hdx_theme continue-on-error: true - name: Check types for ckanext-hdx_user_extra - run: docker-compose exec -T ckan pyright /srv/ckan/ckanext-hdx_user_extra + run: docker compose exec -T ckan pyright /srv/ckan/ckanext-hdx_user_extra continue-on-error: true - name: Check types for ckanext-hdx_users - run: docker-compose exec -T ckan pyright /srv/ckan/ckanext-hdx_users + run: docker compose exec -T ckan pyright /srv/ckan/ckanext-hdx_users continue-on-error: true - name: Check types for ckanext-sitemap - run: docker-compose exec -T ckan pyright /srv/ckan/ckanext-sitemap + run: docker compose exec -T ckan pyright /srv/ckan/ckanext-sitemap continue-on-error: true - name: Check types for ckanext-ytp-request - run: docker-compose exec -T ckan pyright /srv/ckan/ckanext-ytp-request + run: docker compose exec -T ckan pyright /srv/ckan/ckanext-ytp-request continue-on-error: true