integrate ruff (removing bandit, flake8, black and pylint)

CHANGELOG.md

@@ -8,15 +8,14 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/) and this p
 
 ### Changed
 
+- integrate ruff (removing bandit, flake8, black and pylint)
 - dependency to plugin base 4.3.0 (cmem-cmempy >= 23.3)
 - use poetry-dynamic-versioning option bump=true
   - 0.0.1.devX.. instead of 0.0.0.postX
-- add target-version = "py311" to black
 
 ### Fixed
 
 - race condition in deploy task (#19)
-- pylint task ignored the test folder (the other linters did)
 
 
 ## [5.3.4] 2023-11-06

src/.github/workflows/check.yml

@@ -42,21 +42,13 @@ jobs:
         run: |
           poetry self add "poetry-dynamic-versioning[plugin]"
 
-      - name: bandit
-        run: |
-          task check:bandit
-
-      - name: flake8
-        run: |
-          task check:flake8
-
       - name: mypy
         run: |
           task check:mypy
 
-      - name: pylint
+      - name: ruff
         run: |
-          task check:pylint
+          task check:ruff
 
       - name: pytest
         env:

src/.gitlab-ci.yml

@@ -19,25 +19,15 @@ stages:
   - build
   - publish
 
-bandit:
+ruff:
   stage: test
   script:
-    - task check:bandit
+    - task check:ruff
   artifacts:
     when: always
     reports:
       junit:
-        - dist/junit-bandit.xml
-
-flake8:
-  stage: test
-  script:
-    - task check:flake8
-  artifacts:
-    when: always
-    reports:
-      junit:
-        - dist/junit-flake8.xml
+        - dist/junit-ruff.xml
 
 mypy:
   stage: test
@@ -86,7 +76,6 @@ safety:
 build:
   stage: build
   needs:
-    - bandit
     - mypy
     - pytest
     - safety
@@ -102,7 +91,7 @@ pypi:
   # publishing only available on a tag
   stage: publish
   needs:
-    - flake8
+    - ruff
     - pylint
     - build
   allow_failure: true

src/Taskfile.yaml

@@ -66,11 +66,19 @@ tasks:
     cmds:
       - poetry install
 
-  python:format:
+  format:fix:
     desc: Format Python files
     <<: *preparation
     cmds:
-      - poetry run black .
+      - poetry run ruff format tests {{.PACKAGE}}
+      - poetry run ruff check tests {{.PACKAGE}} --fix-only
+
+  format:fix-unsafe:
+    desc: Format Python files
+    <<: *preparation
+    cmds:
+      - poetry run ruff format tests {{.PACKAGE}}
+      - poetry run ruff check tests {{.PACKAGE}} --fix-only --unsafe-fixes
 
   clean:
     desc: Removes dist, *.pyc and some caches
@@ -83,18 +91,16 @@ tasks:
 
   check:
     desc: Run whole test suite incl. unit and integration tests
-    deps:
-      - check:linters
-      - check:pytest
+    cmds:
+      - task: check:linters
+      - task: check:pytest
 
   check:linters:
     desc: Run all linter and static code analysis tests
-    deps:
-      - check:bandit
-      - check:flake8
-      - check:mypy
-      - check:pylint
-      - check:safety
+    cmds:
+      - task: check:ruff
+      - task: check:mypy
+      - task: check:safety
 
   check:pytest:
     desc: Run unit and integration tests
@@ -125,51 +131,32 @@ tasks:
       BADGE_COVERAGE: ./{{.DIST_DIR}}/badge-coverage.svg
       BADGE_TESTS: ./{{.DIST_DIR}}/badge-tests.svg
 
-  check:pylint:
-    desc: Find code smells, errors and style issues
-    <<: *preparation
-    cmds:
-      - poetry run pylint --exit-zero tests {{.PACKAGE}}
-      - poetry run pylint tests {{.PACKAGE}} {{.XML_PARAMS}}
-    vars:
-      FORMAT: --output-format=pylint_junit.JUnitReporter
-      JUNIT_FILE: ./{{.DIST_DIR}}/junit-pylint.xml
-      XML_PARAMS: --output={{.JUNIT_FILE}} {{.FORMAT}}
-
   check:mypy:
-    desc: Find type errors
+    desc: Complain about typing errors
     <<: *preparation
     cmds:
       - poetry run mypy -p tests -p {{.PACKAGE}} --junit-xml {{.JUNIT_FILE}}
     vars:
       JUNIT_FILE: ./{{.DIST_DIR}}/junit-mypy.xml
 
   check:safety:
-    desc: Scan dependencies for vulnerabilities
+    desc: Complain about vulnerabilities in dependencies
     <<: *preparation
     cmds:
       # ignore 51358 safety - dev dependency only
       # ignore 61489 pillow - dev dependency only
       - poetry run safety check -i 51358 -i 61489
 
-  check:bandit:
-    desc: Find common security issues
-    <<: *preparation
-    cmds:
-      - poetry run bandit --exit-zero -r {{.PACKAGE}}
-      - poetry run bandit --format xml -r {{.PACKAGE}} -o {{.JUNIT_FILE}}
-    vars:
-      JUNIT_FILE: ./{{.DIST_DIR}}/junit-bandit.xml
-
-  check:flake8:
-    desc: Enforce standard source code style guide
+  check:ruff:
+    desc: Complain about everything else
     <<: *preparation
     cmds:
-      - poetry run flake8 --exit-zero tests {{.PACKAGE}} {{.XML_PARAMS}}
-      - poetry run flake8 --show-source tests {{.PACKAGE}}
+      - poetry run ruff check --show-source tests {{.PACKAGE}}
+      - poetry run ruff check --exit-zero tests {{.PACKAGE}} {{.XML_PARAMS}}
+      - poetry run ruff format --check tests {{.PACKAGE}}
     vars:
-      JUNIT_FILE: ./{{.DIST_DIR}}/junit-flake8.xml
-      XML_PARAMS: --format junit-xml --output-file {{.JUNIT_FILE}}
+      JUNIT_FILE: ./{{.DIST_DIR}}/junit-ruff.xml
+      XML_PARAMS: --output-format junit --output-file {{.JUNIT_FILE}}
 
   # }}}
   # {{{ build and deploy tasks

src/cmem_plugin_{{ project_slug }}/__init__.py.jinja

@@ -0,0 +1 @@
+"""{{ project_slug }} - main package"""

src/cmem_plugin_{{ project_slug }}/transform/__init__.py.jinja → src/cmem_plugin_{{ project_slug }}/example_transform.py

@@ -1,7 +1,7 @@
 """lifetime(age) transform plugin module"""
 import datetime
+from collections.abc import Sequence
 from datetime import date
-from typing import Sequence
 
 from cmem_plugin_base.dataintegration.description import (
     Plugin,
@@ -11,8 +11,7 @@
 
 
 @Plugin(
-    label="Lifetime (age - {{ project_slug }})",
-    plugin_id="Example-Lifetime-{{ project_slug }}",
+    label="Lifetime (age - example from template)",
     description="From the input date,"
     "the value gets transformed into number of years (age)."
     " Supports only xsd:date(YYYY-MM-DD) format.",
@@ -48,6 +47,7 @@ def __init__(self, start_date: str):
         self.start_date = start_date
 
     def transform(self, inputs: Sequence[Sequence[str]]) -> Sequence[str]:
+        """Do the actual transformation of values"""
         result = []
         if len(inputs) != 0:
             for collection in inputs:
@@ -57,9 +57,9 @@ def transform(self, inputs: Sequence[Sequence[str]]) -> Sequence[str]:
         return result
 
     def _calculate_age(self, value: str) -> int:
-        """calculate age in years"""
-        today = date.today()
-        born = datetime.datetime.strptime(value, self.DATE_FORMAT).date()
+        """Calculate age in years"""
+        today = date.today()  # noqa: DTZ011
+        born = datetime.datetime.strptime(value, self.DATE_FORMAT).date()  # noqa: DTZ007
         try:
             birthday = born.replace(year=today.year)
 

src/cmem_plugin_{{ project_slug }}/workflow/__init__.py.jinja → src/cmem_plugin_{{ project_slug }}/example_workflow.py

@@ -1,25 +1,21 @@
 """Random values workflow plugin module"""
 import uuid
+from collections.abc import Sequence
 from secrets import token_urlsafe
-from typing import Sequence
 
 from cmem_plugin_base.dataintegration.context import ExecutionContext, ExecutionReport
-from cmem_plugin_base.dataintegration.description import (
-    Icon,
-    Plugin,
-    PluginParameter
-)
+from cmem_plugin_base.dataintegration.description import Icon, Plugin, PluginParameter
 from cmem_plugin_base.dataintegration.entity import (
     Entities,
     Entity,
-    EntitySchema,
     EntityPath,
+    EntitySchema,
 )
 from cmem_plugin_base.dataintegration.plugins import WorkflowPlugin
 
 
 @Plugin(
-    label="Random Values ({{ project_slug }})",
+    label="Random Values (example from template)",
     description="Generates random values of X rows a Y values.",
     documentation="""
 This example workflow operator generates random values.
@@ -29,7 +25,7 @@
 - 'number_of_entities': How many rows do you need.
 - 'number_of_values': How many values per row do you need.
 """,
-    icon=Icon(package=__package__, file_name="custom.svg"),
+    icon=Icon(package=__package__, file_name="example_icon.svg"),
     parameters=[
         PluginParameter(
             name="number_of_entities",
@@ -59,14 +55,17 @@ def __init__(self, number_of_entities: int = 10, number_of_values: int = 5) -> N
         self.number_of_values = number_of_values
 
     def execute(
-        self, inputs: Sequence[Entities], context: ExecutionContext
+        self,
+        inputs: Sequence[Entities],  # noqa: ARG002
+        context: ExecutionContext,
     ) -> Entities:
+        """Run the workflow operator."""
         self.log.info("Start creating random values.")
         self.log.info(f"Config length: {len(self.config.get())}")
         value_counter = 0
         entities = []
         for _ in range(self.number_of_entities):
-            entity_uri = f"urn:uuid:{str(uuid.uuid4())}"
+            entity_uri = f"urn:uuid:{uuid.uuid4()!s}"
             values = []
             for _ in range(self.number_of_values):
                 values.append([token_urlsafe(16)])

src/pyproject.toml.jinja

@@ -24,26 +24,14 @@ version = "^4.3.0"
 allow-prereleases = false
 
 [tool.poetry.group.dev.dependencies]
-bandit = "^1.7.5"
-black = "^23.3.0"
-coverage = "^7.2.3"
-defusedxml = "^0.7.1"
-flake8-formatter-junit-xml = "^0.0.6"
-# https://github.com/smarie/python-genbadge/issues/31
-pillow = "^9.5.0"
-genbadge = "^1.1.0"
+genbadge = {extras = ["coverage"], version = "^1.1.1"}
 mypy = "^1.2.0"
-# Avoid safety issue 62044 for pip less than 23.3
-pip = ">=23.3"
-# https://github.com/rasjani/pylint-junit/issues/1
-pylint = "^2"
-pylint-junit = "^0.3.2"
+pip = ">=23.3" # Avoid safety issue 62044 for pip less than 23.3
 pytest = "^7.3.1"
-pytest-cov = "^4.0.0"
-pytest-memray = { version = "^1.4.0",  markers = "platform_system != 'Windows'" }
+pytest-cov = "^4.1.0"
+pytest-memray = { version = "^1.5.0",  markers = "platform_system != 'Windows'" }
+ruff = "^0.1.5"
 safety = "^1.10.3"
-typed-ast = "^1.5.4"
-wheel = "^0.38.4"
 
 [build-system]
 requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning"]
@@ -55,30 +43,36 @@ vcs = "git"
 dirty = true
 bump = true
 
-[tool.pylint.MASTER]
-load-plugins="pylint_junit"
-
-[tool.pylint.General]
-ignore = "version.py"
-
-[tool.pylint.'MESSAGES CONTROL']
-extension-pkg-whitelist = "pydantic"
-disable = "fixme"
-
-# https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html#pylint
-[tool.pylint.messages_control]
-disable = "R0903"
-
-[tool.pylint.format]
-max-line-length = "88"
-
 [tool.mypy]
 warn_return_any = true
 ignore_missing_imports = true
 
 [tool.pytest.ini_options]
 addopts = ""
 
-[tool.black]
+[tool.ruff]
+line-length = 100
 target-version = "py311"
 
+[tool.ruff.format]
+line-ending = "lf" # Use `\n` line endings for all files
+
+[tool.ruff.lint]
+select = ["ALL"]
+ignore = [
+    "ANN101",  # Missing type annotation for self in method
+    "ANN204",  # Missing return type annotation for special method `__init__`
+    "COM812",  # missing-trailing-comma
+    "D107",    # Missing docstring in __init__
+    "D203",    # [*] 1 blank line required before class docstring
+    "D211",    # No blank lines allowed before class docstring
+    "D213",    # Multi-line docstring summary should start at the second line
+    "D400",    # First line should end with a period
+    "D415",    # First line should end with a period, question mark, or exclamation point
+    "EM101",   # Exception must not use a string literal, assign to variable first
+    "G004",    # Logging statement uses f-string
+    "ISC001",  # single-line-implicit-string-concatenation
+    "S101",    # use of assert detected
+    "TRY003",  # Avoid specifying long messages outside the exception class
+]
+

src/tests/__init__.py

@@ -0,0 +1 @@
+"""tests"""