Run yapf, flake8, isort as part of pre-commit (#1859)

Closes #1702 ## By Submitting this PR I confirm: - I am familiar with the [Contributing Guidelines](https://github.com/nv-morpheus/Morpheus/blob/main/docs/source/developer_guide/contributing.md). - When the PR is ready for review, new or existing tests cover these changes. - When the PR is ready for review, the documentation is up to date with these changes. Authors: - Christopher Harris (https://github.com/cwharris) Approvers: - Anuradha Karuppiah (https://github.com/AnuradhaKaruppiah) URL: #1859
nv-morpheus · Aug 30, 2024 · 5a17b92 · 5a17b92
1 parent 8235316
commit 5a17b92
Show file tree

Hide file tree

Showing 13 changed files with 137 additions and 155 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -19,6 +19,26 @@ repos:
         hooks:
             - id: rapids-dependency-file-generator
               args: ["--clean"]
+      - repo: https://github.com/PyCQA/isort
+        rev: 5.12.0
+        hooks:
+              - id: isort
+                args: ["--settings-file=./pyproject.toml"]
+                files: ^python/
+      - repo: https://github.com/PyCQA/flake8
+        rev: 6.1.0
+        hooks:
+              - id: flake8
+                entry: pflake8
+                additional_dependencies: [pyproject-flake8]
+                args: ["--config=./pyproject.toml"]
+                files: ^python/
+      - repo: https://github.com/google/yapf
+        rev: v0.40.2
+        hooks:
+              - id: yapf
+                args: ["--style", "./pyproject.toml"]
+                files: ^python/
 
 default_language_version:
       python: python3
diff --git a/ci/scripts/fix_all.sh b/ci/scripts/fix_all.sh
@@ -105,5 +105,5 @@ fi
 # Run yapf
 if [[ "${SKIP_YAPF}" == "" ]]; then
    echo "Running yapf..."
-   python3 -m yapf -i --style ${PY_CFG} -r ${PY_MODIFIED_FILES[@]}
+   python3 -m yapf -i --style ${PROJ_TOML} -r ${PY_MODIFIED_FILES[@]}
 fi
diff --git a/ci/scripts/python_checks.sh b/ci/scripts/python_checks.sh
@@ -26,10 +26,7 @@ LC_ALL=C.UTF-8
 LANG=C.UTF-8
 
 # Pre-populate the return values in case they are skipped
-ISORT_RETVAL=0
-FLAKE_RETVAL=0
 PYLINT_RETVAL=0
-YAPF_RETVAL=0
 
 get_modified_files ${PYTHON_FILE_REGEX} MORPHEUS_MODIFIED_FILES
 
@@ -41,45 +38,16 @@ if [[ -n "${MORPHEUS_MODIFIED_FILES}" ]]; then
       echo "  $f"
    done
 
-   if [[ "${SKIP_ISORT}" == "" ]]; then
-      ISORT_OUTPUT=`python3 -m isort --settings-file ${PROJ_TOML} --filter-files --check-only  ${MORPHEUS_MODIFIED_FILES[@]} 2>&1`
-      ISORT_RETVAL=$?
-   fi
-
    if [[ "${SKIP_PYLINT}" == "" ]]; then
       NUM_PROC=$(get_num_proc)
       PYLINT_OUTPUT=`pylint -j ${NUM_PROC} ${MORPHEUS_MODIFIED_FILES[@]} 2>&1`
       PYLINT_RETVAL=$?
    fi
 
-   if [[ "${SKIP_FLAKE}" == "" ]]; then
-      FLAKE_OUTPUT=`python3 -m flake8 --config ${PY_CFG} ${MORPHEUS_MODIFIED_FILES[@]} 2>&1`
-      FLAKE_RETVAL=$?
-   fi
-
-   if [[ "${SKIP_YAPF}" == "" ]]; then
-      # Run yapf. Will return 1 if there are any diffs
-      YAPF_OUTPUT=`python3 -m yapf --style ${PY_CFG} --diff ${MORPHEUS_MODIFIED_FILES[@]} 2>&1`
-      YAPF_RETVAL=$?
-   fi
-
 else
    echo "No modified Python files to check"
 fi
 
-# Output results if failure otherwise show pass
-if [[ "${SKIP_ISORT}" != "" ]]; then
-   echo -e "\n\n>>>> SKIPPED: isort check\n\n"
-elif [ "${ISORT_RETVAL}" != "0" ]; then
-   echo -e "\n\n>>>> FAILED: isort style check; begin output\n\n"
-   echo -e "${ISORT_OUTPUT}"
-   echo -e "\n\n>>>> FAILED: isort style check; end output\n\n" \
-           "To auto-fix many issues (not all) run:\n" \
-           "   ./ci/scripts/fix_all.sh\n\n"
-else
-  echo -e "\n\n>>>> PASSED: isort style check\n\n"
-fi
-
 if [[ "${SKIP_PYLINT}" != "" ]]; then
    echo -e "\n\n>>>> SKIPPED: pylint check\n\n"
 elif [ "${PYLINT_RETVAL}" != "0" ]; then
@@ -90,33 +58,7 @@ else
   echo -e "\n\n>>>> PASSED: pylint style check\n\n"
 fi
 
-if [[ "${SKIP_FLAKE}" != "" ]]; then
-   echo -e "\n\n>>>> SKIPPED: flake8 check\n\n"
-elif [ "${FLAKE_RETVAL}" != "0" ]; then
-   echo -e "\n\n>>>> FAILED: flake8 style check; begin output\n\n"
-   echo -e "${FLAKE_OUTPUT}"
-   echo -e "\n\n>>>> FAILED: flake8 style check; end output\n\n" \
-           "To auto-fix many issues (not all) run:\n" \
-           "   ./ci/scripts/fix_all.sh\n\n"
-else
-  echo -e "\n\n>>>> PASSED: flake8 style check\n\n"
-fi
-
-if [[ "${SKIP_YAPF}" != "" ]]; then
-   echo -e "\n\n>>>> SKIPPED: yapf check\n\n"
-elif [ "${YAPF_RETVAL}" != "0" ]; then
-   echo -e "\n\n>>>> FAILED: yapf style check; begin output\n\n"
-   echo -e "Incorrectly formatted files:"
-   YAPF_OUTPUT=`echo "${YAPF_OUTPUT}" | sed -nr 's/^\+\+\+ ([^ ]*) *\(reformatted\)$/\1/p'`
-   echo -e "${YAPF_OUTPUT}"
-   echo -e "\n\n>>>> FAILED: yapf style check; end output\n\n" \
-           "To auto-fix many issues (not all) run:\n" \
-           "   ./ci/scripts/fix_all.sh\n\n"
-else
-  echo -e "\n\n>>>> PASSED: yapf style check\n\n"
-fi
-
-RETVALS=(${ISORT_RETVAL} ${FLAKE_RETVAL} ${PYLINT_RETVAL} ${YAPF_RETVAL})
+RETVALS=(${PYLINT_RETVAL})
 IFS=$'\n'
 RETVAL=`echo "${RETVALS[*]}" | sort -nr | head -n1`
 

diff --git a/morpheus.code-workspace b/morpheus.code-workspace
@@ -679,7 +679,7 @@
         "files.trimFinalNewlines": true,
         "files.trimTrailingWhitespace": true,
         "flake8.args": [
-            "--style=${workspaceFolder}/python/morpheus/setup.cfg"
+            "--style=${workspaceFolder}/pyproject.toml"
         ],
         "pylint.args": [
             "--rcfile=${workspaceFolder}/pyproject.toml",
@@ -730,7 +730,7 @@
             }
         ],
         "yapf.args": [
-            "--style=${workspaceFolder}/python/morpheus/setup.cfg"
+            "--style=${workspaceFolder}/pyproject.toml"
         ]
     }
 }
diff --git a/pyproject.toml b/pyproject.toml
@@ -45,13 +45,6 @@ addopts = "--benchmark-disable"
 
 asyncio_mode = "auto"
 
-[tool.yapfignore]
-ignore_patterns = [
-  "**/*.pyx",
-  "**/*.pxd",
-  "**/_version.py",
-]
-
 [tool.mypy]
 # Allow None for argument default values
 implicit_optional = true
@@ -753,3 +746,81 @@ skip= [
     "models",
     "thirdparty"
 ]
+
+[tool.flake8]
+filename=[
+  "*.py",
+  "*.pyx",
+  "*.pxd"
+]
+# TODO: Remove dfencoder https://github.com/nv-morpheus/Morpheus/issues/786
+exclude=[
+   "__pycache__",
+   ".git",
+   ".tmp/",
+   "*.egg",
+   "build/",
+   "cpp",
+   "docs",
+   "models/",
+   "python/morpheus/morpheus/models/dfencoder/*.py",
+   "thirdparty"
+]
+max-line-length=120
+max-doc-length=120
+
+extend-ignore=[
+    # Ignore missing docstrings __init__ methods as we document those on the class
+    "D107",
+    # Don't require quotes to be placed on the same line as a one-line docstring, useful when the docstring is close
+    # to the line limit
+    "D200",
+    # Allow a blank line between a docstring and the code
+    "D202",
+    # D204: 1 blank line required after class docstring
+    "D204",
+    # D205: 1 blank line required between summary line and description (allow for a paragraph)
+    "D205",
+    # D400: First line should end with a period (only works if we're adhering to D205)
+    "D400",
+    # D401: First line should be in imperative mood
+    "D401"
+]
+
+# Cython Rules ignored:
+# E999: invalid syntax (works for Python, not Cython)
+# E225: Missing whitespace around operators (breaks cython casting syntax like <int>)
+# E226: Missing whitespace around arithmetic operators (breaks cython pointer syntax like int*)
+# E227: Missing whitespace around bitwise or shift operator (Can also break casting syntax)
+# W503: line break before binary operator (breaks lines that start with a pointer)
+# W504: line break after binary operator (breaks lines that end with a pointer)
+
+per-file-ignores =[
+    # imported but unused
+    "__init__.py: F401, E402",
+    # Ignore additional deps needed for examples
+    "examples/*.py: F821",
+    # Cython Exclusions
+    "*.pyx: E999, E225, E226, E227, W503, W504",
+    "*.pxd: E999, E225, E226, E227, W503, W504",
+    # Remove some documentation requirements for tests
+    # D100: Missing docstring in public module
+    # D101: Missing docstring in public class
+    # D102: Missing docstring in public method
+    # D103: Missing docstring in public function
+    "tests/*.py: D100, D101, D102, D103"
+]
+
+[tool.yapfignore]
+ignore_patterns = [
+  "**/*.pyx",
+  "**/*.pxd",
+  "**/_version.py"
+]
+
+[tool.yapf]
+based_on_style="pep8"
+column_limit=120
+split_all_top_level_comma_separated_values=true
+join_multiple_lines=true
+indent_dictionary_value=true
diff --git a/python/morpheus/morpheus/_lib/common/__init__.pyi b/python/morpheus/morpheus/_lib/common/__init__.pyi
@@ -5,9 +5,11 @@
            :toctree: _generate
         """
 from __future__ import annotations
-import morpheus._lib.common
-import typing
+
 import os
+import typing
+
+import morpheus._lib.common
 
 __all__ = [
     "FiberQueue",

diff --git a/python/morpheus/morpheus/_lib/cudf_helpers/__init__.pyi b/python/morpheus/morpheus/_lib/cudf_helpers/__init__.pyi
@@ -1,8 +1,11 @@
 from __future__ import annotations
-import morpheus._lib.cudf_helpers
+
 import typing
-from cudf.core.dtypes import StructDtype
+
 import cudf
+from cudf.core.dtypes import StructDtype
+
+import morpheus._lib.cudf_helpers
 
 __all__ = [
     "StructDtype",

diff --git a/python/morpheus/morpheus/_lib/doca/__init__.pyi b/python/morpheus/morpheus/_lib/doca/__init__.pyi
@@ -1,10 +1,13 @@
 from __future__ import annotations
-import morpheus._lib.doca
-import typing
+
 import datetime
-import morpheus._lib.messages
+import typing
+
 import mrc.core.segment
 
+import morpheus._lib.doca
+import morpheus._lib.messages
+
 __all__ = [
     "DocaConvertStage",
     "DocaSourceStage"

diff --git a/python/morpheus/morpheus/_lib/messages/__init__.pyi b/python/morpheus/morpheus/_lib/messages/__init__.pyi
@@ -6,12 +6,15 @@
 
         """
 from __future__ import annotations
-import morpheus._lib.messages
+
 import typing
+
 import cupy
-import morpheus._lib.common
 import mrc.core.node
 
+import morpheus._lib.common
+import morpheus._lib.messages
+
 __all__ = [
     "ControlMessage",
     "ControlMessageType",

diff --git a/python/morpheus/morpheus/_lib/modules/__init__.pyi b/python/morpheus/morpheus/_lib/modules/__init__.pyi
@@ -6,9 +6,11 @@
 
         """
 from __future__ import annotations
-import morpheus._lib.modules
+
 import typing
 
+import morpheus._lib.modules
+
 __all__ = [
 
 ]

diff --git a/python/morpheus/morpheus/_lib/stages/__init__.pyi b/python/morpheus/morpheus/_lib/stages/__init__.pyi
@@ -6,13 +6,16 @@
 
         """
 from __future__ import annotations
-import morpheus._lib.stages
+
+import os
 import typing
-from morpheus._lib.common import FilterSource
-import morpheus._lib.common
+
 import mrc.core.coro
 import mrc.core.segment
-import os
+
+import morpheus._lib.common
+import morpheus._lib.stages
+from morpheus._lib.common import FilterSource
 
 __all__ = [
     "AddClassificationsControlMessageStage",