Run yapf, flake8, isort as part of pre-commit

.pre-commit-config.yaml

@@ -19,6 +19,26 @@ repos:
         hooks:
             - id: rapids-dependency-file-generator
               args: ["--clean"]
+      - repo: https://github.com/PyCQA/isort
+        rev: 5.12.0
+        hooks:
+              - id: isort
+                args: ["--settings-file=./pyproject.toml"]
+                files: ^python/
+      - repo: https://github.com/PyCQA/flake8
+        rev: 6.1.0
+        hooks:
+              - id: flake8
+                entry: pflake8
+                additional_dependencies: [pyproject-flake8]
+                args: ["--config=./pyproject.toml"]
+                files: ^python/
+      - repo: https://github.com/google/yapf
+        rev: v0.40.2
+        hooks:
+              - id: yapf
+                args: ["--style", "./pyproject.toml"]
+                files: ^python/
 
 default_language_version:
       python: python3

ci/scripts/python_checks.sh

@@ -26,10 +26,7 @@ LC_ALL=C.UTF-8
 LANG=C.UTF-8
 
 # Pre-populate the return values in case they are skipped
-ISORT_RETVAL=0
-FLAKE_RETVAL=0
 PYLINT_RETVAL=0
-YAPF_RETVAL=0
 
 get_modified_files ${PYTHON_FILE_REGEX} MORPHEUS_MODIFIED_FILES
 
@@ -41,45 +38,16 @@ if [[ -n "${MORPHEUS_MODIFIED_FILES}" ]]; then
       echo "  $f"
    done
 
-   if [[ "${SKIP_ISORT}" == "" ]]; then
-      ISORT_OUTPUT=`python3 -m isort --settings-file ${PROJ_TOML} --filter-files --check-only  ${MORPHEUS_MODIFIED_FILES[@]} 2>&1`
-      ISORT_RETVAL=$?
-   fi
-
    if [[ "${SKIP_PYLINT}" == "" ]]; then
       NUM_PROC=$(get_num_proc)
       PYLINT_OUTPUT=`pylint -j ${NUM_PROC} ${MORPHEUS_MODIFIED_FILES[@]} 2>&1`
       PYLINT_RETVAL=$?
    fi
 
-   if [[ "${SKIP_FLAKE}" == "" ]]; then
-      FLAKE_OUTPUT=`python3 -m flake8 --config ${PY_CFG} ${MORPHEUS_MODIFIED_FILES[@]} 2>&1`
-      FLAKE_RETVAL=$?
-   fi
-
-   if [[ "${SKIP_YAPF}" == "" ]]; then
-      # Run yapf. Will return 1 if there are any diffs
-      YAPF_OUTPUT=`python3 -m yapf --style ${PY_CFG} --diff ${MORPHEUS_MODIFIED_FILES[@]} 2>&1`
-      YAPF_RETVAL=$?
-   fi
-
 else
    echo "No modified Python files to check"
 fi
 
-# Output results if failure otherwise show pass
-if [[ "${SKIP_ISORT}" != "" ]]; then
-   echo -e "\n\n>>>> SKIPPED: isort check\n\n"
-elif [ "${ISORT_RETVAL}" != "0" ]; then
-   echo -e "\n\n>>>> FAILED: isort style check; begin output\n\n"
-   echo -e "${ISORT_OUTPUT}"
-   echo -e "\n\n>>>> FAILED: isort style check; end output\n\n" \
-           "To auto-fix many issues (not all) run:\n" \
-           "   ./ci/scripts/fix_all.sh\n\n"
-else
-  echo -e "\n\n>>>> PASSED: isort style check\n\n"
-fi
-
 if [[ "${SKIP_PYLINT}" != "" ]]; then
    echo -e "\n\n>>>> SKIPPED: pylint check\n\n"
 elif [ "${PYLINT_RETVAL}" != "0" ]; then
@@ -90,33 +58,7 @@ else
   echo -e "\n\n>>>> PASSED: pylint style check\n\n"
 fi
 
-if [[ "${SKIP_FLAKE}" != "" ]]; then
-   echo -e "\n\n>>>> SKIPPED: flake8 check\n\n"
-elif [ "${FLAKE_RETVAL}" != "0" ]; then
-   echo -e "\n\n>>>> FAILED: flake8 style check; begin output\n\n"
-   echo -e "${FLAKE_OUTPUT}"
-   echo -e "\n\n>>>> FAILED: flake8 style check; end output\n\n" \
-           "To auto-fix many issues (not all) run:\n" \
-           "   ./ci/scripts/fix_all.sh\n\n"
-else
-  echo -e "\n\n>>>> PASSED: flake8 style check\n\n"
-fi
-
-if [[ "${SKIP_YAPF}" != "" ]]; then
-   echo -e "\n\n>>>> SKIPPED: yapf check\n\n"
-elif [ "${YAPF_RETVAL}" != "0" ]; then
-   echo -e "\n\n>>>> FAILED: yapf style check; begin output\n\n"
-   echo -e "Incorrectly formatted files:"
-   YAPF_OUTPUT=`echo "${YAPF_OUTPUT}" | sed -nr 's/^\+\+\+ ([^ ]*) *\(reformatted\)$/\1/p'`
-   echo -e "${YAPF_OUTPUT}"
-   echo -e "\n\n>>>> FAILED: yapf style check; end output\n\n" \
-           "To auto-fix many issues (not all) run:\n" \
-           "   ./ci/scripts/fix_all.sh\n\n"
-else
-  echo -e "\n\n>>>> PASSED: yapf style check\n\n"
-fi
-
-RETVALS=(${ISORT_RETVAL} ${FLAKE_RETVAL} ${PYLINT_RETVAL} ${YAPF_RETVAL})
+RETVALS=(${PYLINT_RETVAL})
 IFS=$'\n'
 RETVAL=`echo "${RETVALS[*]}" | sort -nr | head -n1`
 

pyproject.toml

@@ -45,13 +45,6 @@ addopts = "--benchmark-disable"
 
 asyncio_mode = "auto"
 
-[tool.yapfignore]
-ignore_patterns = [
-  "**/*.pyx",
-  "**/*.pxd",
-  "**/_version.py",
-]
-
 [tool.mypy]
 # Allow None for argument default values
 implicit_optional = true
@@ -753,3 +746,81 @@ skip= [
     "models",
     "thirdparty"
 ]
+
+[tool.flake8]
+filename=[
+  "*.py",
+  "*.pyx",
+  "*.pxd"
+]
+# TODO: Remove dfencoder https://github.com/nv-morpheus/Morpheus/issues/786
+exclude=[
+   "__pycache__",
+   ".git",
+   ".tmp/",
+   "*.egg",
+   "build/",
+   "cpp",
+   "docs",
+   "models/",
+   "python/morpheus/morpheus/models/dfencoder/*.py",
+   "thirdparty"
+]
+max-line-length=120
+max-doc-length=120
+
+extend-ignore=[
+    # Ignore missing docstrings __init__ methods as we document those on the class
+    "D107",
+    # Don't require quotes to be placed on the same line as a one-line docstring, useful when the docstring is close
+    # to the line limit
+    "D200",
+    # Allow a blank line between a docstring and the code
+    "D202",
+    # D204: 1 blank line required after class docstring
+    "D204",
+    # D205: 1 blank line required between summary line and description (allow for a paragraph)
+    "D205",
+    # D400: First line should end with a period (only works if we're adhering to D205)
+    "D400",
+    # D401: First line should be in imperative mood
+    "D401"
+]
+
+# Cython Rules ignored:
+# E999: invalid syntax (works for Python, not Cython)
+# E225: Missing whitespace around operators (breaks cython casting syntax like <int>)
+# E226: Missing whitespace around arithmetic operators (breaks cython pointer syntax like int*)
+# E227: Missing whitespace around bitwise or shift operator (Can also break casting syntax)
+# W503: line break before binary operator (breaks lines that start with a pointer)
+# W504: line break after binary operator (breaks lines that end with a pointer)
+
+per-file-ignores =[
+    # imported but unused
+    "__init__.py: F401, E402",
+    # Ignore additional deps needed for examples
+    "examples/*.py: F821",
+    # Cython Exclusions
+    "*.pyx: E999, E225, E226, E227, W503, W504",
+    "*.pxd: E999, E225, E226, E227, W503, W504",
+    # Remove some documentation requirements for tests
+    # D100: Missing docstring in public module
+    # D101: Missing docstring in public class
+    # D102: Missing docstring in public method
+    # D103: Missing docstring in public function
+    "tests/*.py: D100, D101, D102, D103"
+]
+
+[tool.yapfignore]
+ignore_patterns = [
+  "**/*.pyx",
+  "**/*.pxd",
+  "**/_version.py"
+]
+
+[tool.yapf]
+based_on_style="pep8"
+column_limit=120
+split_all_top_level_comma_separated_values=true
+join_multiple_lines=true
+indent_dictionary_value=true

python/morpheus/morpheus/_lib/common/__init__.pyi

@@ -5,9 +5,11 @@
            :toctree: _generate
         """
 from __future__ import annotations
-import morpheus._lib.common
-import typing
+
 import os
+import typing
+
+import morpheus._lib.common
 
 __all__ = [
     "FiberQueue",

python/morpheus/morpheus/_lib/cudf_helpers/__init__.pyi

@@ -1,8 +1,11 @@
 from __future__ import annotations
-import morpheus._lib.cudf_helpers
+
 import typing
-from cudf.core.dtypes import StructDtype
+
 import cudf
+from cudf.core.dtypes import StructDtype
+
+import morpheus._lib.cudf_helpers
 
 __all__ = [
     "StructDtype",

python/morpheus/morpheus/_lib/doca/__init__.pyi

@@ -1,10 +1,13 @@
 from __future__ import annotations
-import morpheus._lib.doca
-import typing
+
 import datetime
-import morpheus._lib.messages
+import typing
+
 import mrc.core.segment
 
+import morpheus._lib.doca
+import morpheus._lib.messages
+
 __all__ = [
     "DocaConvertStage",
     "DocaSourceStage"

python/morpheus/morpheus/_lib/messages/__init__.pyi

@@ -6,12 +6,15 @@
 
         """
 from __future__ import annotations
-import morpheus._lib.messages
+
 import typing
+
 import cupy
-import morpheus._lib.common
 import mrc.core.node
 
+import morpheus._lib.common
+import morpheus._lib.messages
+
 __all__ = [
     "ControlMessage",
     "ControlMessageType",

python/morpheus/morpheus/_lib/modules/__init__.pyi

@@ -6,9 +6,11 @@
 
         """
 from __future__ import annotations
-import morpheus._lib.modules
+
 import typing
 
+import morpheus._lib.modules
+
 __all__ = [
 
 ]

python/morpheus/morpheus/_lib/stages/__init__.pyi

@@ -6,13 +6,16 @@
 
         """
 from __future__ import annotations
-import morpheus._lib.stages
+
+import os
 import typing
-from morpheus._lib.common import FilterSource
-import morpheus._lib.common
+
 import mrc.core.coro
 import mrc.core.segment
-import os
+
+import morpheus._lib.common
+import morpheus._lib.stages
+from morpheus._lib.common import FilterSource
 
 __all__ = [
     "AddClassificationsControlMessageStage",

python/morpheus/setup.cfg

@@ -23,73 +23,3 @@ versionfile_source = morpheus/_version.py
 versionfile_build = morpheus/_version.py
 tag_prefix = v
 parentdir_prefix = morpheus-
-
-# ===== flake8 Config =====
-[flake8]
-filename = *.py, *.pyx, *.pxd
-
-# TODO: Remove dfencoder https://github.com/nv-morpheus/Morpheus/issues/786
-exclude =
-   __pycache__,
-   .git,
-   .tmp/,
-   *.egg,
-   build/,
-   cpp,
-   docs,
-   models/,
-   morpheus/models/dfencoder/*.py,
-   thirdparty
-max-line-length = 120
-max-doc-length = 120
-
-extend-ignore =
-    # Ignore missing docstrings __init__ methods as we document those on the class
-    D107,
-    # Don't require quotes to be placed on the same line as a one-line docstring, useful when the docstring is close
-    # to the line limit
-    D200,
-    # Allow a blank line between a docstring and the code
-    D202,
-    # D204: 1 blank line required after class docstring
-    D204,
-    # D205: 1 blank line required between summary line and description (allow for a paragraph)
-    D205,
-    # D400: First line should end with a period (only works if we're adhering to D205)
-    D400,
-    # D401: First line should be in imperative mood
-    D401
-
-# Cython Rules ignored:
-# E999: invalid syntax (works for Python, not Cython)
-# E225: Missing whitespace around operators (breaks cython casting syntax like <int>)
-# E226: Missing whitespace around arithmetic operators (breaks cython pointer syntax like int*)
-# E227: Missing whitespace around bitwise or shift operator (Can also break casting syntax)
-# W503: line break before binary operator (breaks lines that start with a pointer)
-# W504: line break after binary operator (breaks lines that end with a pointer)
-
-per-file-ignores =
-    # imported but unused
-    __init__.py: F401, E402
-    # Ignore additional deps needed for examples
-    examples/*.py: F821
-    # Cython Exclusions
-    *.pyx: E999, E225, E226, E227, W503, W504
-    *.pxd: E999, E225, E226, E227, W503, W504
-    # Remove some documentation requirements for tests
-    tests/*.py:
-        # D100: Missing docstring in public module
-        D100,
-        # D101: Missing docstring in public class
-        D101,
-        # D102: Missing docstring in public method
-        D102,
-        # D103: Missing docstring in public function
-        D103
-
-[yapf]
-based_on_style = pep8
-column_limit = 120
-split_all_top_level_comma_separated_values = true
-join_multiple_lines = true
-indent_dictionary_value = true