run yapf, flake8, isort as part of pre-commit

nv-morpheus · Aug 23, 2024 · 26763a9 · 26763a9
1 parent 31f1214
commit 26763a9
Show file tree

Hide file tree

Showing 9 changed files with 54 additions and 75 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -19,6 +19,24 @@ repos:
         hooks:
             - id: rapids-dependency-file-generator
               args: ["--clean"]
+      - repo: https://github.com/PyCQA/isort
+        rev: 5.12.0
+        hooks:
+              - id: isort
+                args: ["--settings-file=./pyproject.toml"]
+                files: ^python/
+      - repo: https://github.com/PyCQA/flake8
+        rev: 6.1.0
+        hooks:
+              - id: flake8
+                args: ["--config=./python/morpheus/setup.cfg"]
+                files: ^python/
+      - repo: https://github.com/google/yapf
+        rev: v0.40.2
+        hooks:
+              - id: yapf
+                args: ["--style", "./python/morpheus/setup.cfg"]
+                files: ^python/
 
 default_language_version:
       python: python3
diff --git a/ci/scripts/python_checks.sh b/ci/scripts/python_checks.sh
@@ -26,10 +26,7 @@ LC_ALL=C.UTF-8
 LANG=C.UTF-8
 
 # Pre-populate the return values in case they are skipped
-ISORT_RETVAL=0
-FLAKE_RETVAL=0
 PYLINT_RETVAL=0
-YAPF_RETVAL=0
 
 get_modified_files ${PYTHON_FILE_REGEX} MORPHEUS_MODIFIED_FILES
 
@@ -41,45 +38,16 @@ if [[ -n "${MORPHEUS_MODIFIED_FILES}" ]]; then
       echo "  $f"
    done
 
-   if [[ "${SKIP_ISORT}" == "" ]]; then
-      ISORT_OUTPUT=`python3 -m isort --settings-file ${PROJ_TOML} --filter-files --check-only  ${MORPHEUS_MODIFIED_FILES[@]} 2>&1`
-      ISORT_RETVAL=$?
-   fi
-
    if [[ "${SKIP_PYLINT}" == "" ]]; then
       NUM_PROC=$(get_num_proc)
       PYLINT_OUTPUT=`pylint -j ${NUM_PROC} ${MORPHEUS_MODIFIED_FILES[@]} 2>&1`
       PYLINT_RETVAL=$?
    fi
 
-   if [[ "${SKIP_FLAKE}" == "" ]]; then
-      FLAKE_OUTPUT=`python3 -m flake8 --config ${PY_CFG} ${MORPHEUS_MODIFIED_FILES[@]} 2>&1`
-      FLAKE_RETVAL=$?
-   fi
-
-   if [[ "${SKIP_YAPF}" == "" ]]; then
-      # Run yapf. Will return 1 if there are any diffs
-      YAPF_OUTPUT=`python3 -m yapf --style ${PY_CFG} --diff ${MORPHEUS_MODIFIED_FILES[@]} 2>&1`
-      YAPF_RETVAL=$?
-   fi
-
 else
    echo "No modified Python files to check"
 fi
 
-# Output results if failure otherwise show pass
-if [[ "${SKIP_ISORT}" != "" ]]; then
-   echo -e "\n\n>>>> SKIPPED: isort check\n\n"
-elif [ "${ISORT_RETVAL}" != "0" ]; then
-   echo -e "\n\n>>>> FAILED: isort style check; begin output\n\n"
-   echo -e "${ISORT_OUTPUT}"
-   echo -e "\n\n>>>> FAILED: isort style check; end output\n\n" \
-           "To auto-fix many issues (not all) run:\n" \
-           "   ./ci/scripts/fix_all.sh\n\n"
-else
-  echo -e "\n\n>>>> PASSED: isort style check\n\n"
-fi
-
 if [[ "${SKIP_PYLINT}" != "" ]]; then
    echo -e "\n\n>>>> SKIPPED: pylint check\n\n"
 elif [ "${PYLINT_RETVAL}" != "0" ]; then
@@ -90,33 +58,7 @@ else
   echo -e "\n\n>>>> PASSED: pylint style check\n\n"
 fi
 
-if [[ "${SKIP_FLAKE}" != "" ]]; then
-   echo -e "\n\n>>>> SKIPPED: flake8 check\n\n"
-elif [ "${FLAKE_RETVAL}" != "0" ]; then
-   echo -e "\n\n>>>> FAILED: flake8 style check; begin output\n\n"
-   echo -e "${FLAKE_OUTPUT}"
-   echo -e "\n\n>>>> FAILED: flake8 style check; end output\n\n" \
-           "To auto-fix many issues (not all) run:\n" \
-           "   ./ci/scripts/fix_all.sh\n\n"
-else
-  echo -e "\n\n>>>> PASSED: flake8 style check\n\n"
-fi
-
-if [[ "${SKIP_YAPF}" != "" ]]; then
-   echo -e "\n\n>>>> SKIPPED: yapf check\n\n"
-elif [ "${YAPF_RETVAL}" != "0" ]; then
-   echo -e "\n\n>>>> FAILED: yapf style check; begin output\n\n"
-   echo -e "Incorrectly formatted files:"
-   YAPF_OUTPUT=`echo "${YAPF_OUTPUT}" | sed -nr 's/^\+\+\+ ([^ ]*) *\(reformatted\)$/\1/p'`
-   echo -e "${YAPF_OUTPUT}"
-   echo -e "\n\n>>>> FAILED: yapf style check; end output\n\n" \
-           "To auto-fix many issues (not all) run:\n" \
-           "   ./ci/scripts/fix_all.sh\n\n"
-else
-  echo -e "\n\n>>>> PASSED: yapf style check\n\n"
-fi
-
-RETVALS=(${ISORT_RETVAL} ${FLAKE_RETVAL} ${PYLINT_RETVAL} ${YAPF_RETVAL})
+RETVALS=(${PYLINT_RETVAL})
 IFS=$'\n'
 RETVAL=`echo "${RETVALS[*]}" | sort -nr | head -n1`
 

diff --git a/python/morpheus/morpheus/_lib/common/__init__.pyi b/python/morpheus/morpheus/_lib/common/__init__.pyi
@@ -5,9 +5,11 @@
            :toctree: _generate
         """
 from __future__ import annotations
-import morpheus._lib.common
-import typing
+
 import os
+import typing
+
+import morpheus._lib.common
 
 __all__ = [
     "FiberQueue",

diff --git a/python/morpheus/morpheus/_lib/cudf_helpers/__init__.pyi b/python/morpheus/morpheus/_lib/cudf_helpers/__init__.pyi
@@ -1,8 +1,11 @@
 from __future__ import annotations
-import morpheus._lib.cudf_helpers
+
 import typing
-from cudf.core.dtypes import StructDtype
+
 import cudf
+from cudf.core.dtypes import StructDtype
+
+import morpheus._lib.cudf_helpers
 
 __all__ = [
     "StructDtype",

diff --git a/python/morpheus/morpheus/_lib/doca/__init__.pyi b/python/morpheus/morpheus/_lib/doca/__init__.pyi
@@ -1,10 +1,13 @@
 from __future__ import annotations
-import morpheus._lib.doca
-import typing
+
 import datetime
-import morpheus._lib.messages
+import typing
+
 import mrc.core.segment
 
+import morpheus._lib.doca
+import morpheus._lib.messages
+
 __all__ = [
     "DocaConvertStage",
     "DocaSourceStage"

diff --git a/python/morpheus/morpheus/_lib/llm/__init__.pyi b/python/morpheus/morpheus/_lib/llm/__init__.pyi
@@ -6,12 +6,15 @@
 
         """
 from __future__ import annotations
-import morpheus._lib.llm
+
 import typing
-import morpheus._lib.messages
+
 import mrc.core.coro
 import mrc.core.segment
 
+import morpheus._lib.llm
+import morpheus._lib.messages
+
 __all__ = [
     "InputMap",
     "LLMContext",

diff --git a/python/morpheus/morpheus/_lib/messages/__init__.pyi b/python/morpheus/morpheus/_lib/messages/__init__.pyi
@@ -6,12 +6,15 @@
 
         """
 from __future__ import annotations
-import morpheus._lib.messages
+
 import typing
+
 import cupy
-import morpheus._lib.common
 import mrc.core.node
 
+import morpheus._lib.common
+import morpheus._lib.messages
+
 __all__ = [
     "ControlMessage",
     "ControlMessageType",

diff --git a/python/morpheus/morpheus/_lib/modules/__init__.pyi b/python/morpheus/morpheus/_lib/modules/__init__.pyi
@@ -6,9 +6,11 @@
 
         """
 from __future__ import annotations
-import morpheus._lib.modules
+
 import typing
 
+import morpheus._lib.modules
+
 __all__ = [
 
 ]

diff --git a/python/morpheus/morpheus/_lib/stages/__init__.pyi b/python/morpheus/morpheus/_lib/stages/__init__.pyi
@@ -6,12 +6,15 @@
 
         """
 from __future__ import annotations
-import morpheus._lib.stages
+
+import os
 import typing
-from morpheus._lib.common import FilterSource
-import morpheus._lib.common
+
 import mrc.core.segment
-import os
+
+import morpheus._lib.common
+import morpheus._lib.stages
+from morpheus._lib.common import FilterSource
 
 __all__ = [
     "AddClassificationsControlMessageStage",