From 26763a92850d36674ad0c4e0dcf20ed8c7e28928 Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Fri, 23 Aug 2024 18:28:43 +0000 Subject: [PATCH] run yapf, flake8, isort as part of pre-commit --- .pre-commit-config.yaml | 18 ++++++ ci/scripts/python_checks.sh | 60 +------------------ .../morpheus/_lib/common/__init__.pyi | 6 +- .../morpheus/_lib/cudf_helpers/__init__.pyi | 7 ++- .../morpheus/morpheus/_lib/doca/__init__.pyi | 9 ++- .../morpheus/morpheus/_lib/llm/__init__.pyi | 7 ++- .../morpheus/_lib/messages/__init__.pyi | 7 ++- .../morpheus/_lib/modules/__init__.pyi | 4 +- .../morpheus/_lib/stages/__init__.pyi | 11 ++-- 9 files changed, 54 insertions(+), 75 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2682c98fb7..74a549bc1b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,6 +19,24 @@ repos: hooks: - id: rapids-dependency-file-generator args: ["--clean"] + - repo: https://github.com/PyCQA/isort + rev: 5.12.0 + hooks: + - id: isort + args: ["--settings-file=./pyproject.toml"] + files: ^python/ + - repo: https://github.com/PyCQA/flake8 + rev: 6.1.0 + hooks: + - id: flake8 + args: ["--config=./python/morpheus/setup.cfg"] + files: ^python/ + - repo: https://github.com/google/yapf + rev: v0.40.2 + hooks: + - id: yapf + args: ["--style", "./python/morpheus/setup.cfg"] + files: ^python/ default_language_version: python: python3 diff --git a/ci/scripts/python_checks.sh b/ci/scripts/python_checks.sh index e7b9386e95..84acabefe6 100755 --- a/ci/scripts/python_checks.sh +++ b/ci/scripts/python_checks.sh @@ -26,10 +26,7 @@ LC_ALL=C.UTF-8 LANG=C.UTF-8 # Pre-populate the return values in case they are skipped -ISORT_RETVAL=0 -FLAKE_RETVAL=0 PYLINT_RETVAL=0 -YAPF_RETVAL=0 get_modified_files ${PYTHON_FILE_REGEX} MORPHEUS_MODIFIED_FILES @@ -41,45 +38,16 @@ if [[ -n "${MORPHEUS_MODIFIED_FILES}" ]]; then echo " $f" done - if [[ "${SKIP_ISORT}" == "" ]]; then - ISORT_OUTPUT=`python3 -m isort --settings-file ${PROJ_TOML} --filter-files --check-only ${MORPHEUS_MODIFIED_FILES[@]} 2>&1` - ISORT_RETVAL=$? - fi - if [[ "${SKIP_PYLINT}" == "" ]]; then NUM_PROC=$(get_num_proc) PYLINT_OUTPUT=`pylint -j ${NUM_PROC} ${MORPHEUS_MODIFIED_FILES[@]} 2>&1` PYLINT_RETVAL=$? fi - if [[ "${SKIP_FLAKE}" == "" ]]; then - FLAKE_OUTPUT=`python3 -m flake8 --config ${PY_CFG} ${MORPHEUS_MODIFIED_FILES[@]} 2>&1` - FLAKE_RETVAL=$? - fi - - if [[ "${SKIP_YAPF}" == "" ]]; then - # Run yapf. Will return 1 if there are any diffs - YAPF_OUTPUT=`python3 -m yapf --style ${PY_CFG} --diff ${MORPHEUS_MODIFIED_FILES[@]} 2>&1` - YAPF_RETVAL=$? - fi - else echo "No modified Python files to check" fi -# Output results if failure otherwise show pass -if [[ "${SKIP_ISORT}" != "" ]]; then - echo -e "\n\n>>>> SKIPPED: isort check\n\n" -elif [ "${ISORT_RETVAL}" != "0" ]; then - echo -e "\n\n>>>> FAILED: isort style check; begin output\n\n" - echo -e "${ISORT_OUTPUT}" - echo -e "\n\n>>>> FAILED: isort style check; end output\n\n" \ - "To auto-fix many issues (not all) run:\n" \ - " ./ci/scripts/fix_all.sh\n\n" -else - echo -e "\n\n>>>> PASSED: isort style check\n\n" -fi - if [[ "${SKIP_PYLINT}" != "" ]]; then echo -e "\n\n>>>> SKIPPED: pylint check\n\n" elif [ "${PYLINT_RETVAL}" != "0" ]; then @@ -90,33 +58,7 @@ else echo -e "\n\n>>>> PASSED: pylint style check\n\n" fi -if [[ "${SKIP_FLAKE}" != "" ]]; then - echo -e "\n\n>>>> SKIPPED: flake8 check\n\n" -elif [ "${FLAKE_RETVAL}" != "0" ]; then - echo -e "\n\n>>>> FAILED: flake8 style check; begin output\n\n" - echo -e "${FLAKE_OUTPUT}" - echo -e "\n\n>>>> FAILED: flake8 style check; end output\n\n" \ - "To auto-fix many issues (not all) run:\n" \ - " ./ci/scripts/fix_all.sh\n\n" -else - echo -e "\n\n>>>> PASSED: flake8 style check\n\n" -fi - -if [[ "${SKIP_YAPF}" != "" ]]; then - echo -e "\n\n>>>> SKIPPED: yapf check\n\n" -elif [ "${YAPF_RETVAL}" != "0" ]; then - echo -e "\n\n>>>> FAILED: yapf style check; begin output\n\n" - echo -e "Incorrectly formatted files:" - YAPF_OUTPUT=`echo "${YAPF_OUTPUT}" | sed -nr 's/^\+\+\+ ([^ ]*) *\(reformatted\)$/\1/p'` - echo -e "${YAPF_OUTPUT}" - echo -e "\n\n>>>> FAILED: yapf style check; end output\n\n" \ - "To auto-fix many issues (not all) run:\n" \ - " ./ci/scripts/fix_all.sh\n\n" -else - echo -e "\n\n>>>> PASSED: yapf style check\n\n" -fi - -RETVALS=(${ISORT_RETVAL} ${FLAKE_RETVAL} ${PYLINT_RETVAL} ${YAPF_RETVAL}) +RETVALS=(${PYLINT_RETVAL}) IFS=$'\n' RETVAL=`echo "${RETVALS[*]}" | sort -nr | head -n1` diff --git a/python/morpheus/morpheus/_lib/common/__init__.pyi b/python/morpheus/morpheus/_lib/common/__init__.pyi index 7e11e81ccd..f1436eb5e7 100644 --- a/python/morpheus/morpheus/_lib/common/__init__.pyi +++ b/python/morpheus/morpheus/_lib/common/__init__.pyi @@ -5,9 +5,11 @@ :toctree: _generate """ from __future__ import annotations -import morpheus._lib.common -import typing + import os +import typing + +import morpheus._lib.common __all__ = [ "FiberQueue", diff --git a/python/morpheus/morpheus/_lib/cudf_helpers/__init__.pyi b/python/morpheus/morpheus/_lib/cudf_helpers/__init__.pyi index 4ce3dd3269..027b74ecc0 100644 --- a/python/morpheus/morpheus/_lib/cudf_helpers/__init__.pyi +++ b/python/morpheus/morpheus/_lib/cudf_helpers/__init__.pyi @@ -1,8 +1,11 @@ from __future__ import annotations -import morpheus._lib.cudf_helpers + import typing -from cudf.core.dtypes import StructDtype + import cudf +from cudf.core.dtypes import StructDtype + +import morpheus._lib.cudf_helpers __all__ = [ "StructDtype", diff --git a/python/morpheus/morpheus/_lib/doca/__init__.pyi b/python/morpheus/morpheus/_lib/doca/__init__.pyi index 100af5654a..f65ba226dc 100644 --- a/python/morpheus/morpheus/_lib/doca/__init__.pyi +++ b/python/morpheus/morpheus/_lib/doca/__init__.pyi @@ -1,10 +1,13 @@ from __future__ import annotations -import morpheus._lib.doca -import typing + import datetime -import morpheus._lib.messages +import typing + import mrc.core.segment +import morpheus._lib.doca +import morpheus._lib.messages + __all__ = [ "DocaConvertStage", "DocaSourceStage" diff --git a/python/morpheus/morpheus/_lib/llm/__init__.pyi b/python/morpheus/morpheus/_lib/llm/__init__.pyi index 260e9a1393..00924ddb29 100644 --- a/python/morpheus/morpheus/_lib/llm/__init__.pyi +++ b/python/morpheus/morpheus/_lib/llm/__init__.pyi @@ -6,12 +6,15 @@ """ from __future__ import annotations -import morpheus._lib.llm + import typing -import morpheus._lib.messages + import mrc.core.coro import mrc.core.segment +import morpheus._lib.llm +import morpheus._lib.messages + __all__ = [ "InputMap", "LLMContext", diff --git a/python/morpheus/morpheus/_lib/messages/__init__.pyi b/python/morpheus/morpheus/_lib/messages/__init__.pyi index f4ef520152..41a7723c44 100644 --- a/python/morpheus/morpheus/_lib/messages/__init__.pyi +++ b/python/morpheus/morpheus/_lib/messages/__init__.pyi @@ -6,12 +6,15 @@ """ from __future__ import annotations -import morpheus._lib.messages + import typing + import cupy -import morpheus._lib.common import mrc.core.node +import morpheus._lib.common +import morpheus._lib.messages + __all__ = [ "ControlMessage", "ControlMessageType", diff --git a/python/morpheus/morpheus/_lib/modules/__init__.pyi b/python/morpheus/morpheus/_lib/modules/__init__.pyi index 0ec21dfaad..ff105f6d3b 100644 --- a/python/morpheus/morpheus/_lib/modules/__init__.pyi +++ b/python/morpheus/morpheus/_lib/modules/__init__.pyi @@ -6,9 +6,11 @@ """ from __future__ import annotations -import morpheus._lib.modules + import typing +import morpheus._lib.modules + __all__ = [ ] diff --git a/python/morpheus/morpheus/_lib/stages/__init__.pyi b/python/morpheus/morpheus/_lib/stages/__init__.pyi index 12a274c601..cc00c8c3d5 100644 --- a/python/morpheus/morpheus/_lib/stages/__init__.pyi +++ b/python/morpheus/morpheus/_lib/stages/__init__.pyi @@ -6,12 +6,15 @@ """ from __future__ import annotations -import morpheus._lib.stages + +import os import typing -from morpheus._lib.common import FilterSource -import morpheus._lib.common + import mrc.core.segment -import os + +import morpheus._lib.common +import morpheus._lib.stages +from morpheus._lib.common import FilterSource __all__ = [ "AddClassificationsControlMessageStage",