From 5a17b92f045c9c96a75243234b1e75fc2400f775 Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Fri, 30 Aug 2024 11:31:35 -0500 Subject: [PATCH] Run yapf, flake8, isort as part of pre-commit (#1859) Closes #1702 ## By Submitting this PR I confirm: - I am familiar with the [Contributing Guidelines](https://github.com/nv-morpheus/Morpheus/blob/main/docs/source/developer_guide/contributing.md). - When the PR is ready for review, new or existing tests cover these changes. - When the PR is ready for review, the documentation is up to date with these changes. Authors: - Christopher Harris (https://github.com/cwharris) Approvers: - Anuradha Karuppiah (https://github.com/AnuradhaKaruppiah) URL: https://github.com/nv-morpheus/Morpheus/pull/1859 --- .pre-commit-config.yaml | 20 +++++ ci/scripts/fix_all.sh | 2 +- ci/scripts/python_checks.sh | 60 +------------ morpheus.code-workspace | 4 +- pyproject.toml | 85 +++++++++++++++++-- .../morpheus/_lib/common/__init__.pyi | 6 +- .../morpheus/_lib/cudf_helpers/__init__.pyi | 7 +- .../morpheus/morpheus/_lib/doca/__init__.pyi | 9 +- .../morpheus/_lib/messages/__init__.pyi | 7 +- .../morpheus/_lib/modules/__init__.pyi | 4 +- .../morpheus/_lib/stages/__init__.pyi | 11 ++- python/morpheus/setup.cfg | 70 --------------- .../morpheus_llm/_lib/llm/__init__.pyi | 7 +- 13 files changed, 137 insertions(+), 155 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2682c98fb7..5b0e6c5fca 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,6 +19,26 @@ repos: hooks: - id: rapids-dependency-file-generator args: ["--clean"] + - repo: https://github.com/PyCQA/isort + rev: 5.12.0 + hooks: + - id: isort + args: ["--settings-file=./pyproject.toml"] + files: ^python/ + - repo: https://github.com/PyCQA/flake8 + rev: 6.1.0 + hooks: + - id: flake8 + entry: pflake8 + additional_dependencies: [pyproject-flake8] + args: ["--config=./pyproject.toml"] + files: ^python/ + - repo: https://github.com/google/yapf + rev: v0.40.2 + hooks: + - id: yapf + args: ["--style", "./pyproject.toml"] + files: ^python/ default_language_version: python: python3 diff --git a/ci/scripts/fix_all.sh b/ci/scripts/fix_all.sh index 100f0d62f6..e2004c84b6 100755 --- a/ci/scripts/fix_all.sh +++ b/ci/scripts/fix_all.sh @@ -105,5 +105,5 @@ fi # Run yapf if [[ "${SKIP_YAPF}" == "" ]]; then echo "Running yapf..." - python3 -m yapf -i --style ${PY_CFG} -r ${PY_MODIFIED_FILES[@]} + python3 -m yapf -i --style ${PROJ_TOML} -r ${PY_MODIFIED_FILES[@]} fi diff --git a/ci/scripts/python_checks.sh b/ci/scripts/python_checks.sh index e7b9386e95..84acabefe6 100755 --- a/ci/scripts/python_checks.sh +++ b/ci/scripts/python_checks.sh @@ -26,10 +26,7 @@ LC_ALL=C.UTF-8 LANG=C.UTF-8 # Pre-populate the return values in case they are skipped -ISORT_RETVAL=0 -FLAKE_RETVAL=0 PYLINT_RETVAL=0 -YAPF_RETVAL=0 get_modified_files ${PYTHON_FILE_REGEX} MORPHEUS_MODIFIED_FILES @@ -41,45 +38,16 @@ if [[ -n "${MORPHEUS_MODIFIED_FILES}" ]]; then echo " $f" done - if [[ "${SKIP_ISORT}" == "" ]]; then - ISORT_OUTPUT=`python3 -m isort --settings-file ${PROJ_TOML} --filter-files --check-only ${MORPHEUS_MODIFIED_FILES[@]} 2>&1` - ISORT_RETVAL=$? - fi - if [[ "${SKIP_PYLINT}" == "" ]]; then NUM_PROC=$(get_num_proc) PYLINT_OUTPUT=`pylint -j ${NUM_PROC} ${MORPHEUS_MODIFIED_FILES[@]} 2>&1` PYLINT_RETVAL=$? fi - if [[ "${SKIP_FLAKE}" == "" ]]; then - FLAKE_OUTPUT=`python3 -m flake8 --config ${PY_CFG} ${MORPHEUS_MODIFIED_FILES[@]} 2>&1` - FLAKE_RETVAL=$? - fi - - if [[ "${SKIP_YAPF}" == "" ]]; then - # Run yapf. Will return 1 if there are any diffs - YAPF_OUTPUT=`python3 -m yapf --style ${PY_CFG} --diff ${MORPHEUS_MODIFIED_FILES[@]} 2>&1` - YAPF_RETVAL=$? - fi - else echo "No modified Python files to check" fi -# Output results if failure otherwise show pass -if [[ "${SKIP_ISORT}" != "" ]]; then - echo -e "\n\n>>>> SKIPPED: isort check\n\n" -elif [ "${ISORT_RETVAL}" != "0" ]; then - echo -e "\n\n>>>> FAILED: isort style check; begin output\n\n" - echo -e "${ISORT_OUTPUT}" - echo -e "\n\n>>>> FAILED: isort style check; end output\n\n" \ - "To auto-fix many issues (not all) run:\n" \ - " ./ci/scripts/fix_all.sh\n\n" -else - echo -e "\n\n>>>> PASSED: isort style check\n\n" -fi - if [[ "${SKIP_PYLINT}" != "" ]]; then echo -e "\n\n>>>> SKIPPED: pylint check\n\n" elif [ "${PYLINT_RETVAL}" != "0" ]; then @@ -90,33 +58,7 @@ else echo -e "\n\n>>>> PASSED: pylint style check\n\n" fi -if [[ "${SKIP_FLAKE}" != "" ]]; then - echo -e "\n\n>>>> SKIPPED: flake8 check\n\n" -elif [ "${FLAKE_RETVAL}" != "0" ]; then - echo -e "\n\n>>>> FAILED: flake8 style check; begin output\n\n" - echo -e "${FLAKE_OUTPUT}" - echo -e "\n\n>>>> FAILED: flake8 style check; end output\n\n" \ - "To auto-fix many issues (not all) run:\n" \ - " ./ci/scripts/fix_all.sh\n\n" -else - echo -e "\n\n>>>> PASSED: flake8 style check\n\n" -fi - -if [[ "${SKIP_YAPF}" != "" ]]; then - echo -e "\n\n>>>> SKIPPED: yapf check\n\n" -elif [ "${YAPF_RETVAL}" != "0" ]; then - echo -e "\n\n>>>> FAILED: yapf style check; begin output\n\n" - echo -e "Incorrectly formatted files:" - YAPF_OUTPUT=`echo "${YAPF_OUTPUT}" | sed -nr 's/^\+\+\+ ([^ ]*) *\(reformatted\)$/\1/p'` - echo -e "${YAPF_OUTPUT}" - echo -e "\n\n>>>> FAILED: yapf style check; end output\n\n" \ - "To auto-fix many issues (not all) run:\n" \ - " ./ci/scripts/fix_all.sh\n\n" -else - echo -e "\n\n>>>> PASSED: yapf style check\n\n" -fi - -RETVALS=(${ISORT_RETVAL} ${FLAKE_RETVAL} ${PYLINT_RETVAL} ${YAPF_RETVAL}) +RETVALS=(${PYLINT_RETVAL}) IFS=$'\n' RETVAL=`echo "${RETVALS[*]}" | sort -nr | head -n1` diff --git a/morpheus.code-workspace b/morpheus.code-workspace index 5fb3a58375..0ec937642b 100644 --- a/morpheus.code-workspace +++ b/morpheus.code-workspace @@ -679,7 +679,7 @@ "files.trimFinalNewlines": true, "files.trimTrailingWhitespace": true, "flake8.args": [ - "--style=${workspaceFolder}/python/morpheus/setup.cfg" + "--style=${workspaceFolder}/pyproject.toml" ], "pylint.args": [ "--rcfile=${workspaceFolder}/pyproject.toml", @@ -730,7 +730,7 @@ } ], "yapf.args": [ - "--style=${workspaceFolder}/python/morpheus/setup.cfg" + "--style=${workspaceFolder}/pyproject.toml" ] } } diff --git a/pyproject.toml b/pyproject.toml index a2490e0456..c597d71c03 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,13 +45,6 @@ addopts = "--benchmark-disable" asyncio_mode = "auto" -[tool.yapfignore] -ignore_patterns = [ - "**/*.pyx", - "**/*.pxd", - "**/_version.py", -] - [tool.mypy] # Allow None for argument default values implicit_optional = true @@ -753,3 +746,81 @@ skip= [ "models", "thirdparty" ] + +[tool.flake8] +filename=[ + "*.py", + "*.pyx", + "*.pxd" +] +# TODO: Remove dfencoder https://github.com/nv-morpheus/Morpheus/issues/786 +exclude=[ + "__pycache__", + ".git", + ".tmp/", + "*.egg", + "build/", + "cpp", + "docs", + "models/", + "python/morpheus/morpheus/models/dfencoder/*.py", + "thirdparty" +] +max-line-length=120 +max-doc-length=120 + +extend-ignore=[ + # Ignore missing docstrings __init__ methods as we document those on the class + "D107", + # Don't require quotes to be placed on the same line as a one-line docstring, useful when the docstring is close + # to the line limit + "D200", + # Allow a blank line between a docstring and the code + "D202", + # D204: 1 blank line required after class docstring + "D204", + # D205: 1 blank line required between summary line and description (allow for a paragraph) + "D205", + # D400: First line should end with a period (only works if we're adhering to D205) + "D400", + # D401: First line should be in imperative mood + "D401" +] + +# Cython Rules ignored: +# E999: invalid syntax (works for Python, not Cython) +# E225: Missing whitespace around operators (breaks cython casting syntax like ) +# E226: Missing whitespace around arithmetic operators (breaks cython pointer syntax like int*) +# E227: Missing whitespace around bitwise or shift operator (Can also break casting syntax) +# W503: line break before binary operator (breaks lines that start with a pointer) +# W504: line break after binary operator (breaks lines that end with a pointer) + +per-file-ignores =[ + # imported but unused + "__init__.py: F401, E402", + # Ignore additional deps needed for examples + "examples/*.py: F821", + # Cython Exclusions + "*.pyx: E999, E225, E226, E227, W503, W504", + "*.pxd: E999, E225, E226, E227, W503, W504", + # Remove some documentation requirements for tests + # D100: Missing docstring in public module + # D101: Missing docstring in public class + # D102: Missing docstring in public method + # D103: Missing docstring in public function + "tests/*.py: D100, D101, D102, D103" +] + +[tool.yapfignore] +ignore_patterns = [ + "**/*.pyx", + "**/*.pxd", + "**/_version.py" +] + +[tool.yapf] +based_on_style="pep8" +column_limit=120 +split_all_top_level_comma_separated_values=true +join_multiple_lines=true +indent_dictionary_value=true diff --git a/python/morpheus/morpheus/_lib/common/__init__.pyi b/python/morpheus/morpheus/_lib/common/__init__.pyi index 7e11e81ccd..f1436eb5e7 100644 --- a/python/morpheus/morpheus/_lib/common/__init__.pyi +++ b/python/morpheus/morpheus/_lib/common/__init__.pyi @@ -5,9 +5,11 @@ :toctree: _generate """ from __future__ import annotations -import morpheus._lib.common -import typing + import os +import typing + +import morpheus._lib.common __all__ = [ "FiberQueue", diff --git a/python/morpheus/morpheus/_lib/cudf_helpers/__init__.pyi b/python/morpheus/morpheus/_lib/cudf_helpers/__init__.pyi index 4ce3dd3269..027b74ecc0 100644 --- a/python/morpheus/morpheus/_lib/cudf_helpers/__init__.pyi +++ b/python/morpheus/morpheus/_lib/cudf_helpers/__init__.pyi @@ -1,8 +1,11 @@ from __future__ import annotations -import morpheus._lib.cudf_helpers + import typing -from cudf.core.dtypes import StructDtype + import cudf +from cudf.core.dtypes import StructDtype + +import morpheus._lib.cudf_helpers __all__ = [ "StructDtype", diff --git a/python/morpheus/morpheus/_lib/doca/__init__.pyi b/python/morpheus/morpheus/_lib/doca/__init__.pyi index 100af5654a..f65ba226dc 100644 --- a/python/morpheus/morpheus/_lib/doca/__init__.pyi +++ b/python/morpheus/morpheus/_lib/doca/__init__.pyi @@ -1,10 +1,13 @@ from __future__ import annotations -import morpheus._lib.doca -import typing + import datetime -import morpheus._lib.messages +import typing + import mrc.core.segment +import morpheus._lib.doca +import morpheus._lib.messages + __all__ = [ "DocaConvertStage", "DocaSourceStage" diff --git a/python/morpheus/morpheus/_lib/messages/__init__.pyi b/python/morpheus/morpheus/_lib/messages/__init__.pyi index f4ef520152..41a7723c44 100644 --- a/python/morpheus/morpheus/_lib/messages/__init__.pyi +++ b/python/morpheus/morpheus/_lib/messages/__init__.pyi @@ -6,12 +6,15 @@ """ from __future__ import annotations -import morpheus._lib.messages + import typing + import cupy -import morpheus._lib.common import mrc.core.node +import morpheus._lib.common +import morpheus._lib.messages + __all__ = [ "ControlMessage", "ControlMessageType", diff --git a/python/morpheus/morpheus/_lib/modules/__init__.pyi b/python/morpheus/morpheus/_lib/modules/__init__.pyi index 0ec21dfaad..ff105f6d3b 100644 --- a/python/morpheus/morpheus/_lib/modules/__init__.pyi +++ b/python/morpheus/morpheus/_lib/modules/__init__.pyi @@ -6,9 +6,11 @@ """ from __future__ import annotations -import morpheus._lib.modules + import typing +import morpheus._lib.modules + __all__ = [ ] diff --git a/python/morpheus/morpheus/_lib/stages/__init__.pyi b/python/morpheus/morpheus/_lib/stages/__init__.pyi index 86a91e9088..ffd9a46c91 100644 --- a/python/morpheus/morpheus/_lib/stages/__init__.pyi +++ b/python/morpheus/morpheus/_lib/stages/__init__.pyi @@ -6,13 +6,16 @@ """ from __future__ import annotations -import morpheus._lib.stages + +import os import typing -from morpheus._lib.common import FilterSource -import morpheus._lib.common + import mrc.core.coro import mrc.core.segment -import os + +import morpheus._lib.common +import morpheus._lib.stages +from morpheus._lib.common import FilterSource __all__ = [ "AddClassificationsControlMessageStage", diff --git a/python/morpheus/setup.cfg b/python/morpheus/setup.cfg index 1c7bcb39fe..6b57906472 100644 --- a/python/morpheus/setup.cfg +++ b/python/morpheus/setup.cfg @@ -23,73 +23,3 @@ versionfile_source = morpheus/_version.py versionfile_build = morpheus/_version.py tag_prefix = v parentdir_prefix = morpheus- - -# ===== flake8 Config ===== -[flake8] -filename = *.py, *.pyx, *.pxd - -# TODO: Remove dfencoder https://github.com/nv-morpheus/Morpheus/issues/786 -exclude = - __pycache__, - .git, - .tmp/, - *.egg, - build/, - cpp, - docs, - models/, - morpheus/models/dfencoder/*.py, - thirdparty -max-line-length = 120 -max-doc-length = 120 - -extend-ignore = - # Ignore missing docstrings __init__ methods as we document those on the class - D107, - # Don't require quotes to be placed on the same line as a one-line docstring, useful when the docstring is close - # to the line limit - D200, - # Allow a blank line between a docstring and the code - D202, - # D204: 1 blank line required after class docstring - D204, - # D205: 1 blank line required between summary line and description (allow for a paragraph) - D205, - # D400: First line should end with a period (only works if we're adhering to D205) - D400, - # D401: First line should be in imperative mood - D401 - -# Cython Rules ignored: -# E999: invalid syntax (works for Python, not Cython) -# E225: Missing whitespace around operators (breaks cython casting syntax like ) -# E226: Missing whitespace around arithmetic operators (breaks cython pointer syntax like int*) -# E227: Missing whitespace around bitwise or shift operator (Can also break casting syntax) -# W503: line break before binary operator (breaks lines that start with a pointer) -# W504: line break after binary operator (breaks lines that end with a pointer) - -per-file-ignores = - # imported but unused - __init__.py: F401, E402 - # Ignore additional deps needed for examples - examples/*.py: F821 - # Cython Exclusions - *.pyx: E999, E225, E226, E227, W503, W504 - *.pxd: E999, E225, E226, E227, W503, W504 - # Remove some documentation requirements for tests - tests/*.py: - # D100: Missing docstring in public module - D100, - # D101: Missing docstring in public class - D101, - # D102: Missing docstring in public method - D102, - # D103: Missing docstring in public function - D103 - -[yapf] -based_on_style = pep8 -column_limit = 120 -split_all_top_level_comma_separated_values = true -join_multiple_lines = true -indent_dictionary_value = true diff --git a/python/morpheus_llm/morpheus_llm/_lib/llm/__init__.pyi b/python/morpheus_llm/morpheus_llm/_lib/llm/__init__.pyi index 36c202e16c..351790410d 100644 --- a/python/morpheus_llm/morpheus_llm/_lib/llm/__init__.pyi +++ b/python/morpheus_llm/morpheus_llm/_lib/llm/__init__.pyi @@ -6,12 +6,15 @@ """ from __future__ import annotations -import morpheus_llm._lib.llm + import typing -import morpheus._lib.messages + import mrc.core.coro import mrc.core.segment +import morpheus._lib.llm +import morpheus._lib.messages + __all__ = [ "InputMap", "LLMContext",