From bd39e9ba845c76153a0ce5633403f1868a5d45d4 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Mon, 11 Nov 2024 15:31:23 -0800 Subject: [PATCH 1/2] drive-by fix: delete duplicate project file --- pyproject.toml-bak | 127 --------------------------------------------- 1 file changed, 127 deletions(-) delete mode 100644 pyproject.toml-bak diff --git a/pyproject.toml-bak b/pyproject.toml-bak deleted file mode 100644 index af6c4aab5..000000000 --- a/pyproject.toml-bak +++ /dev/null @@ -1,127 +0,0 @@ -[build-system] -requires = ["poetry-core>=1.0.0"] -build-backend = "poetry.core.masonry.api" - -[tool.poetry] -name = "airbyte-cdk" -version = "6.5.2" -description = "A framework for writing Airbyte Connectors." -authors = ["Airbyte "] -license = "MIT" -readme = "README.md" -homepage = "https://github.com/airbytehq/airbyte" -repository = "https://github.com/airbytehq/airbyte" -documentation = "https://docs.airbyte.io/" -classifiers = [ - "Development Status :: 3 - Alpha", - "Intended Audience :: Developers", - "Topic :: Scientific/Engineering", - "Topic :: Software Development :: Libraries :: Python Modules", - "License :: OSI Approved :: MIT License", - "Programming Language :: Python :: 3.10", -] -keywords = ["airbyte", "connector-development-kit", "cdk"] - - -[tool.poetry.dependencies] -python = "^3.10" -airbyte-protocol-models-dataclasses = "^0.13" -backoff = "*" -cachetools = "*" -Deprecated = "~1.2" -dpath = "^2.1.6" -genson = "1.2.2" -isodate = "~0.6.1" -Jinja2 = "~3.1.2" -jsonref = "~0.2" -jsonschema = "~3.2.0" -pandas = "2.2.2" -pendulum = "<3.0.0" -psutil = "6.1.0" -pydantic = "^2.7" -pyrate-limiter = "~3.1.0" -python-dateutil = "*" -PyYAML = "^6.0.1" -requests = "*" -requests_cache = "*" -wcmatch = "8.4" -# Extras depedencies -avro = { version = "~1.11.2", optional = true } -cohere = { version = "4.21", optional = true } -fastavro = { version = "~1.8.0", optional = true } -langchain = { version = "0.1.16", optional = true } -langchain_core = { version = "0.1.42", optional = true } -markdown = { version = "*", optional = true } -openai = { version = "0.27.9", extras = ["embeddings"], optional = true } -pdf2image = { version = "1.16.3", optional = true } -"pdfminer.six" = { version = "20221105", optional = true } -pyarrow = { version = "~15.0.0", optional = true } -pytesseract = { version = "0.3.10", optional = true } -python-calamine = { version = "0.2.3", optional = true } -python-snappy = { version = "0.7.3", optional = true } -Sphinx = { version = "~4.2", optional = true } -sphinx-rtd-theme = { version = "~1.0", optional = true } -tiktoken = { version = "0.4.0", optional = true } -nltk = { version = "3.8.1", optional = true } -# This will ensure that even when you run poetry install or pip install, the compatible version of numpy will always be chosen. -# airbyte-ci will try to install latest version when --use-local-cdk is used, resulting in the conflict. -numpy = "<2" -unstructured = { version = "0.10.27", extras = ["docx", "pptx"], optional = true } -"unstructured.pytesseract" = { version = ">=0.3.12", optional = true } -pyjwt = "^2.8.0" -cryptography = "^42.0.5" -pytz = "2024.1" -orjson = "^3.10.7" -serpyco-rs = "^1.10.2" -sqlalchemy = {version = "^2.0,!=2.0.36", optional = true } -xmltodict = "^0.13.0" - -[tool.poetry.group.dev.dependencies] -freezegun = "*" -mypy = "*" -asyncio = "3.4.3" -poethepoet = "^0.24.2" -pyproject-flake8 = "^6.1.0" -pytest = "^7" -pytest-memray = "^1.6.0" -pytest-cov = "*" -pytest-httpserver = "*" -pytest-mock = "*" -requests-mock = "*" - -[tool.poetry.extras] -file-based = ["avro", "fastavro", "pyarrow", "unstructured", "pdf2image", "pdfminer.six", "unstructured.pytesseract", "pytesseract", "markdown", "python-calamine", "python-snappy"] -sphinx-docs = ["Sphinx", "sphinx-rtd-theme"] -vector-db-based = ["langchain", "openai", "cohere", "tiktoken"] -sql = ["sqlalchemy"] - -[tool.ruff] -# Setting python version to at least 3.10 avoids `from __future__ import annotations`. -target-version = "py310" -# This is consistent with airbytehq/airbyte root pyproject.toml Black rule defined. -line-length = 140 - -[tool.poe.tasks] -# Build tasks -assemble = {cmd = "bin/generate-component-manifest-dagger.sh", help = "Generate component manifest files."} -build-package = {cmd = "poetry build", help = "Build the python package: source and wheels archives."} -build = {sequence = ["assemble", "build-package"], help = "Run all tasks to build the package."} - -# Check tasks -lint = {cmd = "pflake8 --config ../../pyproject.toml ./", help = "Lint with flake8."} -type-check = {cmd = "bin/run-mypy-on-modified-files.sh", help = "Type check modified files with mypy."} -unit-test-with-cov = {cmd = "pytest -s unit_tests -c pytest.ini --cov=airbyte_cdk --cov-report=term --cov-config ../../pyproject.toml", help = "Run unit tests and create a coverage report."} -# TODO: find a version of the modified mypy check that works both locally and in CI. -check-lockfile = {cmd = "poetry check", help = "Check the poetry lock file."} -check-local = {sequence = ["lint", "type-check", "check-lockfile", "unit-test-with-cov"], help = "Lint all code, type-check modified files, and run unit tests."} -check-ci = {sequence = ["check-lockfile", "build", "lint", "unit-test-with-cov"], help = "Build the package, lint and run unit tests. Does not include type-checking."} - -# Build and check -pre-push = {sequence = ["build", "check-local"], help = "Run all build and check tasks."} - -[tool.airbyte_ci] -python_versions = ["3.10", "3.11"] -optional_poetry_groups = ["dev"] -poetry_extras = ["file-based", "sphinx-docs", "vector-db-based"] -poe_tasks = ["check-ci"] -mount_docker_socket = true From 2a2f543cbd54f7f83e6f024100d65a0b37b5f1b0 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Mon, 11 Nov 2024 15:31:51 -0800 Subject: [PATCH 2/2] chore(linting): import rules from other PR --- pyproject.toml | 159 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index c2a4c4d00..dfffbaa65 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -103,9 +103,168 @@ skip = ["__init__.py"] # TODO: Remove after this is fixed: https://github.com/a [tool.ruff] exclude = ["__init__.py"] # TODO: Remove after this is fixed: https://github.com/airbytehq/airbyte-python-cdk/issues/12 +include = ["airbyte_cdk/**/*.py"] # Ignore test and script folders target-version = "py310" line-length = 100 +preview = true + +[tool.ruff.lint] +select = [ + # For rules reference, see https://docs.astral.sh/ruff/rules/ + "A", # flake8-builtins + "ANN", # flake8-annotations + "ARG", # flake8-unused-arguments + "ASYNC", # flake8-async + "B", # flake8-bugbear + "BLE", # Blind except + "C4", # flake8-comprehensions + "C90", # mccabe (complexity) + "COM", # flake8-commas + "CPY", # missing copyright notice + "DTZ", # flake8-datetimez + "E", # pycodestyle (errors) + "ERA", # flake8-eradicate (commented out code) + "EXE", # flake8-executable + "F", # Pyflakes + "FA", # flake8-future-annotations + "FBT", # flake8-boolean-trap + "FIX", # flake8-fixme + "FLY", # flynt + "FURB", # Refurb + "I", # isort + "ICN", # flake8-import-conventions + "INP", # flake8-no-pep420 + "INT", # flake8-gettext + "ISC", # flake8-implicit-str-concat + "LOG", # flake8-logging + "N", # pep8-naming + "PERF", # Perflint + "PGH", # pygrep-hooks + "PIE", # flake8-pie + "PL", # Pylint + "PT", # flake8-pytest-style + "PTH", # flake8-use-pathlib + "PYI", # flake8-pyi + "Q", # flake8-quotes + "RET", # flake8-return + "RSE", # flake8-raise + "RUF", # Ruff-specific rules + "SIM", # flake8-simplify + "SLF", # flake8-self + "SLOT", # flake8-slots + "T10", # debugger calls + "TCH", # flake8-type-checking + "TD", # flake8-todos + "TID", # flake8-tidy-imports + "TRY", # tryceratops + "TRY002", # Disallow raising vanilla Exception. Create or use a custom exception instead. + "UP", # pyupgrade + "W", # pycodestyle (warnings) + "YTT", # flake8-2020 +] +ignore = [ + # For rules reference, see https://docs.astral.sh/ruff/rules/ + + # Consider re-enabling these when we have time to address them: + "A003", # Class attribute 'type' is shadowing a Python builtin + "BLE001", # Do not catch blind exception: Exception + "C416", # Allow unnecessary-comprehensions. Auto-fix sometimes unsafe if operating over a mapping. + "DTZ005", # Allow use of 'datetime.datetime.now()' without timezone (we should fix these eventually) + "DTZ007", # Allow use of 'strptime()' without timezone (we should fix these eventually) + "D", # pydocstyle (Docstring conventions) + "D102", # Missing docstring in public method + "D103", # Missing docstring in public function + "E501", # Line too long + "ERA001", # Remove commented-out code + "FIX002", # Allow "TODO:" comments + "PGH003", # Allow non-specific "type: ignore" comments + "PLW0108", # Lambda may be unnecessary; consider inlining inner function + "PLW0603", # Using the global statement to update _cache is discouraged + "T20", # flake8-print, consider re-enabling once we have logging + "TD003", # Require issue links for TODOs + + # These we don't agree with or don't want to prioritize to enforce: + "ANN003", # kwargs missing type annotations + "ANN101", # Type annotations for 'self' args + "ANN102", # Type annotations for 'cls' args + "ASYNC1", # flake8-trio (opinionated, noisy) + "COM812", # Because it conflicts with ruff auto-format + "DJ", # Django linting + "EM", # flake8-errmsgs (may reconsider later) + "FURB189", # Subclassing safety at the cost if isinstance() instability + "G", # flake8-logging-format + "INP001", # Dir 'examples' is part of an implicit namespace package. Add an __init__.py. + "ISC001", # Conflicts with ruff auto-format + "N818", # Custom exception names should use the suffix "Error" + "NPY", # NumPy-specific rules + "N805", # Enforce first-arg is 'self' (false positive for class methods in Pydantic) + "PD", # pandas-vet + "PERF203", # exception handling in loop + "PIE790", # Allow unnecssary 'pass' (sometimes useful for readability) + "PLR6201", # Allow membership checks in lists (set-based check is unsafe when values are unhashable) + "PLR6301", # Allow class methods that don't use 'self' (otherwise noisy) + "RET504", # Ignore unnecessary assign before return + "RUF022", # Allow unsorted __all__ (sometimes useful for grouping by type with pdoc) + "S", # flake8-bandit (noisy, security related) + "SIM910", # Allow "None" as second argument to Dict.get(). "Explicit is better than implicit." + "TCH003", # Moving standard library imports under `TYPE_CHECKING` blocks is unsafe with Pydantic and Serpyco models + "TD002", # Require author for TODOs + "TRY003", # Allow string passing to exception constructor. + "TRY400", # Ignore for now: prefer logging.exception over logging.error +] +fixable = ["ALL"] +unfixable = [ + "ERA001", # Commented-out code (avoid silent loss of code) + "T201", # print() calls (avoid silent loss of code / log messages) + "TCH001", # Moving application imports under `TYPE_CHECKING` blocks is unsafe with Pydantic and Serpyco models + "TCH002", # Moving 3rd part imports under `TYPE_CHECKING` blocks is unsafe with Pydantic and Serpyco models +] +dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" + +[tool.ruff.lint.pylint] +max-args = 8 # Relaxed from default of 5 +max-branches = 15 # Relaxed from default of 12 + + +[tool.ruff.lint.isort] +force-sort-within-sections = false +lines-after-imports = 2 +known-first-party = [ + "airbyte_protocol", + "airbyte_protocol_dataclasses", +] +known-local-folder = ["airbyte_cdk"] +required-imports = ["from __future__ import annotations"] +known-third-party = [] +section-order = [ + "future", + "standard-library", + "third-party", + "first-party", + "local-folder", +] + +[tool.ruff.lint.mccabe] +max-complexity = 24 + +[tool.ruff.lint.pycodestyle] +ignore-overlong-task-comments = true + +[tool.ruff.lint.pydocstyle] +convention = "google" + +[tool.ruff.lint.flake8-annotations] +allow-star-arg-any = false +ignore-fully-untyped = false + +[tool.ruff.format] +quote-style = "double" +indent-style = "space" +skip-magic-trailing-comma = false +line-ending = "auto" +preview = false +docstring-code-format = true [tool.poe.tasks] # Build tasks