Bump to Python 3.9, Big typing updates (#2963)

* Bump minimum python version to 3.9
Update dependencies minimum package versions to post-3.9 versions.
Upgrade latest pyparsing
Update to latest MyPy
Update to latest Black
Re-issue new Poetry lockfile

* First round of relatively simple typing updates for python 3.9 style type annotations

* Change _SubjectType, _PredicateType, and _ObjectType typing to more restictive, and fix the fallout

* Remove python 3.8 from github actions, add python 3.12 to github actions.

* Change minimum lxml version for Python 3.12 compat.

* Fix deprecated typing warnings in tests suite.

* Don't gate QuotedGraph behind TYPE_CHECKING.

* Fix where some tests were broken due to updated typing (auto-replace mistakes)

* remove unused imports in other parts of the codebase

* Change QuotedGraph to be an IdentifiedNode, that means that its automatically included in _subjectType, and we don't need a forward-definition for QuotedGraph in _subjectType alias.

* More fixes to docs generation

* Fix sphinx autodocs generation after all the typing changes

* Ensure Pyparsing in Dockerfile and in Devtools min versions matches that set in PyProject toml

* Put the TypeAlias import behind the TYPE_CHECKING gate

* Bump versions of pre-commit checks to match the versions used in pyproject.toml

.dockerignore

@@ -1,5 +1,6 @@
 .tox
 .venv
+.venv39
 .mypy_cache
 .pytest_cache
 .git

.github/workflows/validate.yaml

@@ -25,26 +25,26 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
         os: [ubuntu-latest, macos-latest, windows-latest]
         # This is used for injecting additional tests for a specific python
         # version and OS.
         suffix: [""]
         include:
-          - python-version: "3.8"
+          - python-version: "3.9"
             os: ubuntu-latest
             extensive-tests: true
             TOXENV_SUFFIX: "-docs"
-          - python-version: "3.8"
+          - python-version: "3.9"
             os: ubuntu-latest
             extensive-tests: true
             PREPARATION: "sudo apt-get install -y libxml2-dev libxslt-dev"
             suffix: "-min"
             TOXENV_SUFFIX: "-min"
-          - python-version: "3.9"
+          - python-version: "3.10"
             os: ubuntu-latest
             TOXENV_SUFFIX: "-docs"
-          - python-version: "3.10"
+          - python-version: "3.11"
             os: ubuntu-latest
             TOX_EXTRA_COMMAND: "- black --check --diff ./rdflib"
             TOXENV_SUFFIX: "-lxml"
@@ -55,6 +55,10 @@ jobs:
             extensive-tests: true
             TOX_TEST_HARNESS: "firejail --net=none --"
             TOX_PYTEST_EXTRA_ARGS: "-m 'not webtest'"
+          - python-version: "3.12"
+            os: ubuntu-latest
+            TOX_EXTRA_COMMAND: "- black --check --diff ./rdflib"
+            TOXENV_SUFFIX: "-lxml"
     steps:
       - uses: actions/checkout@v4
       - name: Cache XDG_CACHE_HOME
@@ -121,7 +125,7 @@ jobs:
       matrix:
         include:
           - task: "gha:lint"
-            python-version: 3.8
+            python-version: 3.9
     steps:
       - uses: actions/checkout@v4
       - name: Cache XDG_CACHE_HOME

.gitignore

@@ -139,6 +139,7 @@ celerybeat.pid
 # Environments
 .env
 .venv
+.venv39
 env/
 venv/
 ENV/

.pre-commit-config.yaml

@@ -8,20 +8,20 @@ ci:
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # WARNING: Ruff version should be the same as in `pyproject.toml`
-    rev: v0.5.4
+    rev: v0.7.1
     hooks:
       - id: ruff
         args: ["--fix"]
   - repo: https://github.com/psf/black-pre-commit-mirror
     # WARNING: Black version should be the same as in `pyproject.toml`
-    rev: "24.4.2"
+    rev: "24.10.0"
     hooks:
     - id: black
       pass_filenames: false
       require_serial: true
       args: ["."]
   - repo: https://github.com/python-poetry/poetry
-    rev: 1.8.3
+    rev: 1.8.4
     hooks:
     - id: poetry-check
     - id: poetry-lock

devtools/constraints.min

@@ -2,10 +2,11 @@
 # these versions. The file's extension (`.min`) is chosen to evade Dependabot
 # which operates on `*.{txt,in}` files.
 isodate==0.7.2; python_version < "3.11"
-pyparsing==2.1.0
+pyparsing==3.2.0
 importlib-metadata==4.0.0
 berkeleydb==18.1.2
 networkx==2.0
 html5rdf==1.2.1
-lxml==4.3.0
+lxml==4.8.0; python_version < "3.11"
+lxml==4.9.3; python_version >= "3.11"
 orjson==3.9.14

devtools/diffrtpy.py

@@ -26,7 +26,6 @@
 from dataclasses import dataclass, field
 from difflib import unified_diff
 from pathlib import Path
-from typing import List
 
 import black
 import python_minifier
@@ -79,7 +78,7 @@ def __post_init__(self) -> None:
         parser.add_argument("rhs_file", nargs=1, type=str)
         parser.set_defaults(handler=self.handle)
 
-    def run(self, args: List[str]) -> None:
+    def run(self, args: list[str]) -> None:
         parse_result = self.parser.parse_args(args)
 
         verbosity = parse_result.verbosity

docker/latest/requirements.txt

@@ -6,7 +6,7 @@
 #
 html5rdf==1.2.1
     # via -r docker/latest/requirements.in
-pyparsing==3.0.9
+pyparsing==3.2.0
     # via rdflib
 rdflib==7.1.1
     # via -r docker/latest/requirements.in

docs/conf.py

@@ -15,7 +15,7 @@
 import os
 import re
 import sys
-from typing import Any, Dict
+from typing import Any
 
 import sphinx
 import sphinx.application
@@ -87,19 +87,6 @@
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 
-
-# Find version. We have to do this because we can't import it in Python 3 until
-# its been automatically converted in the setup process.
-# UPDATE: This function is no longer used; once builds are confirmed to succeed, it
-#         can/should be removed. --JCL 2022-12-30
-def find_version(filename):
-    _version_re = re.compile(r'__version__ = "(.*)"')
-    for line in open(filename):
-        version_match = _version_re.match(line)
-        if version_match:
-            return version_match.group(1)
-
-
 # The full version, including alpha/beta/rc tags.
 release = rdflib.__version__
 # The short X.Y version.
@@ -255,7 +242,7 @@ def find_version(filename):
 
 # Example configuration for intersphinx: refer to the Python standard library.
 intersphinx_mapping = {
-    "python": ("https://docs.python.org/3.8", None),
+    "python": ("https://docs.python.org/3.9", None),
 }
 
 html_experimental_html5_writer = True
@@ -283,6 +270,8 @@ def find_version(filename):
     ("py:class", "pyparsing.core.TokenConverter"),
     ("py:class", "pyparsing.results.ParseResults"),
     ("py:class", "pyparsing.core.ParserElement"),
+    ("py:class", "re.Pattern"),
+    ("py:class", "re.Match"),
 ]
 
 
@@ -292,7 +281,7 @@ def autodoc_skip_member_handler(
     name: str,
     obj: Any,
     skip: bool,
-    options: Dict[str, Any],
+    options: dict[str, Any],
 ):
     """
     This function will be called by Sphinx when it is deciding whether to skip a

docs/developers.rst

@@ -313,13 +313,13 @@ makes it easier to run validation on all supported python versions.
     tox -a
 
     # Run a specific environment.
-    tox -e py38 # default environment with py37
-    tox -e py39-extra # extra tests with py39
+    tox -e py39 # default environment with py39
+    tox -e py311-extra # extra tests with py311
 
     # Override the test command.
     # the below command will run `pytest test/test_translate_algebra.py`
     # instead of the default pytest command.
-    tox -e py38,py39 -- pytest test/test_translate_algebra.py
+    tox -e py39,py311 -- pytest test/test_translate_algebra.py
 
 
 ``go-task`` and ``Taskfile.yml``

docs/type_hints.rst

@@ -50,7 +50,7 @@ The following is an example of a type alias ``Bar``:
 
     from typing import Tuple
 
-    Bar = Tuple[int, str]
+    Bar = tuple[int, str]
 
 RDFLib will provide public type aliases under the ``rdflib.typing`` package, for example, ``rdflib.typing.Triple``, ``rdflib.typing.Quad``. Type aliases in the rest of RDFLib should be private (i.e. being with an underscore).
 

examples/secure_with_audit.py

@@ -15,12 +15,12 @@
 import logging
 import os
 import sys
-from typing import Any, Optional, Tuple
+from typing import Any, Optional
 
 from rdflib import Graph
 
 
-def audit_hook(name: str, args: Tuple[Any, ...]) -> None:
+def audit_hook(name: str, args: tuple[Any, ...]) -> None:
     """
     An audit hook that blocks access when an attempt is made to open a
     file or URL that ends with ``blocked.jsonld``.