Merge pull request #204 from scrapinghub/update-typing

Update mypy, fix new errors.

pyproject.toml

@@ -3,7 +3,6 @@ profile = "black"
 multi_line_output = 3
 
 [tool.mypy]
-show_error_codes = true
 ignore_missing_imports = true
 no_warn_no_return = true
 

tests/test_testing.py

@@ -3,7 +3,7 @@
 import sys
 from collections import deque
 from pathlib import Path
-from typing import Any, Optional
+from typing import Any, Dict, Optional
 
 import attrs
 import dateutil.tz
@@ -252,7 +252,7 @@ def _assert_frozen_item(
     pytester: "pytest.Pytester",
     response: HttpResponse,
     *,
-    outcomes: dict = None,
+    outcomes: Optional[Dict[str, int]] = None,
 ) -> None:
     # this makes an item with datetime fields corresponding to frozen_time
     item = ItemAdapter(_get_product_item(frozen_time)).asdict()

tests_typing/test_utils.mypy-testing

@@ -8,7 +8,7 @@ def test_memoizemethod_noargs():
     class Foo:
         @memoizemethod_noargs
         def meth(self) -> str:
-            ...
+            return ''
 
     foo = Foo()
     reveal_type(foo.meth())  # R: builtins.str
@@ -19,7 +19,7 @@ def test_cached_method_sync():
     class Foo:
         @cached_method
         def meth(self) -> str:
-            ...
+            return ''
 
     foo = Foo()
     reveal_type(foo.meth())  # R: builtins.str
@@ -30,7 +30,7 @@ async def test_cached_method_async():
     class Foo:
         @cached_method
         async def meth(self) -> str:
-            ...
+            return ''
 
     foo = Foo()
     reveal_type(await foo.meth())  # R: builtins.str

tox.ini

@@ -31,7 +31,7 @@ commands =
 
 [testenv:mypy]
 deps =
-    mypy==0.971
+    mypy==1.8.0
     types-requests
     types-python-dateutil
 
@@ -41,9 +41,7 @@ commands = mypy web_poet tests
 deps =
     {[testenv]deps}
     {[testenv:mypy]deps}
-    # waiting for https://github.com/davidfritzsche/pytest-mypy-testing/pull/31
-    pytest-mypy-testing @ https://github.com/kmike/pytest-mypy-testing/archive/refs/heads/async-support.zip
-;    pytest-mypy-testing==0.0.11
+    pytest-mypy-testing==0.1.3
 
 commands = py.test {posargs: tests_typing}
 

web_poet/exceptions/core.py

@@ -5,7 +5,9 @@
 These exceptions are tied to how **web-poet** operates.
 """
 
-from typing import TYPE_CHECKING
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Optional
 
 if TYPE_CHECKING:
     from web_poet import HttpRequest
@@ -55,7 +57,9 @@ class NoSavedHttpResponse(AssertionError):
     :type request: HttpRequest
     """
 
-    def __init__(self, msg: str = None, request: "HttpRequest" = None):
+    def __init__(
+        self, msg: Optional[str] = None, request: Optional[HttpRequest] = None
+    ):
         self.request = request
         if msg is None:
             msg = f"There is no saved response available for this HTTP Request: {self.request}"

web_poet/exceptions/http.py

@@ -24,7 +24,9 @@ class HttpError(IOError):
     :type request: HttpRequest
     """
 
-    def __init__(self, msg: str = None, request: HttpRequest = None):
+    def __init__(
+        self, msg: Optional[str] = None, request: Optional[HttpRequest] = None
+    ):
         #: Request that triggered the exception.
         self.request: Optional[HttpRequest] = request
         if msg is None:
@@ -66,9 +68,9 @@ class HttpResponseError(HttpError):
 
     def __init__(
         self,
-        msg: str = None,
-        response: HttpResponse = None,
-        request: HttpRequest = None,
+        msg: Optional[str] = None,
+        response: Optional[HttpResponse] = None,
+        request: Optional[HttpRequest] = None,
     ):
         #: Response that triggered the exception.
         self.response: Optional[HttpResponse] = response

web_poet/page_inputs/client.py

@@ -2,7 +2,7 @@
 import logging
 from dataclasses import dataclass
 from http import HTTPStatus
-from typing import Callable, Dict, Iterable, List, Optional, Union
+from typing import Callable, Dict, Iterable, List, Optional, Union, cast
 
 from web_poet.exceptions import HttpError, HttpResponseError
 from web_poet.exceptions.core import NoSavedHttpResponse
@@ -57,7 +57,7 @@ class HttpClient:
 
     def __init__(
         self,
-        request_downloader: Callable = None,
+        request_downloader: Optional[Callable] = None,
         *,
         save_responses: bool = False,
         return_only_saved_responses: bool = False,
@@ -75,7 +75,7 @@ def _handle_status(
         response: HttpResponse,
         request: HttpRequest,
         *,
-        allow_status: _StatusList = None,
+        allow_status: Optional[_StatusList] = None,
     ) -> None:
         allow_status_normalized = list(map(str, as_list(allow_status)))
         allow_all_status = any(
@@ -101,7 +101,7 @@ async def request(
         method: str = "GET",
         headers: Optional[_Headers] = None,
         body: Optional[_Body] = None,
-        allow_status: _StatusList = None,
+        allow_status: Optional[_StatusList] = None,
     ) -> HttpResponse:
         """This is a shortcut for creating an :class:`~.HttpRequest` instance and
         executing that request.
@@ -136,7 +136,7 @@ async def get(
         url: Union[str, _Url],
         *,
         headers: Optional[_Headers] = None,
-        allow_status: _StatusList = None,
+        allow_status: Optional[_StatusList] = None,
     ) -> HttpResponse:
         """Similar to :meth:`~.HttpClient.request` but peforming a ``GET``
         request.
@@ -154,7 +154,7 @@ async def post(
         *,
         headers: Optional[_Headers] = None,
         body: Optional[_Body] = None,
-        allow_status: _StatusList = None,
+        allow_status: Optional[_StatusList] = None,
     ) -> HttpResponse:
         """Similar to :meth:`~.HttpClient.request` but performing a ``POST``
         request.
@@ -168,7 +168,7 @@ async def post(
         )
 
     async def execute(
-        self, request: HttpRequest, *, allow_status: _StatusList = None
+        self, request: HttpRequest, *, allow_status: Optional[_StatusList] = None
     ) -> HttpResponse:
         """Execute the specified :class:`~.HttpRequest` instance using the
         request implementation configured in the :class:`~.HttpClient`
@@ -227,7 +227,7 @@ async def batch_execute(
         self,
         *requests: HttpRequest,
         return_exceptions: bool = False,
-        allow_status: _StatusList = None,
+        allow_status: Optional[_StatusList] = None,
     ) -> List[Union[HttpResponse, HttpResponseError]]:
         """Similar to :meth:`~.HttpClient.execute` but accepts a collection of
         :class:`~.HttpRequest` instances that would be batch executed.
@@ -260,7 +260,7 @@ async def batch_execute(
         responses = await asyncio.gather(
             *coroutines, return_exceptions=return_exceptions
         )
-        return responses
+        return cast(List[Union[HttpResponse, HttpResponseError]], responses)
 
     def get_saved_responses(self) -> Iterable[_SavedResponseData]:
         """Return saved requests and responses."""