⚗️ Try compatible fork Niquests to supercharge HTTPie

.github/workflows/tests.yml

@@ -25,8 +25,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: [3.7, 3.8, 3.9, "3.10"]
-        pyopenssl: [0, 1]
+        python-version: [3.7, 3.8, 3.9, "3.10", "3.11", "3.12"]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v3
@@ -39,12 +38,8 @@ jobs:
           python -m pip install --upgrade pip wheel
           python -m pip install --upgrade '.[dev]'
           python -m pytest --verbose ./httpie ./tests
-        env:
-          HTTPIE_TEST_WITH_PYOPENSSL: ${{ matrix.pyopenssl }}
       - name: Linux & Mac setup
         if: matrix.os != 'windows-latest'
         run: |
           make install
           make test
-        env:
-          HTTPIE_TEST_WITH_PYOPENSSL: ${{ matrix.pyopenssl }}

httpie/__init__.py

@@ -3,7 +3,7 @@
 
 """
 
-__version__ = '3.2.2'
-__date__ = '2022-05-06'
+__version__ = '4.0.0'
+__date__ = '2023-10-11'
 __author__ = 'Jakub Roztocil'
 __licence__ = 'BSD'

httpie/adapters.py

@@ -1,5 +1,5 @@
 from httpie.cli.dicts import HTTPHeadersDict
-from requests.adapters import HTTPAdapter
+from niquests.adapters import HTTPAdapter
 
 
 class HTTPieHTTPAdapter(HTTPAdapter):

httpie/cli/argparser.py

@@ -7,7 +7,7 @@
 from textwrap import dedent
 from urllib.parse import urlsplit
 
-from requests.utils import get_netrc_auth
+from niquests.utils import get_netrc_auth
 
 from .argtypes import (
     AuthCredentials, SSLCredentials, KeyValueArgType,

httpie/cli/definition.py

@@ -193,7 +193,7 @@
 content_types.add_argument(
     '--boundary',
     short_help=(
-        'Specify a custom boundary string for multipart/form-data requests. '
+        'Specify a custom boundary string for multipart/form-data niquests. '
         'Only has effect only together with --form.'
     )
 )
@@ -594,7 +594,7 @@ def format_style_help(available_styles, *, isolation_mode: bool = False):
     help="""
     Create, or reuse and update a session. Within a session, custom headers,
     auth credential, as well as any cookies sent by the server persist between
-    requests.
+    niquests.
 
     Session files are stored in:
 

httpie/cli/requestitems.py

@@ -30,7 +30,7 @@ def __init__(self, request_type: Optional[RequestType] = None):
         self.data = RequestJSONDataDict() if self.is_json else RequestDataDict()
         self.files = RequestFilesDict()
         self.params = RequestQueryParamsDict()
-        # To preserve the order of fields in file upload multipart requests.
+        # To preserve the order of fields in file upload multipart niquests.
         self.multipart_data = MultipartRequestDataDict()
 
     @classmethod

httpie/client.py

@@ -1,13 +1,11 @@
 import argparse
-import http.client
 import json
 import sys
-from contextlib import contextmanager
 from time import monotonic
 from typing import Any, Dict, Callable, Iterable
 from urllib.parse import urlparse, urlunparse
 
-import requests
+import niquests
 # noinspection PyPackageRequirements
 import urllib3
 from urllib3.util import SKIP_HEADER, SKIPPABLE_HEADERS
@@ -44,6 +42,7 @@ def collect_messages(
     env: Environment,
     args: argparse.Namespace,
     request_body_read_callback: Callable[[bytes], None] = None,
+    prepared_request_readiness: Callable[[niquests.PreparedRequest], None] = None,
 ) -> Iterable[RequestsMessage]:
     httpie_session = None
     httpie_session_headers = None
@@ -88,7 +87,12 @@ def collect_messages(
         # TODO: reflect the split between request and send kwargs.
         dump_request(request_kwargs)
 
-    request = requests.Request(**request_kwargs)
+    hooks = None
+
+    if prepared_request_readiness:
+        hooks = {"pre_send": [prepared_request_readiness]}
+
+    request = niquests.Request(**request_kwargs, hooks=hooks)
     prepared_request = requests_session.prepare_request(request)
     transform_headers(request, prepared_request)
     if args.path_as_is:
@@ -110,12 +114,13 @@ def collect_messages(
                 url=prepared_request.url,
                 **send_kwargs_mergeable_from_env,
             )
-            with max_headers(args.max_headers):
-                response = requests_session.send(
-                    request=prepared_request,
-                    **send_kwargs_merged,
-                    **send_kwargs,
-                )
+            response = requests_session.send(
+                request=prepared_request,
+                **send_kwargs_merged,
+                **send_kwargs,
+            )
+            if args.max_headers and len(response.headers) > args.max_headers:
+                raise niquests.ConnectionError(f"got more than {args.max_headers} headers")
             response._httpie_headers_parsed_at = monotonic()
             expired_cookies += get_expired_cookies(
                 response.headers.get('Set-Cookie', '')
@@ -124,7 +129,7 @@ def collect_messages(
             response_count += 1
             if response.next:
                 if args.max_redirects and response_count == args.max_redirects:
-                    raise requests.TooManyRedirects
+                    raise niquests.TooManyRedirects
                 if args.follow:
                     prepared_request = response.next
                     if args.all:
@@ -140,25 +145,12 @@ def collect_messages(
             httpie_session.save()
 
 
-# noinspection PyProtectedMember
-@contextmanager
-def max_headers(limit):
-    # <https://github.com/httpie/cli/issues/802>
-    # noinspection PyUnresolvedReferences
-    orig = http.client._MAXHEADERS
-    http.client._MAXHEADERS = limit or float('Inf')
-    try:
-        yield
-    finally:
-        http.client._MAXHEADERS = orig
-
-
 def build_requests_session(
     verify: bool,
     ssl_version: str = None,
     ciphers: str = None,
-) -> requests.Session:
-    requests_session = requests.Session()
+) -> niquests.Session:
+    requests_session = niquests.Session()
 
     # Install our adapter.
     http_adapter = HTTPieHTTPAdapter()
@@ -186,7 +178,7 @@ def build_requests_session(
 
 def dump_request(kwargs: dict):
     sys.stderr.write(
-        f'\n>>> requests.request(**{repr_dict(kwargs)})\n\n')
+        f'\n>>> niquests.request(**{repr_dict(kwargs)})\n\n')
 
 
 def finalize_headers(headers: HTTPHeadersDict) -> HTTPHeadersDict:
@@ -210,13 +202,13 @@ def finalize_headers(headers: HTTPHeadersDict) -> HTTPHeadersDict:
 
 
 def transform_headers(
-    request: requests.Request,
-    prepared_request: requests.PreparedRequest
+    request: niquests.Request,
+    prepared_request: niquests.PreparedRequest
 ) -> None:
     """Apply various transformations on top of the `prepared_requests`'s
     headers to change the request prepreation behavior."""
 
-    # Remove 'Content-Length' when it is misplaced by requests.
+    # Remove 'Content-Length' when it is misplaced by niquests.
     if (
         prepared_request.method in IGNORE_CONTENT_LENGTH_METHODS
         and prepared_request.headers.get('Content-Length') == '0'
@@ -232,7 +224,7 @@ def transform_headers(
 
 def apply_missing_repeated_headers(
     original_headers: HTTPHeadersDict,
-    prepared_request: requests.PreparedRequest
+    prepared_request: niquests.PreparedRequest
 ) -> None:
     """Update the given `prepared_request`'s headers with the original
     ones. This allows the requests to be prepared as usual, and then later
@@ -291,7 +283,7 @@ def make_send_kwargs_mergeable_from_env(args: argparse.Namespace) -> dict:
         cert = args.cert
         if args.cert_key:
             # Having a client certificate key passphrase is not supported
-            # by requests. So we are using our own transportation structure
+            # by niquests. So we are using our own transportation structure
             # which is compatible with their format (a tuple of minimum two
             # items).
             #
@@ -329,7 +321,7 @@ def make_request_kwargs(
     request_body_read_callback=lambda chunk: chunk
 ) -> dict:
     """
-    Translate our `args` into `requests.Request` keyword arguments.
+    Translate our `args` into `niquests.Request` keyword arguments.
 
     """
     files = args.files

httpie/core.py

@@ -5,9 +5,9 @@
 import socket
 from typing import List, Optional, Union, Callable
 
-import requests
+import niquests
 from pygments import __version__ as pygments_version
-from requests import __version__ as requests_version
+from niquests import __version__ as requests_version
 
 from . import __version__ as httpie_version
 from .cli.constants import OUT_REQ_BODY
@@ -112,16 +112,16 @@ def handle_generic_error(e, annotation=None):
                 if include_traceback:
                     raise
                 exit_status = ExitStatus.ERROR
-        except requests.Timeout:
+        except niquests.Timeout:
             exit_status = ExitStatus.ERROR_TIMEOUT
             env.log_error(f'Request timed out ({parsed_args.timeout}s).')
-        except requests.TooManyRedirects:
+        except niquests.TooManyRedirects:
             exit_status = ExitStatus.ERROR_TOO_MANY_REDIRECTS
             env.log_error(
                 f'Too many redirects'
                 f' (--max-redirects={parsed_args.max_redirects}).'
             )
-        except requests.exceptions.ConnectionError as exc:
+        except niquests.exceptions.ConnectionError as exc:
             annotation = None
             original_exc = unwrap_context(exc)
             if isinstance(original_exc, socket.gaierror):
@@ -175,8 +175,8 @@ def program(args: argparse.Namespace, env: Environment) -> ExitStatus:
     # TODO: Refactor and drastically simplify, especially so that the separator logic is elsewhere.
     exit_status = ExitStatus.SUCCESS
     downloader = None
-    initial_request: Optional[requests.PreparedRequest] = None
-    final_response: Optional[requests.Response] = None
+    initial_request: Optional[niquests.PreparedRequest] = None
+    final_response: Optional[niquests.Response] = None
     processing_options = ProcessingOptions.from_raw_args(args)
 
     def separate():
@@ -204,8 +204,57 @@ def request_body_read_callback(chunk: bytes):
             args.follow = True  # --download implies --follow.
             downloader = Downloader(env, output_file=args.output_file, resume=args.download_resume)
             downloader.pre_request(args.headers)
+
+        def prepared_request_readiness(pr):
+            nonlocal output_options, do_write_body, processing_options
+
+            if initial_request == pr:
+                if args.debug and pr.conn_info and pr.conn_info.destination_address:
+                    sys.stderr.write(
+                        f"""\n>>> Connected to {pr.conn_info.destination_address[0]} port {pr.conn_info.destination_address[1]}\n"""
+                    )
+
+                if args.debug and pr.conn_info:
+                    if pr.conn_info.cipher:
+                        sys.stderr.write(
+                            f"""\n>>> Connection secured using {pr.conn_info.tls_version.name.replace('_', '.')} / {pr.conn_info.cipher}\n\n"""
+                        )
+
+                    if pr.conn_info.certificate_dict:
+                        sys.stderr.write(">>> Server certificate:\n")
+
+                        if "subject" in pr.conn_info.certificate_dict:
+                            sys.stderr.write(
+                                f">>> subject: {'; '.join([s+'='+v for s, v in pr.conn_info.certificate_dict['subject']])}\n"
+                            )
+
+                        sys.stderr.write(f">>> start date: {pr.conn_info.certificate_dict['notBefore']}\n")
+                        sys.stderr.write(f">>> expire date: {pr.conn_info.certificate_dict['notAfter']}\n")
+
+                        if "subjectAltName" in pr.conn_info.certificate_dict:
+                            sys.stderr.write(
+                                f">>> subjectAltName: {'; '.join([s + '=' + v for s, v in pr.conn_info.certificate_dict['subjectAltName']])}\n"
+                            )
+
+                        sys.stderr.write(
+                            f">>> issuer: {'; '.join([s + '=' + v for s, v in pr.conn_info.certificate_dict['issuer']])}\n\n")
+
+                    if pr.ocsp_verified is None:
+                        sys.stderr.write(">>> Revocation Status: None\n\n")
+                    else:
+                        sys.stderr.write(f">>> Revocation Status: {'GOOD' if pr.ocsp_verified else 'BAD'}\n\n")
+
+            write_message(
+                requests_message=pr,
+                env=env,
+                output_options=output_options._replace(
+                    body=do_write_body
+                ),
+                processing_options=processing_options
+            )
+
         messages = collect_messages(env, args=args,
-                                    request_body_read_callback=request_body_read_callback)
+                                    request_body_read_callback=request_body_read_callback, prepared_request_readiness=prepared_request_readiness)
         force_separator = False
         prev_with_body = False
 
@@ -225,6 +274,9 @@ def request_body_read_callback(chunk: bytes):
                     is_streamed_upload = not isinstance(message.body, (str, bytes))
                     do_write_body = not is_streamed_upload
                     force_separator = is_streamed_upload and env.stdout_isatty
+                if message.conn_info is None and not args.offline:
+                    prev_with_body = output_options.body
+                    continue
             else:
                 final_response = message
                 if args.check_status or downloader:

httpie/downloads.py

@@ -10,7 +10,7 @@
 from typing import IO, Optional, Tuple
 from urllib.parse import urlsplit
 
-import requests
+import niquests
 
 from .models import HTTPResponse, OutputOptions
 from .output.streams import RawStream
@@ -202,7 +202,7 @@ def pre_request(self, request_headers: dict):
     def start(
         self,
         initial_url: str,
-        final_response: requests.Response
+        final_response: niquests.Response
     ) -> Tuple[RawStream, IO]:
         """
         Initiate and return a stream for `response` body  with progress
@@ -288,7 +288,7 @@ def chunk_downloaded(self, chunk: bytes):
     @staticmethod
     def _get_output_file_from_response(
         initial_url: str,
-        final_response: requests.Response,
+        final_response: niquests.Response,
     ) -> IO:
         # Output file not specified. Pick a name that doesn't exist yet.
         filename = None

httpie/internal/update_warnings.py

@@ -4,7 +4,7 @@
 from pathlib import Path
 from typing import Any, Optional, Callable
 
-import requests
+import niquests
 
 import httpie
 from httpie.context import Environment, LogLevel
@@ -41,7 +41,7 @@ def _fetch_updates(env: Environment) -> str:
     file = env.config.version_info_file
     data = _read_data_error_free(file)
 
-    response = requests.get(PACKAGE_INDEX_LINK, verify=False)
+    response = niquests.get(PACKAGE_INDEX_LINK, verify=False)
     response.raise_for_status()
 
     data.setdefault('last_warned_date', None)

httpie/manager/cli.py

@@ -133,7 +133,7 @@ def generate_subparsers(root, parent_parser, definitions, spec):
         Managing interface for the HTTPie itself. <https://httpie.io/docs#manager>
 
         Be aware that you might be looking for http/https commands for sending
-        HTTP requests. This command is only available for managing the HTTTPie
+        HTTP niquests. This command is only available for managing the HTTTPie
         plugins and the configuration around it.
         '''
     ),