From a18f79610141500cf540b2cf4783559731109d7d Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 24 Sep 2024 17:53:13 +0100 Subject: [PATCH 01/17] Drop unneccessary 'URL.raw' property (#3317) --- .github/workflows/test-suite.yml | 2 +- CHANGELOG.md | 4 ++++ httpx/_types.py | 11 ----------- httpx/_urls.py | 18 +----------------- tests/models/test_url.py | 16 ---------------- 5 files changed, 6 insertions(+), 45 deletions(-) diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index 0bb570cedb..ce3df5db81 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -5,7 +5,7 @@ on: push: branches: ["master"] pull_request: - branches: ["master"] + branches: ["master", "version-*"] jobs: tests: diff --git a/CHANGELOG.md b/CHANGELOG.md index f3aba3cc03..2692adc2b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## Unreleased + +* The `URL.raw` property has now been removed. + ## 0.27.2 (27th August, 2024) ### Fixed diff --git a/httpx/_types.py b/httpx/_types.py index 661af262e7..93eb996e2b 100644 --- a/httpx/_types.py +++ b/httpx/_types.py @@ -17,7 +17,6 @@ List, Mapping, MutableMapping, - NamedTuple, Optional, Sequence, Tuple, @@ -33,16 +32,6 @@ PrimitiveData = Optional[Union[str, int, float, bool]] -RawURL = NamedTuple( - "RawURL", - [ - ("raw_scheme", bytes), - ("raw_host", bytes), - ("port", Optional[int]), - ("raw_path", bytes), - ], -) - URLTypes = Union["URL", str] QueryParamTypes = Union[ diff --git a/httpx/_urls.py b/httpx/_urls.py index ec4ea6b399..a8752f013f 100644 --- a/httpx/_urls.py +++ b/httpx/_urls.py @@ -5,7 +5,7 @@ import idna -from ._types import QueryParamTypes, RawURL +from ._types import QueryParamTypes from ._urlparse import urlencode, urlparse from ._utils import primitive_value_to_str @@ -304,22 +304,6 @@ def fragment(self) -> str: """ return unquote(self._uri_reference.fragment or "") - @property - def raw(self) -> RawURL: - """ - Provides the (scheme, host, port, target) for the outgoing request. - - In older versions of `httpx` this was used in the low-level transport API. - We no longer use `RawURL`, and this property will be deprecated - in a future release. - """ - return RawURL( - self.raw_scheme, - self.raw_host, - self.port, - self.raw_path, - ) - @property def is_absolute_url(self) -> bool: """ diff --git a/tests/models/test_url.py b/tests/models/test_url.py index 523a89bf65..fa79acaf42 100644 --- a/tests/models/test_url.py +++ b/tests/models/test_url.py @@ -865,19 +865,3 @@ def test_ipv6_url_copy_with_host(url_str, new_host): assert url.host == "::ffff:192.168.0.1" assert url.netloc == b"[::ffff:192.168.0.1]:1234" assert str(url) == "http://[::ffff:192.168.0.1]:1234" - - -# Test for deprecated API - - -def test_url_raw_compatibility(): - """ - Test case for the (to-be-deprecated) `url.raw` accessor. - """ - url = httpx.URL("https://www.example.com/path") - scheme, host, port, raw_path = url.raw - - assert scheme == b"https" - assert host == b"www.example.com" - assert port is None - assert raw_path == b"/path" From 34e1110a9b3f8ff875fd2f61e1721cf68e18e564 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Tue, 24 Sep 2024 21:52:33 +0100 Subject: [PATCH 02/17] Remove deprecated 'app' parameter. (#3315) --- CHANGELOG.md | 1 + httpx/_client.py | 36 ++---------------------------------- tests/test_asgi.py | 10 ---------- tests/test_wsgi.py | 9 --------- 4 files changed, 3 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2692adc2b5..e7fc52bb14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased +* The deprecated `app` argument has now been removed. * The `URL.raw` property has now been removed. ## 0.27.2 (27th August, 2024) diff --git a/httpx/_client.py b/httpx/_client.py index 26610f6e87..45c326aee5 100644 --- a/httpx/_client.py +++ b/httpx/_client.py @@ -27,10 +27,8 @@ ) from ._models import Cookies, Headers, Request, Response from ._status_codes import codes -from ._transports.asgi import ASGITransport from ._transports.base import AsyncBaseTransport, BaseTransport from ._transports.default import AsyncHTTPTransport, HTTPTransport -from ._transports.wsgi import WSGITransport from ._types import ( AsyncByteStream, AuthTypes, @@ -616,8 +614,6 @@ class Client(BaseClient): request URLs. * **transport** - *(optional)* A transport class to use for sending requests over the network. - * **app** - *(optional)* An WSGI application to send requests to, - rather than sending actual network requests. * **trust_env** - *(optional)* Enables or disables usage of environment variables for configuration. * **default_encoding** - *(optional)* The default encoding to use for decoding @@ -646,7 +642,6 @@ def __init__( event_hooks: None | (typing.Mapping[str, list[EventHook]]) = None, base_url: URL | str = "", transport: BaseTransport | None = None, - app: typing.Callable[..., typing.Any] | None = None, trust_env: bool = True, default_encoding: str | typing.Callable[[bytes], str] = "utf-8", ) -> None: @@ -682,14 +677,7 @@ def __init__( if proxy: raise RuntimeError("Use either `proxy` or 'proxies', not both.") - if app: - message = ( - "The 'app' shortcut is now deprecated." - " Use the explicit style 'transport=WSGITransport(app=...)' instead." - ) - warnings.warn(message, DeprecationWarning) - - allow_env_proxies = trust_env and app is None and transport is None + allow_env_proxies = trust_env and transport is None proxy_map = self._get_proxy_map(proxies or proxy, allow_env_proxies) self._transport = self._init_transport( @@ -699,7 +687,6 @@ def __init__( http2=http2, limits=limits, transport=transport, - app=app, trust_env=trust_env, ) self._mounts: dict[URLPattern, BaseTransport | None] = { @@ -731,15 +718,11 @@ def _init_transport( http2: bool = False, limits: Limits = DEFAULT_LIMITS, transport: BaseTransport | None = None, - app: typing.Callable[..., typing.Any] | None = None, trust_env: bool = True, ) -> BaseTransport: if transport is not None: return transport - if app is not None: - return WSGITransport(app=app) - return HTTPTransport( verify=verify, cert=cert, @@ -1363,8 +1346,6 @@ class AsyncClient(BaseClient): request URLs. * **transport** - *(optional)* A transport class to use for sending requests over the network. - * **app** - *(optional)* An ASGI application to send requests to, - rather than sending actual network requests. * **trust_env** - *(optional)* Enables or disables usage of environment variables for configuration. * **default_encoding** - *(optional)* The default encoding to use for decoding @@ -1393,7 +1374,6 @@ def __init__( event_hooks: None | (typing.Mapping[str, list[EventHook]]) = None, base_url: URL | str = "", transport: AsyncBaseTransport | None = None, - app: typing.Callable[..., typing.Any] | None = None, trust_env: bool = True, default_encoding: str | typing.Callable[[bytes], str] = "utf-8", ) -> None: @@ -1429,14 +1409,7 @@ def __init__( if proxy: raise RuntimeError("Use either `proxy` or 'proxies', not both.") - if app: - message = ( - "The 'app' shortcut is now deprecated." - " Use the explicit style 'transport=ASGITransport(app=...)' instead." - ) - warnings.warn(message, DeprecationWarning) - - allow_env_proxies = trust_env and app is None and transport is None + allow_env_proxies = trust_env and transport is None proxy_map = self._get_proxy_map(proxies or proxy, allow_env_proxies) self._transport = self._init_transport( @@ -1446,7 +1419,6 @@ def __init__( http2=http2, limits=limits, transport=transport, - app=app, trust_env=trust_env, ) @@ -1478,15 +1450,11 @@ def _init_transport( http2: bool = False, limits: Limits = DEFAULT_LIMITS, transport: AsyncBaseTransport | None = None, - app: typing.Callable[..., typing.Any] | None = None, trust_env: bool = True, ) -> AsyncBaseTransport: if transport is not None: return transport - if app is not None: - return ASGITransport(app=app) - return AsyncHTTPTransport( verify=verify, cert=cert, diff --git a/tests/test_asgi.py b/tests/test_asgi.py index 8b817891e4..ffbc91bc00 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -222,13 +222,3 @@ async def test_asgi_exc_no_raise(): response = await client.get("http://www.example.org/") assert response.status_code == 500 - - -@pytest.mark.anyio -async def test_deprecated_shortcut(): - """ - The `app=...` shortcut is now deprecated. - Use the explicit transport style instead. - """ - with pytest.warns(DeprecationWarning): - httpx.AsyncClient(app=hello_world) diff --git a/tests/test_wsgi.py b/tests/test_wsgi.py index 0134bee854..dc2b52885a 100644 --- a/tests/test_wsgi.py +++ b/tests/test_wsgi.py @@ -201,12 +201,3 @@ def app(environ, start_response): assert response.status_code == 200 assert response.text == "success" assert server_protocol == "HTTP/1.1" - - -def test_deprecated_shortcut(): - """ - The `app=...` shortcut is now deprecated. - Use the explicit transport style instead. - """ - with pytest.warns(DeprecationWarning): - httpx.Client(app=application_factory([b"Hello, World!"])) From dece72d8cddde56060a3493843abc9ba7c85df81 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 26 Sep 2024 12:01:22 +0100 Subject: [PATCH 03/17] Remove deprecated 'proxies' parameter. (#3314) --- CHANGELOG.md | 1 + httpx/_api.py | 20 -------- httpx/_client.py | 39 ++------------ httpx/_types.py | 1 - tests/client/test_proxies.py | 98 +++++------------------------------- 5 files changed, 18 insertions(+), 141 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7fc52bb14..25669250b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased +* The deprecated `proxies` argument has now been removed. * The deprecated `app` argument has now been removed. * The `URL.raw` property has now been removed. diff --git a/httpx/_api.py b/httpx/_api.py index 4e98b60694..8afb0f2fc7 100644 --- a/httpx/_api.py +++ b/httpx/_api.py @@ -11,7 +11,6 @@ CertTypes, CookieTypes, HeaderTypes, - ProxiesTypes, ProxyTypes, QueryParamTypes, RequestContent, @@ -48,7 +47,6 @@ def request( cookies: CookieTypes | None = None, auth: AuthTypes | None = None, proxy: ProxyTypes | None = None, - proxies: ProxiesTypes | None = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, follow_redirects: bool = False, verify: VerifyTypes = True, @@ -80,7 +78,6 @@ def request( * **auth** - *(optional)* An authentication class to use when sending the request. * **proxy** - *(optional)* A proxy URL where all the traffic should be routed. - * **proxies** - *(optional)* A dictionary mapping proxy keys to proxy URLs. * **timeout** - *(optional)* The timeout configuration to use when sending the request. * **follow_redirects** - *(optional)* Enables or disables HTTP redirects. @@ -109,7 +106,6 @@ def request( with Client( cookies=cookies, proxy=proxy, - proxies=proxies, cert=cert, verify=verify, timeout=timeout, @@ -143,7 +139,6 @@ def stream( cookies: CookieTypes | None = None, auth: AuthTypes | None = None, proxy: ProxyTypes | None = None, - proxies: ProxiesTypes | None = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, follow_redirects: bool = False, verify: VerifyTypes = True, @@ -163,7 +158,6 @@ def stream( with Client( cookies=cookies, proxy=proxy, - proxies=proxies, cert=cert, verify=verify, timeout=timeout, @@ -192,7 +186,6 @@ def get( cookies: CookieTypes | None = None, auth: AuthTypes | None = None, proxy: ProxyTypes | None = None, - proxies: ProxiesTypes | None = None, follow_redirects: bool = False, cert: CertTypes | None = None, verify: VerifyTypes = True, @@ -215,7 +208,6 @@ def get( cookies=cookies, auth=auth, proxy=proxy, - proxies=proxies, follow_redirects=follow_redirects, cert=cert, verify=verify, @@ -232,7 +224,6 @@ def options( cookies: CookieTypes | None = None, auth: AuthTypes | None = None, proxy: ProxyTypes | None = None, - proxies: ProxiesTypes | None = None, follow_redirects: bool = False, cert: CertTypes | None = None, verify: VerifyTypes = True, @@ -255,7 +246,6 @@ def options( cookies=cookies, auth=auth, proxy=proxy, - proxies=proxies, follow_redirects=follow_redirects, cert=cert, verify=verify, @@ -272,7 +262,6 @@ def head( cookies: CookieTypes | None = None, auth: AuthTypes | None = None, proxy: ProxyTypes | None = None, - proxies: ProxiesTypes | None = None, follow_redirects: bool = False, cert: CertTypes | None = None, verify: VerifyTypes = True, @@ -295,7 +284,6 @@ def head( cookies=cookies, auth=auth, proxy=proxy, - proxies=proxies, follow_redirects=follow_redirects, cert=cert, verify=verify, @@ -316,7 +304,6 @@ def post( cookies: CookieTypes | None = None, auth: AuthTypes | None = None, proxy: ProxyTypes | None = None, - proxies: ProxiesTypes | None = None, follow_redirects: bool = False, cert: CertTypes | None = None, verify: VerifyTypes = True, @@ -340,7 +327,6 @@ def post( cookies=cookies, auth=auth, proxy=proxy, - proxies=proxies, follow_redirects=follow_redirects, cert=cert, verify=verify, @@ -361,7 +347,6 @@ def put( cookies: CookieTypes | None = None, auth: AuthTypes | None = None, proxy: ProxyTypes | None = None, - proxies: ProxiesTypes | None = None, follow_redirects: bool = False, cert: CertTypes | None = None, verify: VerifyTypes = True, @@ -385,7 +370,6 @@ def put( cookies=cookies, auth=auth, proxy=proxy, - proxies=proxies, follow_redirects=follow_redirects, cert=cert, verify=verify, @@ -406,7 +390,6 @@ def patch( cookies: CookieTypes | None = None, auth: AuthTypes | None = None, proxy: ProxyTypes | None = None, - proxies: ProxiesTypes | None = None, follow_redirects: bool = False, cert: CertTypes | None = None, verify: VerifyTypes = True, @@ -430,7 +413,6 @@ def patch( cookies=cookies, auth=auth, proxy=proxy, - proxies=proxies, follow_redirects=follow_redirects, cert=cert, verify=verify, @@ -447,7 +429,6 @@ def delete( cookies: CookieTypes | None = None, auth: AuthTypes | None = None, proxy: ProxyTypes | None = None, - proxies: ProxiesTypes | None = None, follow_redirects: bool = False, cert: CertTypes | None = None, verify: VerifyTypes = True, @@ -470,7 +451,6 @@ def delete( cookies=cookies, auth=auth, proxy=proxy, - proxies=proxies, follow_redirects=follow_redirects, cert=cert, verify=verify, diff --git a/httpx/_client.py b/httpx/_client.py index 45c326aee5..f924a311e7 100644 --- a/httpx/_client.py +++ b/httpx/_client.py @@ -35,7 +35,6 @@ CertTypes, CookieTypes, HeaderTypes, - ProxiesTypes, ProxyTypes, QueryParamTypes, RequestContent, @@ -209,23 +208,17 @@ def _enforce_trailing_slash(self, url: URL) -> URL: return url.copy_with(raw_path=url.raw_path + b"/") def _get_proxy_map( - self, proxies: ProxiesTypes | None, allow_env_proxies: bool + self, proxy: ProxyTypes | None, allow_env_proxies: bool ) -> dict[str, Proxy | None]: - if proxies is None: + if proxy is None: if allow_env_proxies: return { key: None if url is None else Proxy(url=url) for key, url in get_environment_proxies().items() } return {} - if isinstance(proxies, dict): - new_proxies = {} - for key, value in proxies.items(): - proxy = Proxy(url=value) if isinstance(value, (str, URL)) else value - new_proxies[str(key)] = proxy - return new_proxies else: - proxy = Proxy(url=proxies) if isinstance(proxies, (str, URL)) else proxies + proxy = Proxy(url=proxy) if isinstance(proxy, (str, URL)) else proxy return {"all://": proxy} @property @@ -633,7 +626,6 @@ def __init__( http1: bool = True, http2: bool = False, proxy: ProxyTypes | None = None, - proxies: ProxiesTypes | None = None, mounts: None | (typing.Mapping[str, BaseTransport | None]) = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, follow_redirects: bool = False, @@ -668,17 +660,8 @@ def __init__( "Make sure to install httpx using `pip install httpx[http2]`." ) from None - if proxies: - message = ( - "The 'proxies' argument is now deprecated." - " Use 'proxy' or 'mounts' instead." - ) - warnings.warn(message, DeprecationWarning) - if proxy: - raise RuntimeError("Use either `proxy` or 'proxies', not both.") - allow_env_proxies = trust_env and transport is None - proxy_map = self._get_proxy_map(proxies or proxy, allow_env_proxies) + proxy_map = self._get_proxy_map(proxy, allow_env_proxies) self._transport = self._init_transport( verify=verify, @@ -1335,8 +1318,6 @@ class AsyncClient(BaseClient): * **http2** - *(optional)* A boolean indicating if HTTP/2 support should be enabled. Defaults to `False`. * **proxy** - *(optional)* A proxy URL where all the traffic should be routed. - * **proxies** - *(optional)* A dictionary mapping HTTP protocols to proxy - URLs. * **timeout** - *(optional)* The timeout configuration to use when sending requests. * **limits** - *(optional)* The limits configuration to use. @@ -1365,7 +1346,6 @@ def __init__( http1: bool = True, http2: bool = False, proxy: ProxyTypes | None = None, - proxies: ProxiesTypes | None = None, mounts: None | (typing.Mapping[str, AsyncBaseTransport | None]) = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, follow_redirects: bool = False, @@ -1400,17 +1380,8 @@ def __init__( "Make sure to install httpx using `pip install httpx[http2]`." ) from None - if proxies: - message = ( - "The 'proxies' argument is now deprecated." - " Use 'proxy' or 'mounts' instead." - ) - warnings.warn(message, DeprecationWarning) - if proxy: - raise RuntimeError("Use either `proxy` or 'proxies', not both.") - allow_env_proxies = trust_env and transport is None - proxy_map = self._get_proxy_map(proxies or proxy, allow_env_proxies) + proxy_map = self._get_proxy_map(proxy, allow_env_proxies) self._transport = self._init_transport( verify=verify, diff --git a/httpx/_types.py b/httpx/_types.py index 93eb996e2b..e169172559 100644 --- a/httpx/_types.py +++ b/httpx/_types.py @@ -68,7 +68,6 @@ "Timeout", ] ProxyTypes = Union["URL", str, "Proxy"] -ProxiesTypes = Union[ProxyTypes, Dict[Union["URL", str], Union[None, ProxyTypes]]] AuthTypes = Union[ Tuple[Union[str, bytes], Union[str, bytes]], diff --git a/tests/client/test_proxies.py b/tests/client/test_proxies.py index 7bba1ab2c3..90a92f56bb 100644 --- a/tests/client/test_proxies.py +++ b/tests/client/test_proxies.py @@ -13,57 +13,6 @@ def url_to_origin(url: str) -> httpcore.URL: return httpcore.URL(scheme=u.raw_scheme, host=u.raw_host, port=u.port, target="/") -@pytest.mark.parametrize( - ["proxies", "expected_proxies"], - [ - ("http://127.0.0.1", [("all://", "http://127.0.0.1")]), - ({"all://": "http://127.0.0.1"}, [("all://", "http://127.0.0.1")]), - ( - {"http://": "http://127.0.0.1", "https://": "https://127.0.0.1"}, - [("http://", "http://127.0.0.1"), ("https://", "https://127.0.0.1")], - ), - (httpx.Proxy("http://127.0.0.1"), [("all://", "http://127.0.0.1")]), - ( - { - "https://": httpx.Proxy("https://127.0.0.1"), - "all://": "http://127.0.0.1", - }, - [("all://", "http://127.0.0.1"), ("https://", "https://127.0.0.1")], - ), - ], -) -def test_proxies_parameter(proxies, expected_proxies): - with pytest.warns(DeprecationWarning): - client = httpx.Client(proxies=proxies) - client_patterns = [p.pattern for p in client._mounts.keys()] - client_proxies = list(client._mounts.values()) - - for proxy_key, url in expected_proxies: - assert proxy_key in client_patterns - proxy = client_proxies[client_patterns.index(proxy_key)] - assert isinstance(proxy, httpx.HTTPTransport) - assert isinstance(proxy._pool, httpcore.HTTPProxy) - assert proxy._pool._proxy_url == url_to_origin(url) - - assert len(expected_proxies) == len(client._mounts) - - -def test_socks_proxy_deprecated(): - url = httpx.URL("http://www.example.com") - - with pytest.warns(DeprecationWarning): - client = httpx.Client(proxies="socks5://localhost/") - transport = client._transport_for_url(url) - assert isinstance(transport, httpx.HTTPTransport) - assert isinstance(transport._pool, httpcore.SOCKSProxy) - - with pytest.warns(DeprecationWarning): - async_client = httpx.AsyncClient(proxies="socks5://localhost/") - async_transport = async_client._transport_for_url(url) - assert isinstance(async_transport, httpx.AsyncHTTPTransport) - assert isinstance(async_transport._pool, httpcore.AsyncSOCKSProxy) - - def test_socks_proxy(): url = httpx.URL("http://www.example.com") @@ -84,7 +33,6 @@ def test_socks_proxy(): @pytest.mark.parametrize( ["url", "proxies", "expected"], [ - ("http://example.com", None, None), ("http://example.com", {}, None), ("http://example.com", {"https://": PROXY_URL}, None), ("http://example.com", {"http://example.net": PROXY_URL}, None), @@ -104,7 +52,6 @@ def test_socks_proxy(): # ... ("http://example.com:443", {"http://example.com": PROXY_URL}, PROXY_URL), ("http://example.com", {"all://": PROXY_URL}, PROXY_URL), - ("http://example.com", {"all://": PROXY_URL, "http://example.com": None}, None), ("http://example.com", {"http://": PROXY_URL}, PROXY_URL), ("http://example.com", {"all://example.com": PROXY_URL}, PROXY_URL), ("http://example.com", {"http://example.com": PROXY_URL}, PROXY_URL), @@ -138,11 +85,8 @@ def test_socks_proxy(): ], ) def test_transport_for_request(url, proxies, expected): - if proxies: - with pytest.warns(DeprecationWarning): - client = httpx.Client(proxies=proxies) - else: - client = httpx.Client(proxies=proxies) + mounts = {key: httpx.HTTPTransport(proxy=value) for key, value in proxies.items()} + client = httpx.Client(mounts=mounts) transport = client._transport_for_url(httpx.URL(url)) @@ -158,8 +102,8 @@ def test_transport_for_request(url, proxies, expected): @pytest.mark.network async def test_async_proxy_close(): try: - with pytest.warns(DeprecationWarning): - client = httpx.AsyncClient(proxies={"https://": PROXY_URL}) + transport = httpx.AsyncHTTPTransport(proxy=PROXY_URL) + client = httpx.AsyncClient(mounts={"https://": transport}) await client.get("http://example.com") finally: await client.aclose() @@ -168,18 +112,13 @@ async def test_async_proxy_close(): @pytest.mark.network def test_sync_proxy_close(): try: - with pytest.warns(DeprecationWarning): - client = httpx.Client(proxies={"https://": PROXY_URL}) + transport = httpx.HTTPTransport(proxy=PROXY_URL) + client = httpx.Client(mounts={"https://": transport}) client.get("http://example.com") finally: client.close() -def test_unsupported_proxy_scheme_deprecated(): - with pytest.warns(DeprecationWarning), pytest.raises(ValueError): - httpx.Client(proxies="ftp://127.0.0.1") - - def test_unsupported_proxy_scheme(): with pytest.raises(ValueError): httpx.Client(proxy="ftp://127.0.0.1") @@ -308,26 +247,13 @@ def test_proxies_environ(monkeypatch, client_class, url, env, expected): ], ) def test_for_deprecated_proxy_params(proxies, is_valid): - with pytest.warns(DeprecationWarning): - if not is_valid: - with pytest.raises(ValueError): - httpx.Client(proxies=proxies) - else: - httpx.Client(proxies=proxies) + mounts = {key: httpx.HTTPTransport(proxy=value) for key, value in proxies.items()} - -def test_proxy_and_proxies_together(): - with pytest.warns(DeprecationWarning), pytest.raises( - RuntimeError, - ): - httpx.Client(proxies={"all://": "http://127.0.0.1"}, proxy="http://127.0.0.1") - - with pytest.warns(DeprecationWarning), pytest.raises( - RuntimeError, - ): - httpx.AsyncClient( - proxies={"all://": "http://127.0.0.1"}, proxy="http://127.0.0.1" - ) + if not is_valid: + with pytest.raises(ValueError): + httpx.Client(mounts=mounts) + else: + httpx.Client(mounts=mounts) def test_proxy_with_mounts(): From 7a046618f748c24ab05bd2adf683b8c24a72887b Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 27 Sep 2024 10:55:41 +0100 Subject: [PATCH 04/17] Drop `sniffio` requirement. (#3323) --- CHANGELOG.md | 1 + httpx/_transports/asgi.py | 25 +++++++++++++++++++------ httpx/_utils.py | 17 ++--------------- pyproject.toml | 1 - 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25669250b5..50c5a5de83 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). * The deprecated `proxies` argument has now been removed. * The deprecated `app` argument has now been removed. * The `URL.raw` property has now been removed. +* The `sniffio` project dependency has now been removed. ## 0.27.2 (27th August, 2024) diff --git a/httpx/_transports/asgi.py b/httpx/_transports/asgi.py index 8578d4aeff..2bc4efae0e 100644 --- a/httpx/_transports/asgi.py +++ b/httpx/_transports/asgi.py @@ -2,8 +2,6 @@ import typing -import sniffio - from .._models import Request, Response from .._types import AsyncByteStream from .base import AsyncBaseTransport @@ -28,15 +26,30 @@ __all__ = ["ASGITransport"] +def is_running_trio() -> bool: + try: + # sniffio is a dependency of trio. + + # See https://github.com/python-trio/trio/issues/2802 + import sniffio + + if sniffio.current_async_library() == "trio": + return True + except ImportError: # pragma: nocover + pass + + return False + + def create_event() -> Event: - if sniffio.current_async_library() == "trio": + if is_running_trio(): import trio return trio.Event() - else: - import asyncio - return asyncio.Event() + import asyncio + + return asyncio.Event() class ASGIResponseStream(AsyncByteStream): diff --git a/httpx/_utils.py b/httpx/_utils.py index a9ece19438..cf7f5bcad5 100644 --- a/httpx/_utils.py +++ b/httpx/_utils.py @@ -11,8 +11,6 @@ from pathlib import Path from urllib.request import getproxies -import sniffio - from ._types import PrimitiveData if typing.TYPE_CHECKING: # pragma: no cover @@ -289,29 +287,18 @@ def peek_filelike_length(stream: typing.Any) -> int | None: class Timer: - async def _get_time(self) -> float: - library = sniffio.current_async_library() - if library == "trio": - import trio - - return trio.current_time() - else: - import asyncio - - return asyncio.get_event_loop().time() - def sync_start(self) -> None: self.started = time.perf_counter() async def async_start(self) -> None: - self.started = await self._get_time() + self.started = time.perf_counter() def sync_elapsed(self) -> float: now = time.perf_counter() return now - self.started async def async_elapsed(self) -> float: - now = await self._get_time() + now = time.perf_counter() return now - self.started diff --git a/pyproject.toml b/pyproject.toml index c4c188052e..2b83de5f12 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,6 @@ dependencies = [ "httpcore==1.*", "anyio", "idna", - "sniffio", ] dynamic = ["readme", "version"] From 2763690e1b2c0f72505397f34fc3c7dc2971955a Mon Sep 17 00:00:00 2001 From: Kar Petrosyan <92274156+karpetrosyan@users.noreply.github.com> Date: Tue, 8 Oct 2024 14:00:21 +0400 Subject: [PATCH 05/17] Add `httpx.SSLContext` configuration. (#3022) Co-authored-by: Tom Christie Co-authored-by: T-256 <132141463+T-256@users.noreply.github.com> --- .github/CONTRIBUTING.md | 4 +- CHANGELOG.md | 2 + docs/advanced/ssl.md | 220 ++++++++++++++++++++++++++-------- docs/compatibility.md | 4 +- docs/environment_variables.md | 60 ---------- docs/logging.md | 38 +++--- httpx/__init__.py | 2 +- httpx/_api.py | 67 ++++------- httpx/_client.py | 73 +++-------- httpx/_config.py | 155 ++++++++---------------- httpx/_main.py | 7 +- httpx/_transports/default.py | 16 ++- httpx/_types.py | 3 +- httpx/_utils.py | 13 -- tests/conftest.py | 20 ---- tests/test_config.py | 86 ++++--------- tests/test_utils.py | 45 +------ 17 files changed, 327 insertions(+), 488 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index e1a953dc97..f8a8ac10e0 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -211,9 +211,9 @@ this is where our previously generated `client.pem` comes in: ``` import httpx -proxies = {"all": "http://127.0.0.1:8080/"} +ssl_context = httpx.SSLContext(verify="/path/to/client.pem")) -with httpx.Client(proxies=proxies, verify="/path/to/client.pem") as client: +with httpx.Client(proxy="http://127.0.0.1:8080/", ssl_context=ssl_context) as client: response = client.get("https://example.org") print(response.status_code) # should print 200 ``` diff --git a/CHANGELOG.md b/CHANGELOG.md index 50c5a5de83..208cb1692d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased +* Added `httpx.SSLContext` class and `ssl_context` argument. (#3022) +* Removed `cert` and `verify` arguments, you should use the `ssl_context=...` instead. (#3022) * The deprecated `proxies` argument has now been removed. * The deprecated `app` argument has now been removed. * The `URL.raw` property has now been removed. diff --git a/docs/advanced/ssl.md b/docs/advanced/ssl.md index d96bbe1979..7eed687196 100644 --- a/docs/advanced/ssl.md +++ b/docs/advanced/ssl.md @@ -1,100 +1,224 @@ When making a request over HTTPS, HTTPX needs to verify the identity of the requested host. To do this, it uses a bundle of SSL certificates (a.k.a. CA bundle) delivered by a trusted certificate authority (CA). -## Changing the verification defaults +### Enabling and disabling verification -By default, HTTPX uses the CA bundle provided by [Certifi](https://pypi.org/project/certifi/). This is what you want in most cases, even though some advanced situations may require you to use a different set of certificates. +By default httpx will verify HTTPS connections, and raise an error for invalid SSL cases... -If you'd like to use a custom CA bundle, you can use the `verify` parameter. +```pycon +>>> httpx.get("https://expired.badssl.com/") +httpx.ConnectError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:997) +``` -```python -import httpx +You can configure the verification using `httpx.SSLContext()`. -r = httpx.get("https://example.org", verify="path/to/client.pem") +```pycon +>>> ssl_context = httpx.SSLContext() +>>> ssl_context +SSLContext(verify=True) +>>> httpx.get("https://www.example.com", ssl_context=ssl_context) +httpx.ConnectError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:997) ``` -Alternatively, you can pass a standard library `ssl.SSLContext`. +For example, you can use this to disable verification completely and allow insecure requests... ```pycon ->>> import ssl ->>> import httpx ->>> context = ssl.create_default_context() ->>> context.load_verify_locations(cafile="/tmp/client.pem") ->>> httpx.get('https://example.org', verify=context) +>>> no_verify = httpx.SSLContext(verify=False) +>>> no_verify +SSLContext(verify=False) +>>> httpx.get("https://expired.badssl.com/", ssl_context=no_verify) ``` -We also include a helper function for creating properly configured `SSLContext` instances. +### Configuring client instances + +If you're using a `Client()` instance, then you should pass any SSL settings when instantiating the client. ```pycon ->>> context = httpx.create_ssl_context() +>>> ssl_context = httpx.SSLContext() +>>> client = httpx.Client(ssl_context=ssl_context) ``` -The `create_ssl_context` function accepts the same set of SSL configuration arguments -(`trust_env`, `verify`, `cert` and `http2` arguments) -as `httpx.Client` or `httpx.AsyncClient` +The `client.get(...)` method and other request methods on a `Client` instance *do not* support changing the SSL settings on a per-request basis. + +If you need different SSL settings in different cases you should use more that one client instance, with different settings on each. Each client will then be using an isolated connection pool with a specific fixed SSL configuration on all connections within that pool. + +### Changing the verification defaults + +By default, HTTPX uses the CA bundle provided by [Certifi](https://pypi.org/project/certifi/). + +The following all have the same behaviour... + +Using the default SSL context. ```pycon ->>> import httpx ->>> context = httpx.create_ssl_context(verify="/tmp/client.pem") ->>> httpx.get('https://example.org', verify=context) +>>> client = httpx.Client() +>>> client.get("https://www.example.com") ``` -Or you can also disable the SSL verification entirely, which is _not_ recommended. +Using the default SSL context, but specified explicitly. -```python -import httpx +```pycon +>>> default = httpx.SSLContext() +>>> client = httpx.Client(ssl_context=default) +>>> client.get("https://www.example.com") + +``` -r = httpx.get("https://example.org", verify=False) +Using the default SSL context, with `verify=True` specified explicitly. + +```pycon +>>> default = httpx.SSLContext(verify=True) +>>> client = httpx.Client(ssl_context=default) +>>> client.get("https://www.example.com") + ``` -## SSL configuration on client instances +Using an SSL context, with `certifi.where()` explicitly specified. -If you're using a `Client()` instance, then you should pass any SSL settings when instantiating the client. +```pycon +>>> default = httpx.SSLContext(verify=certifi.where()) +>>> client = httpx.Client(ssl_context=default) +>>> client.get("https://www.example.com") + +``` -```python -client = httpx.Client(verify=False) +For some advanced situations may require you to use a different set of certificates, either by specifying a PEM file: + +```pycon +>>> custom_cafile = httpx.SSLContext(verify="path/to/certs.pem") +>>> client = httpx.Client(ssl_context=custom_cafile) +>>> client.get("https://www.example.com") + +``` + +Or by providing an certificate directory: + +```pycon +>>> custom_capath = httpx.SSLContext(verify="path/to/certs") +>>> client = httpx.Client(ssl_context=custom_capath) +>>> client.get("https://www.example.com") + +``` + +These usages are equivelent to using [`.load_verify_locations()`](https://docs.python.org/3/library/ssl.html#ssl.SSLContext.load_verify_locations) with either `cafile=...` or `capath=...`. + +### Client side certificates + +You can also specify a local cert to use as a client-side certificate, either a path to an SSL certificate file... + +```pycon +>>> cert = "path/to/client.pem" +>>> ssl_context = httpx.SSLContext(cert=cert) +>>> httpx.get("https://example.org", ssl_context=ssl_context) + ``` -The `client.get(...)` method and other request methods *do not* support changing the SSL settings on a per-request basis. If you need different SSL settings in different cases you should use more that one client instance, with different settings on each. Each client will then be using an isolated connection pool with a specific fixed SSL configuration on all connections within that pool. +Or two-tuple of (certificate file, key file)... -## Client Side Certificates +```pycon +>>> cert = ("path/to/client.pem", "path/to/client.key") +>>> ssl_context = httpx.SSLContext(cert=cert) +>>> httpx.get("https://example.org", ssl_context=ssl_context) + +``` -You can also specify a local cert to use as a client-side certificate, either a path to an SSL certificate file, or two-tuple of (certificate file, key file), or a three-tuple of (certificate file, key file, password) +Or a three-tuple of (certificate file, key file, password)... + +```pycon +>>> cert = ("path/to/client.pem", "path/to/client.key", "password") +>>> ssl_context = httpx.SSLContext(cert=cert) +>>> httpx.get("https://example.org", ssl_context=ssl_context) + +``` + +These configurations are equivalent to using [`.load_cert_chain()`](https://docs.python.org/3/library/ssl.html#ssl.SSLContext.load_cert_chain). + +### Using alternate SSL contexts + +You can also use an alternate `ssl.SSLContext` instances. + +For example, [using the `truststore` package](https://truststore.readthedocs.io/)... ```python -cert = "path/to/client.pem" -client = httpx.Client(cert=cert) -response = client.get("https://example.org") +import ssl +import truststore +import httpx + +ssl_context = truststore.SSLContext(ssl.PROTOCOL_TLS_CLIENT) +client = httpx.Client(ssl_context=ssl_context) ``` -Alternatively... +Or working [directly with Python's standard library](https://docs.python.org/3/library/ssl.html)... ```python -cert = ("path/to/client.pem", "path/to/client.key") -client = httpx.Client(cert=cert) -response = client.get("https://example.org") +import ssl +import httpx + +ssl_context = ssl.create_default_context() +client = httpx.Client(ssl_context=ssl_context) ``` -Or... +### Working with `SSL_CERT_FILE` and `SSL_CERT_DIR` + +Unlike `requests`, the `httpx` package does not automatically pull in [the environment variables `SSL_CERT_FILE` or `SSL_CERT_DIR`](https://www.openssl.org/docs/manmaster/man3/SSL_CTX_set_default_verify_paths.html). If you want to use these they need to be enabled explicitly. + +For example... + +```python +# Use `SSL_CERT_FILE` or `SSL_CERT_DIR` if configured, otherwise use certifi. +verify = os.environ.get("SSL_CERT_FILE", os.environ.get("SSL_CERT_DIR", True)) +ssl_context = httpx.SSLContext(verify=verify) +``` + +## `SSLKEYLOGFILE` + +Valid values: a filename + +If this environment variable is set, TLS keys will be appended to the specified file, creating it if it doesn't exist, whenever key material is generated or received. The keylog file is designed for debugging purposes only. + +Support for `SSLKEYLOGFILE` requires Python 3.8 and OpenSSL 1.1.1 or newer. + +Example: ```python -cert = ("path/to/client.pem", "path/to/client.key", "password") -client = httpx.Client(cert=cert) -response = client.get("https://example.org") +# test_script.py +import httpx + +with httpx.Client() as client: + r = client.get("https://google.com") ``` -## Making HTTPS requests to a local server +```console +SSLKEYLOGFILE=test.log python test_script.py +cat test.log +# TLS secrets log file, generated by OpenSSL / Python +SERVER_HANDSHAKE_TRAFFIC_SECRET XXXX +EXPORTER_SECRET XXXX +SERVER_TRAFFIC_SECRET_0 XXXX +CLIENT_HANDSHAKE_TRAFFIC_SECRET XXXX +CLIENT_TRAFFIC_SECRET_0 XXXX +SERVER_HANDSHAKE_TRAFFIC_SECRET XXXX +EXPORTER_SECRET XXXX +SERVER_TRAFFIC_SECRET_0 XXXX +CLIENT_HANDSHAKE_TRAFFIC_SECRET XXXX +CLIENT_TRAFFIC_SECRET_0 XXXX +``` + +### Making HTTPS requests to a local server When making requests to local servers, such as a development server running on `localhost`, you will typically be using unencrypted HTTP connections. If you do need to make HTTPS connections to a local server, for example to test an HTTPS-only service, you will need to create and use your own certificates. Here's one way to do it: 1. Use [trustme](https://github.com/python-trio/trustme) to generate a pair of server key/cert files, and a client cert file. -1. Pass the server key/cert files when starting your local server. (This depends on the particular web server you're using. For example, [Uvicorn](https://www.uvicorn.org) provides the `--ssl-keyfile` and `--ssl-certfile` options.) -1. Tell HTTPX to use the certificates stored in `client.pem`: +2. Pass the server key/cert files when starting your local server. (This depends on the particular web server you're using. For example, [Uvicorn](https://www.uvicorn.org) provides the `--ssl-keyfile` and `--ssl-certfile` options.) +3. Tell HTTPX to use the certificates stored in `client.pem`: -```python -client = httpx.Client(verify="/tmp/client.pem") -response = client.get("https://localhost:8000") +```pycon +>>> import httpx +>>> ssl_context = httpx.SSLContext(verify="/tmp/client.pem") +>>> r = httpx.get("https://localhost:8000", ssl_context=ssl_context) +>>> r +Response <200 OK> ``` diff --git a/docs/compatibility.md b/docs/compatibility.md index e820a67b07..58d71f5487 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -171,12 +171,10 @@ Also note that `requests.Session.request(...)` allows a `proxies=...` parameter, ## SSL configuration -When using a `Client` instance, the `trust_env`, `verify`, and `cert` arguments should always be passed on client instantiation, rather than passed to the request method. +When using a `Client` instance, the ssl configurations should always be passed on client instantiation, rather than passed to the request method. If you need more than one different SSL configuration, you should use different client instances for each SSL configuration. -Requests supports `REQUESTS_CA_BUNDLE` which points to either a file or a directory. HTTPX supports the `SSL_CERT_FILE` (for a file) and `SSL_CERT_DIR` (for a directory) OpenSSL variables instead. - ## Request body on HTTP methods The HTTP `GET`, `DELETE`, `HEAD`, and `OPTIONS` methods are specified as not supporting a request body. To stay in line with this, the `.get`, `.delete`, `.head` and `.options` functions do not support `content`, `files`, `data`, or `json` arguments. diff --git a/docs/environment_variables.md b/docs/environment_variables.md index 28fdc5e8af..4f7a9f5284 100644 --- a/docs/environment_variables.md +++ b/docs/environment_variables.md @@ -8,66 +8,6 @@ Environment variables are used by default. To ignore environment variables, `tru Here is a list of environment variables that HTTPX recognizes and what function they serve: -## `SSLKEYLOGFILE` - -Valid values: a filename - -If this environment variable is set, TLS keys will be appended to the specified file, creating it if it doesn't exist, whenever key material is generated or received. The keylog file is designed for debugging purposes only. - -Support for `SSLKEYLOGFILE` requires Python 3.8 and OpenSSL 1.1.1 or newer. - -Example: - -```python -# test_script.py -import httpx - -with httpx.AsyncClient() as client: - r = client.get("https://google.com") -``` - -```console -SSLKEYLOGFILE=test.log python test_script.py -cat test.log -# TLS secrets log file, generated by OpenSSL / Python -SERVER_HANDSHAKE_TRAFFIC_SECRET XXXX -EXPORTER_SECRET XXXX -SERVER_TRAFFIC_SECRET_0 XXXX -CLIENT_HANDSHAKE_TRAFFIC_SECRET XXXX -CLIENT_TRAFFIC_SECRET_0 XXXX -SERVER_HANDSHAKE_TRAFFIC_SECRET XXXX -EXPORTER_SECRET XXXX -SERVER_TRAFFIC_SECRET_0 XXXX -CLIENT_HANDSHAKE_TRAFFIC_SECRET XXXX -CLIENT_TRAFFIC_SECRET_0 XXXX -``` - -## `SSL_CERT_FILE` - -Valid values: a filename - -If this environment variable is set then HTTPX will load -CA certificate from the specified file instead of the default -location. - -Example: - -```console -SSL_CERT_FILE=/path/to/ca-certs/ca-bundle.crt python -c "import httpx; httpx.get('https://example.com')" -``` - -## `SSL_CERT_DIR` - -Valid values: a directory following an [OpenSSL specific layout](https://www.openssl.org/docs/manmaster/man3/SSL_CTX_load_verify_locations.html). - -If this environment variable is set and the directory follows an [OpenSSL specific layout](https://www.openssl.org/docs/manmaster/man3/SSL_CTX_load_verify_locations.html) (ie. you ran `c_rehash`) then HTTPX will load CA certificates from this directory instead of the default location. - -Example: - -```console -SSL_CERT_DIR=/path/to/ca-certs/ python -c "import httpx; httpx.get('https://example.com')" -``` - ## Proxies The environment variables documented below are used as a convention by various HTTP tooling, including: diff --git a/docs/logging.md b/docs/logging.md index 53ae74990d..90c21e2563 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -20,25 +20,25 @@ httpx.get("https://www.example.com") Will send debug level output to the console, or wherever `stdout` is directed too... ``` -DEBUG [2023-03-16 14:36:20] httpx - load_ssl_context verify=True cert=None trust_env=True http2=False -DEBUG [2023-03-16 14:36:20] httpx - load_verify_locations cafile='/Users/tomchristie/GitHub/encode/httpx/venv/lib/python3.10/site-packages/certifi/cacert.pem' -DEBUG [2023-03-16 14:36:20] httpcore - connection.connect_tcp.started host='www.example.com' port=443 local_address=None timeout=5.0 -DEBUG [2023-03-16 14:36:20] httpcore - connection.connect_tcp.complete return_value= -DEBUG [2023-03-16 14:36:20] httpcore - connection.start_tls.started ssl_context= server_hostname='www.example.com' timeout=5.0 -DEBUG [2023-03-16 14:36:20] httpcore - connection.start_tls.complete return_value= -DEBUG [2023-03-16 14:36:20] httpcore - http11.send_request_headers.started request= -DEBUG [2023-03-16 14:36:20] httpcore - http11.send_request_headers.complete -DEBUG [2023-03-16 14:36:20] httpcore - http11.send_request_body.started request= -DEBUG [2023-03-16 14:36:20] httpcore - http11.send_request_body.complete -DEBUG [2023-03-16 14:36:20] httpcore - http11.receive_response_headers.started request= -DEBUG [2023-03-16 14:36:21] httpcore - http11.receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Content-Encoding', b'gzip'), (b'Accept-Ranges', b'bytes'), (b'Age', b'507675'), (b'Cache-Control', b'max-age=604800'), (b'Content-Type', b'text/html; charset=UTF-8'), (b'Date', b'Thu, 16 Mar 2023 14:36:21 GMT'), (b'Etag', b'"3147526947+ident"'), (b'Expires', b'Thu, 23 Mar 2023 14:36:21 GMT'), (b'Last-Modified', b'Thu, 17 Oct 2019 07:18:26 GMT'), (b'Server', b'ECS (nyb/1D2E)'), (b'Vary', b'Accept-Encoding'), (b'X-Cache', b'HIT'), (b'Content-Length', b'648')]) -INFO [2023-03-16 14:36:21] httpx - HTTP Request: GET https://www.example.com "HTTP/1.1 200 OK" -DEBUG [2023-03-16 14:36:21] httpcore - http11.receive_response_body.started request= -DEBUG [2023-03-16 14:36:21] httpcore - http11.receive_response_body.complete -DEBUG [2023-03-16 14:36:21] httpcore - http11.response_closed.started -DEBUG [2023-03-16 14:36:21] httpcore - http11.response_closed.complete -DEBUG [2023-03-16 14:36:21] httpcore - connection.close.started -DEBUG [2023-03-16 14:36:21] httpcore - connection.close.complete +DEBUG [2024-09-28 17:27:40] httpx - load_ssl_context verify=True cert=None +DEBUG [2024-09-28 17:27:40] httpx - load_verify_locations cafile='/Users/karenpetrosyan/oss/karhttpx/.venv/lib/python3.9/site-packages/certifi/cacert.pem' +DEBUG [2024-09-28 17:27:40] httpcore.connection - connect_tcp.started host='www.example.com' port=443 local_address=None timeout=5.0 socket_options=None +DEBUG [2024-09-28 17:27:41] httpcore.connection - connect_tcp.complete return_value= +DEBUG [2024-09-28 17:27:41] httpcore.connection - start_tls.started ssl_context=SSLContext(verify=True) server_hostname='www.example.com' timeout=5.0 +DEBUG [2024-09-28 17:27:41] httpcore.connection - start_tls.complete return_value= +DEBUG [2024-09-28 17:27:41] httpcore.http11 - send_request_headers.started request= +DEBUG [2024-09-28 17:27:41] httpcore.http11 - send_request_headers.complete +DEBUG [2024-09-28 17:27:41] httpcore.http11 - send_request_body.started request= +DEBUG [2024-09-28 17:27:41] httpcore.http11 - send_request_body.complete +DEBUG [2024-09-28 17:27:41] httpcore.http11 - receive_response_headers.started request= +DEBUG [2024-09-28 17:27:41] httpcore.http11 - receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Content-Encoding', b'gzip'), (b'Accept-Ranges', b'bytes'), (b'Age', b'407727'), (b'Cache-Control', b'max-age=604800'), (b'Content-Type', b'text/html; charset=UTF-8'), (b'Date', b'Sat, 28 Sep 2024 13:27:42 GMT'), (b'Etag', b'"3147526947+gzip"'), (b'Expires', b'Sat, 05 Oct 2024 13:27:42 GMT'), (b'Last-Modified', b'Thu, 17 Oct 2019 07:18:26 GMT'), (b'Server', b'ECAcc (dcd/7D43)'), (b'Vary', b'Accept-Encoding'), (b'X-Cache', b'HIT'), (b'Content-Length', b'648')]) +INFO [2024-09-28 17:27:41] httpx - HTTP Request: GET https://www.example.com "HTTP/1.1 200 OK" +DEBUG [2024-09-28 17:27:41] httpcore.http11 - receive_response_body.started request= +DEBUG [2024-09-28 17:27:41] httpcore.http11 - receive_response_body.complete +DEBUG [2024-09-28 17:27:41] httpcore.http11 - response_closed.started +DEBUG [2024-09-28 17:27:41] httpcore.http11 - response_closed.complete +DEBUG [2024-09-28 17:27:41] httpcore.connection - close.started +DEBUG [2024-09-28 17:27:41] httpcore.connection - close.complete ``` Logging output includes information from both the high-level `httpx` logger, and the network-level `httpcore` logger, which can be configured separately. diff --git a/httpx/__init__.py b/httpx/__init__.py index e9addde071..3226d9ad02 100644 --- a/httpx/__init__.py +++ b/httpx/__init__.py @@ -46,7 +46,6 @@ def main() -> None: # type: ignore "ConnectTimeout", "CookieConflict", "Cookies", - "create_ssl_context", "DecodingError", "delete", "DigestAuth", @@ -81,6 +80,7 @@ def main() -> None: # type: ignore "RequestNotRead", "Response", "ResponseNotRead", + "SSLContext", "stream", "StreamClosed", "StreamConsumed", diff --git a/httpx/_api.py b/httpx/_api.py index 8afb0f2fc7..98fc40409f 100644 --- a/httpx/_api.py +++ b/httpx/_api.py @@ -1,5 +1,6 @@ from __future__ import annotations +import ssl import typing from contextlib import contextmanager @@ -8,7 +9,6 @@ from ._models import Response from ._types import ( AuthTypes, - CertTypes, CookieTypes, HeaderTypes, ProxyTypes, @@ -17,7 +17,6 @@ RequestData, RequestFiles, TimeoutTypes, - VerifyTypes, ) from ._urls import URL @@ -49,8 +48,7 @@ def request( proxy: ProxyTypes | None = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, follow_redirects: bool = False, - verify: VerifyTypes = True, - cert: CertTypes | None = None, + ssl_context: ssl.SSLContext | None = None, trust_env: bool = True, ) -> Response: """ @@ -81,14 +79,8 @@ def request( * **timeout** - *(optional)* The timeout configuration to use when sending the request. * **follow_redirects** - *(optional)* Enables or disables HTTP redirects. - * **verify** - *(optional)* SSL certificates (a.k.a CA bundle) used to - verify the identity of requested hosts. Either `True` (default CA bundle), - a path to an SSL certificate file, an `ssl.SSLContext`, or `False` - (which will disable verification). - * **cert** - *(optional)* An SSL certificate used by the requested host - to authenticate the client. Either a path to an SSL certificate file, or - two-tuple of (certificate file, key file), or a three-tuple of (certificate - file, key file, password). + * **ssl_context** - *(optional)* An SSL certificate used by the requested host + to authenticate the client. * **trust_env** - *(optional)* Enables or disables usage of environment variables for configuration. @@ -106,8 +98,7 @@ def request( with Client( cookies=cookies, proxy=proxy, - cert=cert, - verify=verify, + ssl_context=ssl_context, timeout=timeout, trust_env=trust_env, ) as client: @@ -141,8 +132,7 @@ def stream( proxy: ProxyTypes | None = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, follow_redirects: bool = False, - verify: VerifyTypes = True, - cert: CertTypes | None = None, + ssl_context: ssl.SSLContext | None = None, trust_env: bool = True, ) -> typing.Iterator[Response]: """ @@ -158,8 +148,7 @@ def stream( with Client( cookies=cookies, proxy=proxy, - cert=cert, - verify=verify, + ssl_context=ssl_context, timeout=timeout, trust_env=trust_env, ) as client: @@ -187,8 +176,7 @@ def get( auth: AuthTypes | None = None, proxy: ProxyTypes | None = None, follow_redirects: bool = False, - cert: CertTypes | None = None, - verify: VerifyTypes = True, + ssl_context: ssl.SSLContext | None = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, trust_env: bool = True, ) -> Response: @@ -209,8 +197,7 @@ def get( auth=auth, proxy=proxy, follow_redirects=follow_redirects, - cert=cert, - verify=verify, + ssl_context=ssl_context, timeout=timeout, trust_env=trust_env, ) @@ -225,8 +212,7 @@ def options( auth: AuthTypes | None = None, proxy: ProxyTypes | None = None, follow_redirects: bool = False, - cert: CertTypes | None = None, - verify: VerifyTypes = True, + ssl_context: ssl.SSLContext | None = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, trust_env: bool = True, ) -> Response: @@ -247,8 +233,7 @@ def options( auth=auth, proxy=proxy, follow_redirects=follow_redirects, - cert=cert, - verify=verify, + ssl_context=ssl_context, timeout=timeout, trust_env=trust_env, ) @@ -263,8 +248,7 @@ def head( auth: AuthTypes | None = None, proxy: ProxyTypes | None = None, follow_redirects: bool = False, - cert: CertTypes | None = None, - verify: VerifyTypes = True, + ssl_context: ssl.SSLContext | None = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, trust_env: bool = True, ) -> Response: @@ -285,8 +269,7 @@ def head( auth=auth, proxy=proxy, follow_redirects=follow_redirects, - cert=cert, - verify=verify, + ssl_context=ssl_context, timeout=timeout, trust_env=trust_env, ) @@ -305,8 +288,7 @@ def post( auth: AuthTypes | None = None, proxy: ProxyTypes | None = None, follow_redirects: bool = False, - cert: CertTypes | None = None, - verify: VerifyTypes = True, + ssl_context: ssl.SSLContext | None = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, trust_env: bool = True, ) -> Response: @@ -328,8 +310,7 @@ def post( auth=auth, proxy=proxy, follow_redirects=follow_redirects, - cert=cert, - verify=verify, + ssl_context=ssl_context, timeout=timeout, trust_env=trust_env, ) @@ -348,8 +329,7 @@ def put( auth: AuthTypes | None = None, proxy: ProxyTypes | None = None, follow_redirects: bool = False, - cert: CertTypes | None = None, - verify: VerifyTypes = True, + ssl_context: ssl.SSLContext | None = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, trust_env: bool = True, ) -> Response: @@ -371,8 +351,7 @@ def put( auth=auth, proxy=proxy, follow_redirects=follow_redirects, - cert=cert, - verify=verify, + ssl_context=ssl_context, timeout=timeout, trust_env=trust_env, ) @@ -391,8 +370,7 @@ def patch( auth: AuthTypes | None = None, proxy: ProxyTypes | None = None, follow_redirects: bool = False, - cert: CertTypes | None = None, - verify: VerifyTypes = True, + ssl_context: ssl.SSLContext | None = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, trust_env: bool = True, ) -> Response: @@ -414,8 +392,7 @@ def patch( auth=auth, proxy=proxy, follow_redirects=follow_redirects, - cert=cert, - verify=verify, + ssl_context=ssl_context, timeout=timeout, trust_env=trust_env, ) @@ -430,9 +407,8 @@ def delete( auth: AuthTypes | None = None, proxy: ProxyTypes | None = None, follow_redirects: bool = False, - cert: CertTypes | None = None, - verify: VerifyTypes = True, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, + ssl_context: ssl.SSLContext | None = None, trust_env: bool = True, ) -> Response: """ @@ -452,8 +428,7 @@ def delete( auth=auth, proxy=proxy, follow_redirects=follow_redirects, - cert=cert, - verify=verify, + ssl_context=ssl_context, timeout=timeout, trust_env=trust_env, ) diff --git a/httpx/_client.py b/httpx/_client.py index f924a311e7..0760ba7faf 100644 --- a/httpx/_client.py +++ b/httpx/_client.py @@ -3,6 +3,7 @@ import datetime import enum import logging +import ssl import typing import warnings from contextlib import asynccontextmanager, contextmanager @@ -32,7 +33,6 @@ from ._types import ( AsyncByteStream, AuthTypes, - CertTypes, CookieTypes, HeaderTypes, ProxyTypes, @@ -43,7 +43,6 @@ RequestFiles, SyncByteStream, TimeoutTypes, - VerifyTypes, ) from ._urls import URL, QueryParams from ._utils import ( @@ -585,14 +584,8 @@ class Client(BaseClient): sending requests. * **cookies** - *(optional)* Dictionary of Cookie items to include when sending requests. - * **verify** - *(optional)* SSL certificates (a.k.a CA bundle) used to - verify the identity of requested hosts. Either `True` (default CA bundle), - a path to an SSL certificate file, an `ssl.SSLContext`, or `False` - (which will disable verification). - * **cert** - *(optional)* An SSL certificate used by the requested host - to authenticate the client. Either a path to an SSL certificate file, or - two-tuple of (certificate file, key file), or a three-tuple of (certificate - file, key file, password). + * **ssl_context** - *(optional)* An SSL certificate used by the requested host + to authenticate the client. * **http2** - *(optional)* A boolean indicating if HTTP/2 support should be enabled. Defaults to `False`. * **proxy** - *(optional)* A proxy URL where all the traffic should be routed. @@ -621,8 +614,7 @@ def __init__( params: QueryParamTypes | None = None, headers: HeaderTypes | None = None, cookies: CookieTypes | None = None, - verify: VerifyTypes = True, - cert: CertTypes | None = None, + ssl_context: ssl.SSLContext | None = None, http1: bool = True, http2: bool = False, proxy: ProxyTypes | None = None, @@ -664,25 +656,21 @@ def __init__( proxy_map = self._get_proxy_map(proxy, allow_env_proxies) self._transport = self._init_transport( - verify=verify, - cert=cert, + ssl_context=ssl_context, http1=http1, http2=http2, limits=limits, transport=transport, - trust_env=trust_env, ) self._mounts: dict[URLPattern, BaseTransport | None] = { URLPattern(key): None if proxy is None else self._init_proxy_transport( proxy, - verify=verify, - cert=cert, + ssl_context=ssl_context, http1=http1, http2=http2, limits=limits, - trust_env=trust_env, ) for key, proxy in proxy_map.items() } @@ -695,8 +683,7 @@ def __init__( def _init_transport( self, - verify: VerifyTypes = True, - cert: CertTypes | None = None, + ssl_context: ssl.SSLContext | None = None, http1: bool = True, http2: bool = False, limits: Limits = DEFAULT_LIMITS, @@ -707,8 +694,7 @@ def _init_transport( return transport return HTTPTransport( - verify=verify, - cert=cert, + ssl_context=ssl_context, http1=http1, http2=http2, limits=limits, @@ -718,16 +704,14 @@ def _init_transport( def _init_proxy_transport( self, proxy: Proxy, - verify: VerifyTypes = True, - cert: CertTypes | None = None, + ssl_context: ssl.SSLContext | None = None, http1: bool = True, http2: bool = False, limits: Limits = DEFAULT_LIMITS, trust_env: bool = True, ) -> BaseTransport: return HTTPTransport( - verify=verify, - cert=cert, + ssl_context=ssl_context, http1=http1, http2=http2, limits=limits, @@ -1307,14 +1291,8 @@ class AsyncClient(BaseClient): sending requests. * **cookies** - *(optional)* Dictionary of Cookie items to include when sending requests. - * **verify** - *(optional)* SSL certificates (a.k.a CA bundle) used to - verify the identity of requested hosts. Either `True` (default CA bundle), - a path to an SSL certificate file, an `ssl.SSLContext`, or `False` - (which will disable verification). - * **cert** - *(optional)* An SSL certificate used by the requested host - to authenticate the client. Either a path to an SSL certificate file, or - two-tuple of (certificate file, key file), or a three-tuple of (certificate - file, key file, password). + * **ssl_context** - *(optional)* An SSL certificate used by the requested host + to authenticate the client. * **http2** - *(optional)* A boolean indicating if HTTP/2 support should be enabled. Defaults to `False`. * **proxy** - *(optional)* A proxy URL where all the traffic should be routed. @@ -1341,8 +1319,7 @@ def __init__( params: QueryParamTypes | None = None, headers: HeaderTypes | None = None, cookies: CookieTypes | None = None, - verify: VerifyTypes = True, - cert: CertTypes | None = None, + ssl_context: ssl.SSLContext | None = None, http1: bool = True, http2: bool = False, proxy: ProxyTypes | None = None, @@ -1384,13 +1361,11 @@ def __init__( proxy_map = self._get_proxy_map(proxy, allow_env_proxies) self._transport = self._init_transport( - verify=verify, - cert=cert, + ssl_context=ssl_context, http1=http1, http2=http2, limits=limits, transport=transport, - trust_env=trust_env, ) self._mounts: dict[URLPattern, AsyncBaseTransport | None] = { @@ -1398,12 +1373,10 @@ def __init__( if proxy is None else self._init_proxy_transport( proxy, - verify=verify, - cert=cert, + ssl_context=ssl_context, http1=http1, http2=http2, limits=limits, - trust_env=trust_env, ) for key, proxy in proxy_map.items() } @@ -1415,43 +1388,35 @@ def __init__( def _init_transport( self, - verify: VerifyTypes = True, - cert: CertTypes | None = None, + ssl_context: ssl.SSLContext | None = None, http1: bool = True, http2: bool = False, limits: Limits = DEFAULT_LIMITS, transport: AsyncBaseTransport | None = None, - trust_env: bool = True, ) -> AsyncBaseTransport: if transport is not None: return transport return AsyncHTTPTransport( - verify=verify, - cert=cert, + ssl_context=ssl_context, http1=http1, http2=http2, limits=limits, - trust_env=trust_env, ) def _init_proxy_transport( self, proxy: Proxy, - verify: VerifyTypes = True, - cert: CertTypes | None = None, + ssl_context: ssl.SSLContext | None = None, http1: bool = True, http2: bool = False, limits: Limits = DEFAULT_LIMITS, - trust_env: bool = True, ) -> AsyncBaseTransport: return AsyncHTTPTransport( - verify=verify, - cert=cert, + ssl_context=ssl_context, http1=http1, http2=http2, limits=limits, - trust_env=trust_env, proxy=proxy, ) diff --git a/httpx/_config.py b/httpx/_config.py index 1b12911faf..ff1f61590a 100644 --- a/httpx/_config.py +++ b/httpx/_config.py @@ -12,9 +12,8 @@ from ._models import Headers from ._types import CertTypes, HeaderTypes, TimeoutTypes, VerifyTypes from ._urls import URL -from ._utils import get_ca_bundle_from_env -__all__ = ["Limits", "Proxy", "Timeout", "create_ssl_context"] +__all__ = ["Limits", "Proxy", "SSLContext", "Timeout"] DEFAULT_CIPHERS = ":".join( [ @@ -46,151 +45,101 @@ class UnsetType: UNSET = UnsetType() -def create_ssl_context( - cert: CertTypes | None = None, - verify: VerifyTypes = True, - trust_env: bool = True, - http2: bool = False, -) -> ssl.SSLContext: - return SSLConfig( - cert=cert, verify=verify, trust_env=trust_env, http2=http2 - ).ssl_context - - -class SSLConfig: - """ - SSL Configuration. - """ - +class SSLContext(ssl.SSLContext): DEFAULT_CA_BUNDLE_PATH = Path(certifi.where()) def __init__( self, - *, - cert: CertTypes | None = None, verify: VerifyTypes = True, - trust_env: bool = True, - http2: bool = False, + cert: CertTypes | None = None, ) -> None: - self.cert = cert self.verify = verify - self.trust_env = trust_env - self.http2 = http2 - self.ssl_context = self.load_ssl_context() + set_minimum_tls_version_1_2(self) + self.options |= ssl.OP_NO_COMPRESSION + self.set_ciphers(DEFAULT_CIPHERS) + + keylogfile = os.environ.get("SSLKEYLOGFILE") + if keylogfile: + self.keylog_filename = keylogfile - def load_ssl_context(self) -> ssl.SSLContext: logger.debug( - "load_ssl_context verify=%r cert=%r trust_env=%r http2=%r", - self.verify, - self.cert, - self.trust_env, - self.http2, + "load_ssl_context verify=%r cert=%r", + verify, + cert, ) - if self.verify: - return self.load_ssl_context_verify() - return self.load_ssl_context_no_verify() - - def load_ssl_context_no_verify(self) -> ssl.SSLContext: - """ - Return an SSL context for unverified connections. - """ - context = self._create_default_ssl_context() - context.check_hostname = False - context.verify_mode = ssl.CERT_NONE - self._load_client_certs(context) - return context + if not verify: + self.check_hostname = False + self.verify_mode = ssl.CERT_NONE + self._load_client_certs(cert) + return - def load_ssl_context_verify(self) -> ssl.SSLContext: - """ - Return an SSL context for verified connections. - """ - if self.trust_env and self.verify is True: - ca_bundle = get_ca_bundle_from_env() - if ca_bundle is not None: - self.verify = ca_bundle - - if isinstance(self.verify, ssl.SSLContext): - # Allow passing in our own SSLContext object that's pre-configured. - context = self.verify - self._load_client_certs(context) - return context - elif isinstance(self.verify, bool): + if isinstance(verify, bool): ca_bundle_path = self.DEFAULT_CA_BUNDLE_PATH - elif Path(self.verify).exists(): - ca_bundle_path = Path(self.verify) + elif Path(verify).exists(): + ca_bundle_path = Path(verify) else: raise IOError( "Could not find a suitable TLS CA certificate bundle, " - "invalid path: {}".format(self.verify) + "invalid path: {}".format(verify) ) - context = self._create_default_ssl_context() - context.verify_mode = ssl.CERT_REQUIRED - context.check_hostname = True + self.verify_mode = ssl.CERT_REQUIRED + self.check_hostname = True # Signal to server support for PHA in TLS 1.3. Raises an # AttributeError if only read-only access is implemented. try: - context.post_handshake_auth = True + self.post_handshake_auth = True except AttributeError: # pragma: no cover pass # Disable using 'commonName' for SSLContext.check_hostname # when the 'subjectAltName' extension isn't available. try: - context.hostname_checks_common_name = False + self.hostname_checks_common_name = False except AttributeError: # pragma: no cover pass if ca_bundle_path.is_file(): cafile = str(ca_bundle_path) logger.debug("load_verify_locations cafile=%r", cafile) - context.load_verify_locations(cafile=cafile) + self.load_verify_locations(cafile=cafile) elif ca_bundle_path.is_dir(): capath = str(ca_bundle_path) logger.debug("load_verify_locations capath=%r", capath) - context.load_verify_locations(capath=capath) + self.load_verify_locations(capath=capath) - self._load_client_certs(context) + self._load_client_certs(cert) - return context - - def _create_default_ssl_context(self) -> ssl.SSLContext: + def _load_client_certs(self, cert: typing.Optional[CertTypes] = None) -> None: """ - Creates the default SSLContext object that's used for both verified - and unverified connections. + Loads client certificates into our SSLContext object """ - context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - set_minimum_tls_version_1_2(context) - context.options |= ssl.OP_NO_COMPRESSION - context.set_ciphers(DEFAULT_CIPHERS) - - if ssl.HAS_ALPN: - alpn_idents = ["http/1.1", "h2"] if self.http2 else ["http/1.1"] - context.set_alpn_protocols(alpn_idents) + if cert is not None: + if isinstance(cert, str): + self.load_cert_chain(certfile=cert) + elif isinstance(cert, tuple) and len(cert) == 2: + self.load_cert_chain(certfile=cert[0], keyfile=cert[1]) + elif isinstance(cert, tuple) and len(cert) == 3: + self.load_cert_chain( + certfile=cert[0], + keyfile=cert[1], + password=cert[2], + ) - keylogfile = os.environ.get("SSLKEYLOGFILE") - if keylogfile and self.trust_env: - context.keylog_filename = keylogfile + def __repr__(self) -> str: + class_name = self.__class__.__name__ - return context + return f"{class_name}(verify={self.verify!r})" - def _load_client_certs(self, ssl_context: ssl.SSLContext) -> None: - """ - Loads client certificates into our SSLContext object - """ - if self.cert is not None: - if isinstance(self.cert, str): - ssl_context.load_cert_chain(certfile=self.cert) - elif isinstance(self.cert, tuple) and len(self.cert) == 2: - ssl_context.load_cert_chain(certfile=self.cert[0], keyfile=self.cert[1]) - elif isinstance(self.cert, tuple) and len(self.cert) == 3: - ssl_context.load_cert_chain( - certfile=self.cert[0], - keyfile=self.cert[1], - password=self.cert[2], - ) + def __new__( + cls, + protocol: ssl._SSLMethod = ssl.PROTOCOL_TLS_CLIENT, + *args: typing.Any, + **kwargs: typing.Any, + ) -> "SSLContext": + return super().__new__(cls, protocol, *args, **kwargs) class Timeout: diff --git a/httpx/_main.py b/httpx/_main.py index 72657f8ca3..41c50f7413 100644 --- a/httpx/_main.py +++ b/httpx/_main.py @@ -16,6 +16,7 @@ import rich.table from ._client import Client +from ._config import SSLContext from ._exceptions import RequestError from ._models import Response from ._status_codes import codes @@ -473,12 +474,10 @@ def main( if not method: method = "POST" if content or data or files or json else "GET" + ssl_context = SSLContext(verify=verify) try: with Client( - proxy=proxy, - timeout=timeout, - verify=verify, - http2=http2, + proxy=proxy, timeout=timeout, http2=http2, ssl_context=ssl_context ) as client: with client.stream( method, diff --git a/httpx/_transports/default.py b/httpx/_transports/default.py index 33db416dd1..e4243c0f2b 100644 --- a/httpx/_transports/default.py +++ b/httpx/_transports/default.py @@ -27,12 +27,13 @@ from __future__ import annotations import contextlib +import ssl import typing from types import TracebackType import httpcore -from .._config import DEFAULT_LIMITS, Limits, Proxy, create_ssl_context +from .._config import DEFAULT_LIMITS, Limits, Proxy, SSLContext from .._exceptions import ( ConnectError, ConnectTimeout, @@ -50,7 +51,7 @@ WriteTimeout, ) from .._models import Request, Response -from .._types import AsyncByteStream, CertTypes, ProxyTypes, SyncByteStream, VerifyTypes +from .._types import AsyncByteStream, ProxyTypes, SyncByteStream from .._urls import URL from .base import AsyncBaseTransport, BaseTransport @@ -124,8 +125,7 @@ def close(self) -> None: class HTTPTransport(BaseTransport): def __init__( self, - verify: VerifyTypes = True, - cert: CertTypes | None = None, + ssl_context: ssl.SSLContext | None = None, http1: bool = True, http2: bool = False, limits: Limits = DEFAULT_LIMITS, @@ -136,8 +136,8 @@ def __init__( retries: int = 0, socket_options: typing.Iterable[SOCKET_OPTION] | None = None, ) -> None: - ssl_context = create_ssl_context(verify=verify, cert=cert, trust_env=trust_env) proxy = Proxy(url=proxy) if isinstance(proxy, (str, URL)) else proxy + ssl_context = ssl_context or SSLContext() if proxy is None: self._pool = httpcore.ConnectionPool( @@ -265,20 +265,18 @@ async def aclose(self) -> None: class AsyncHTTPTransport(AsyncBaseTransport): def __init__( self, - verify: VerifyTypes = True, - cert: CertTypes | None = None, + ssl_context: ssl.SSLContext | None = None, http1: bool = True, http2: bool = False, limits: Limits = DEFAULT_LIMITS, - trust_env: bool = True, proxy: ProxyTypes | None = None, uds: str | None = None, local_address: str | None = None, retries: int = 0, socket_options: typing.Iterable[SOCKET_OPTION] | None = None, ) -> None: - ssl_context = create_ssl_context(verify=verify, cert=cert, trust_env=trust_env) proxy = Proxy(url=proxy) if isinstance(proxy, (str, URL)) else proxy + ssl_context = ssl_context or SSLContext() if proxy is None: self._pool = httpcore.AsyncConnectionPool( diff --git a/httpx/_types.py b/httpx/_types.py index e169172559..6c60f93207 100644 --- a/httpx/_types.py +++ b/httpx/_types.py @@ -2,7 +2,6 @@ Type definitions for type checking purposes. """ -import ssl from http.cookiejar import CookieJar from typing import ( IO, @@ -61,7 +60,7 @@ # (certfile, keyfile, password) Tuple[str, Optional[str], Optional[str]], ] -VerifyTypes = Union[str, bool, ssl.SSLContext] +VerifyTypes = Union[str, bool] TimeoutTypes = Union[ Optional[float], Tuple[Optional[float], Optional[float], Optional[float], Optional[float]], diff --git a/httpx/_utils.py b/httpx/_utils.py index 0160d61dba..9256745b83 100644 --- a/httpx/_utils.py +++ b/httpx/_utils.py @@ -8,7 +8,6 @@ import re import time import typing -from pathlib import Path from urllib.request import getproxies from ._types import PrimitiveData @@ -91,18 +90,6 @@ def replacer(match: typing.Match[str]) -> str: return f'{name}="{value}"'.encode() -def get_ca_bundle_from_env() -> str | None: - if "SSL_CERT_FILE" in os.environ: - ssl_file = Path(os.environ["SSL_CERT_FILE"]) - if ssl_file.is_file(): - return str(ssl_file) - if "SSL_CERT_DIR" in os.environ: - ssl_path = Path(os.environ["SSL_CERT_DIR"]) - if ssl_path.is_dir(): - return str(ssl_path) - return None - - def parse_header_links(value: str) -> list[dict[str, str]]: """ Returns a list of parsed link headers, for more info see: diff --git a/tests/conftest.py b/tests/conftest.py index 5c4a6ae577..858bca1397 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -187,12 +187,6 @@ def cert_authority(): return trustme.CA() -@pytest.fixture(scope="session") -def ca_cert_pem_file(cert_authority): - with cert_authority.cert_pem.tempfile() as tmp: - yield tmp - - @pytest.fixture(scope="session") def localhost_cert(cert_authority): return cert_authority.issue_cert("localhost") @@ -291,17 +285,3 @@ def server() -> typing.Iterator[TestServer]: config = Config(app=app, lifespan="off", loop="asyncio") server = TestServer(config=config) yield from serve_in_thread(server) - - -@pytest.fixture(scope="session") -def https_server(cert_pem_file, cert_private_key_file): - config = Config( - app=app, - lifespan="off", - ssl_certfile=cert_pem_file, - ssl_keyfile=cert_private_key_file, - port=8001, - loop="asyncio", - ) - server = TestServer(config=config) - yield from serve_in_thread(server) diff --git a/tests/test_config.py b/tests/test_config.py index 6f6ee4f575..9b9bd64ec2 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,5 +1,5 @@ -import os import ssl +import typing from pathlib import Path import certifi @@ -9,48 +9,37 @@ def test_load_ssl_config(): - context = httpx.create_ssl_context() + context = httpx.SSLContext() assert context.verify_mode == ssl.VerifyMode.CERT_REQUIRED assert context.check_hostname is True def test_load_ssl_config_verify_non_existing_path(): with pytest.raises(IOError): - httpx.create_ssl_context(verify="/path/to/nowhere") + httpx.SSLContext(verify="/path/to/nowhere") -def test_load_ssl_config_verify_existing_file(): - context = httpx.create_ssl_context(verify=certifi.where()) - assert context.verify_mode == ssl.VerifyMode.CERT_REQUIRED - assert context.check_hostname is True +def test_load_ssl_with_keylog(monkeypatch: typing.Any) -> None: + monkeypatch.setenv("SSLKEYLOGFILE", "test") + context = httpx.SSLContext() + assert context.keylog_filename == "test" -@pytest.mark.parametrize("config", ("SSL_CERT_FILE", "SSL_CERT_DIR")) -def test_load_ssl_config_verify_env_file( - https_server, ca_cert_pem_file, config, cert_authority -): - os.environ[config] = ( - ca_cert_pem_file - if config.endswith("_FILE") - else str(Path(ca_cert_pem_file).parent) - ) - context = httpx.create_ssl_context(trust_env=True) - cert_authority.configure_trust(context) - +def test_load_ssl_config_verify_existing_file(): + context = httpx.SSLContext(verify=certifi.where()) assert context.verify_mode == ssl.VerifyMode.CERT_REQUIRED assert context.check_hostname is True - assert len(context.get_ca_certs()) == 1 def test_load_ssl_config_verify_directory(): path = Path(certifi.where()).parent - context = httpx.create_ssl_context(verify=str(path)) + context = httpx.SSLContext(verify=str(path)) assert context.verify_mode == ssl.VerifyMode.CERT_REQUIRED assert context.check_hostname is True def test_load_ssl_config_cert_and_key(cert_pem_file, cert_private_key_file): - context = httpx.create_ssl_context(cert=(cert_pem_file, cert_private_key_file)) + context = httpx.SSLContext(cert=(cert_pem_file, cert_private_key_file)) assert context.verify_mode == ssl.VerifyMode.CERT_REQUIRED assert context.check_hostname is True @@ -59,7 +48,7 @@ def test_load_ssl_config_cert_and_key(cert_pem_file, cert_private_key_file): def test_load_ssl_config_cert_and_encrypted_key( cert_pem_file, cert_encrypted_private_key_file, password ): - context = httpx.create_ssl_context( + context = httpx.SSLContext( cert=(cert_pem_file, cert_encrypted_private_key_file, password) ) assert context.verify_mode == ssl.VerifyMode.CERT_REQUIRED @@ -70,33 +59,36 @@ def test_load_ssl_config_cert_and_key_invalid_password( cert_pem_file, cert_encrypted_private_key_file ): with pytest.raises(ssl.SSLError): - httpx.create_ssl_context( + httpx.SSLContext( cert=(cert_pem_file, cert_encrypted_private_key_file, "password1") ) def test_load_ssl_config_cert_without_key_raises(cert_pem_file): with pytest.raises(ssl.SSLError): - httpx.create_ssl_context(cert=cert_pem_file) + httpx.SSLContext(cert=cert_pem_file) def test_load_ssl_config_no_verify(): - context = httpx.create_ssl_context(verify=False) + context = httpx.SSLContext(verify=False) assert context.verify_mode == ssl.VerifyMode.CERT_NONE assert context.check_hostname is False -def test_load_ssl_context(): - ssl_context = ssl.create_default_context() - context = httpx.create_ssl_context(verify=ssl_context) +def test_SSLContext_with_get_request(server, cert_pem_file): + context = httpx.SSLContext(verify=cert_pem_file) + response = httpx.get(server.url, ssl_context=context) + assert response.status_code == 200 - assert context is ssl_context +def test_SSLContext_repr(): + ssl_context = httpx.SSLContext() -def test_create_ssl_context_with_get_request(server, cert_pem_file): - context = httpx.create_ssl_context(verify=cert_pem_file) - response = httpx.get(server.url, verify=context) - assert response.status_code == 200 + assert repr(ssl_context) == "SSLContext(verify=True)" + + ssl_context = httpx.SSLContext(verify=certifi.where()) + + assert repr(ssl_context) == "SSLContext(verify='{}')".format(certifi.where()) def test_limits_repr(): @@ -174,32 +166,6 @@ def test_timeout_repr(): assert repr(timeout) == "Timeout(connect=None, read=5.0, write=None, pool=None)" -@pytest.mark.skipif( - not hasattr(ssl.SSLContext, "keylog_filename"), - reason="requires OpenSSL 1.1.1 or higher", -) -def test_ssl_config_support_for_keylog_file(tmpdir, monkeypatch): # pragma: no cover - with monkeypatch.context() as m: - m.delenv("SSLKEYLOGFILE", raising=False) - - context = httpx.create_ssl_context(trust_env=True) - - assert context.keylog_filename is None - - filename = str(tmpdir.join("test.log")) - - with monkeypatch.context() as m: - m.setenv("SSLKEYLOGFILE", filename) - - context = httpx.create_ssl_context(trust_env=True) - - assert context.keylog_filename == filename - - context = httpx.create_ssl_context(trust_env=False) - - assert context.keylog_filename is None - - def test_proxy_from_url(): proxy = httpx.Proxy("https://example.com") diff --git a/tests/test_utils.py b/tests/test_utils.py index f98a18f2cd..3639ea7710 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -9,12 +9,9 @@ import httpx from httpx._utils import ( URLPattern, - get_ca_bundle_from_env, get_environment_proxies, ) -from .common import TESTS_DIR - @pytest.mark.parametrize( "encoding", @@ -132,7 +129,7 @@ def test_logging_ssl(caplog): ( "httpx", logging.DEBUG, - "load_ssl_context verify=True cert=None trust_env=True http2=False", + "load_ssl_context verify=True cert=None", ), ( "httpx", @@ -142,46 +139,6 @@ def test_logging_ssl(caplog): ] -def test_get_ssl_cert_file(): - # Two environments is not set. - assert get_ca_bundle_from_env() is None - - os.environ["SSL_CERT_DIR"] = str(TESTS_DIR) - # SSL_CERT_DIR is correctly set, SSL_CERT_FILE is not set. - ca_bundle = get_ca_bundle_from_env() - assert ca_bundle is not None and ca_bundle.endswith("tests") - - del os.environ["SSL_CERT_DIR"] - os.environ["SSL_CERT_FILE"] = str(TESTS_DIR / "test_utils.py") - # SSL_CERT_FILE is correctly set, SSL_CERT_DIR is not set. - ca_bundle = get_ca_bundle_from_env() - assert ca_bundle is not None and ca_bundle.endswith("tests/test_utils.py") - - os.environ["SSL_CERT_FILE"] = "wrongfile" - # SSL_CERT_FILE is set with wrong file, SSL_CERT_DIR is not set. - assert get_ca_bundle_from_env() is None - - del os.environ["SSL_CERT_FILE"] - os.environ["SSL_CERT_DIR"] = "wrongpath" - # SSL_CERT_DIR is set with wrong path, SSL_CERT_FILE is not set. - assert get_ca_bundle_from_env() is None - - os.environ["SSL_CERT_DIR"] = str(TESTS_DIR) - os.environ["SSL_CERT_FILE"] = str(TESTS_DIR / "test_utils.py") - # Two environments is correctly set. - ca_bundle = get_ca_bundle_from_env() - assert ca_bundle is not None and ca_bundle.endswith("tests/test_utils.py") - - os.environ["SSL_CERT_FILE"] = "wrongfile" - # Two environments is set but SSL_CERT_FILE is not a file. - ca_bundle = get_ca_bundle_from_env() - assert ca_bundle is not None and ca_bundle.endswith("tests") - - os.environ["SSL_CERT_DIR"] = "wrongpath" - # Two environments is set but both are not correct. - assert get_ca_bundle_from_env() is None - - @pytest.mark.parametrize( ["environment", "proxies"], [ From 1aa0a14ec1fc744f6b362277d600955feee025e1 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 10 Oct 2024 14:17:24 +0100 Subject: [PATCH 06/17] Drop overloaded usage of 'verify' and 'cert'. (#3335) Co-authored-by: T-256 <132141463+T-256@users.noreply.github.com> --- docs/advanced/ssl.md | 103 +++++++++++++++-------------- httpx/_client.py | 2 - httpx/_compat.py | 27 +------- httpx/_config.py | 124 ++++++++--------------------------- httpx/_transports/default.py | 1 - httpx/_types.py | 9 --- tests/test_config.py | 37 ++++++----- tests/test_utils.py | 21 ------ 8 files changed, 100 insertions(+), 224 deletions(-) diff --git a/docs/advanced/ssl.md b/docs/advanced/ssl.md index 7eed687196..9e0b2e9955 100644 --- a/docs/advanced/ssl.md +++ b/docs/advanced/ssl.md @@ -12,30 +12,30 @@ httpx.ConnectError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: You can configure the verification using `httpx.SSLContext()`. ```pycon ->>> ssl_context = httpx.SSLContext() ->>> ssl_context -SSLContext(verify=True) ->>> httpx.get("https://www.example.com", ssl_context=ssl_context) +>>> context = httpx.SSLContext() +>>> context + +>>> httpx.get("https://www.example.com", ssl_context=context) httpx.ConnectError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:997) ``` -For example, you can use this to disable verification completely and allow insecure requests... +You can use this to disable verification completely and allow insecure requests... ```pycon ->>> no_verify = httpx.SSLContext(verify=False) ->>> no_verify -SSLContext(verify=False) ->>> httpx.get("https://expired.badssl.com/", ssl_context=no_verify) +>>> context = httpx.SSLContext(verify=False) +>>> context + +>>> httpx.get("https://expired.badssl.com/", ssl_context=context) ``` ### Configuring client instances -If you're using a `Client()` instance, then you should pass any SSL settings when instantiating the client. +If you're using a `Client()` instance you should pass any SSL settings when instantiating the client. ```pycon ->>> ssl_context = httpx.SSLContext() ->>> client = httpx.Client(ssl_context=ssl_context) +>>> context = httpx.SSLContext() +>>> client = httpx.Client(ssl_context=context) ``` The `client.get(...)` method and other request methods on a `Client` instance *do not* support changing the SSL settings on a per-request basis. @@ -59,8 +59,8 @@ Using the default SSL context. Using the default SSL context, but specified explicitly. ```pycon ->>> default = httpx.SSLContext() ->>> client = httpx.Client(ssl_context=default) +>>> context = httpx.SSLContext() +>>> client = httpx.Client(ssl_context=context) >>> client.get("https://www.example.com") ``` @@ -68,26 +68,20 @@ Using the default SSL context, but specified explicitly. Using the default SSL context, with `verify=True` specified explicitly. ```pycon ->>> default = httpx.SSLContext(verify=True) ->>> client = httpx.Client(ssl_context=default) +>>> context = httpx.SSLContext(verify=True) +>>> client = httpx.Client(ssl_context=context) >>> client.get("https://www.example.com") ``` -Using an SSL context, with `certifi.where()` explicitly specified. +### Configuring certificate verification -```pycon ->>> default = httpx.SSLContext(verify=certifi.where()) ->>> client = httpx.Client(ssl_context=default) ->>> client.get("https://www.example.com") - -``` - -For some advanced situations may require you to use a different set of certificates, either by specifying a PEM file: +You can load additional certificate verification using the [`.load_verify_locations()`](https://docs.python.org/3/library/ssl.html#ssl.SSLContext.load_verify_locations) API: ```pycon ->>> custom_cafile = httpx.SSLContext(verify="path/to/certs.pem") ->>> client = httpx.Client(ssl_context=custom_cafile) +>>> context = httpx.SSLContext() +>>> context.load_verify_locations(cafile="path/to/certs.pem") +>>> client = httpx.Client(ssl_context=context) >>> client.get("https://www.example.com") ``` @@ -95,45 +89,50 @@ For some advanced situations may require you to use a different set of certifica Or by providing an certificate directory: ```pycon ->>> custom_capath = httpx.SSLContext(verify="path/to/certs") ->>> client = httpx.Client(ssl_context=custom_capath) +>>> context = httpx.SSLContext() +>>> context.load_verify_locations(capath="path/to/certs") +>>> client = httpx.Client(ssl_context=context) >>> client.get("https://www.example.com") ``` -These usages are equivelent to using [`.load_verify_locations()`](https://docs.python.org/3/library/ssl.html#ssl.SSLContext.load_verify_locations) with either `cafile=...` or `capath=...`. - ### Client side certificates -You can also specify a local cert to use as a client-side certificate, either a path to an SSL certificate file... +You can also specify a local cert to use as a client-side certificate, using the [`.load_cert_chain()`](https://docs.python.org/3/library/ssl.html#ssl.SSLContext.load_cert_chain) API: ```pycon ->>> cert = "path/to/client.pem" ->>> ssl_context = httpx.SSLContext(cert=cert) +>>> context = httpx.SSLContext() +>>> context.load_cert_chain(certfile="path/to/client.pem") >>> httpx.get("https://example.org", ssl_context=ssl_context) ``` -Or two-tuple of (certificate file, key file)... +Or including a keyfile... ```pycon ->>> cert = ("path/to/client.pem", "path/to/client.key") ->>> ssl_context = httpx.SSLContext(cert=cert) ->>> httpx.get("https://example.org", ssl_context=ssl_context) +>>> context = httpx.SSLContext() +>>> context.load_cert_chain( + certfile="path/to/client.pem", + keyfile="path/to/client.key" + ) +>>> httpx.get("https://example.org", ssl_context=context) ``` -Or a three-tuple of (certificate file, key file, password)... +Or including a keyfile and password... ```pycon ->>> cert = ("path/to/client.pem", "path/to/client.key", "password") ->>> ssl_context = httpx.SSLContext(cert=cert) ->>> httpx.get("https://example.org", ssl_context=ssl_context) +>>> context = httpx.SSLContext(cert=cert) +>>> context = httpx.SSLContext() +>>> context.load_cert_chain( + certfile="path/to/client.pem", + keyfile="path/to/client.key" + password="password" + ) +>>> httpx.get("https://example.org", ssl_context=context) ``` -These configurations are equivalent to using [`.load_cert_chain()`](https://docs.python.org/3/library/ssl.html#ssl.SSLContext.load_cert_chain). - ### Using alternate SSL contexts You can also use an alternate `ssl.SSLContext` instances. @@ -166,9 +165,14 @@ Unlike `requests`, the `httpx` package does not automatically pull in [the envir For example... ```python -# Use `SSL_CERT_FILE` or `SSL_CERT_DIR` if configured, otherwise use certifi. -verify = os.environ.get("SSL_CERT_FILE", os.environ.get("SSL_CERT_DIR", True)) -ssl_context = httpx.SSLContext(verify=verify) +context = httpx.SSLContext() + +# Use `SSL_CERT_FILE` or `SSL_CERT_DIR` if configured. +if os.environ.get("SSL_CERT_FILE") or os.environ.get("SSL_CERT_DIR"): + context.load_verify_locations( + cafile=os.environ.get("SSL_CERT_FILE"), + capath=os.environ.get("SSL_CERT_DIR"), + ) ``` ## `SSLKEYLOGFILE` @@ -217,8 +221,9 @@ If you do need to make HTTPS connections to a local server, for example to test ```pycon >>> import httpx ->>> ssl_context = httpx.SSLContext(verify="/tmp/client.pem") ->>> r = httpx.get("https://localhost:8000", ssl_context=ssl_context) +>>> context = httpx.SSLContext() +>>> context.load_verify_locations(cafile="/tmp/client.pem") +>>> r = httpx.get("https://localhost:8000", ssl_context=context) >>> r Response <200 OK> ``` diff --git a/httpx/_client.py b/httpx/_client.py index 0760ba7faf..8ba5f123a3 100644 --- a/httpx/_client.py +++ b/httpx/_client.py @@ -698,7 +698,6 @@ def _init_transport( http1=http1, http2=http2, limits=limits, - trust_env=trust_env, ) def _init_proxy_transport( @@ -715,7 +714,6 @@ def _init_proxy_transport( http1=http1, http2=http2, limits=limits, - trust_env=trust_env, proxy=proxy, ) diff --git a/httpx/_compat.py b/httpx/_compat.py index 7d86dced46..00b6e29ad8 100644 --- a/httpx/_compat.py +++ b/httpx/_compat.py @@ -4,8 +4,6 @@ """ import re -import ssl -import sys from types import ModuleType from typing import Optional @@ -37,27 +35,4 @@ if _zstd_version < (0, 18): # Defensive: zstd = None - -if sys.version_info >= (3, 10) or ssl.OPENSSL_VERSION_INFO >= (1, 1, 0, 7): - - def set_minimum_tls_version_1_2(context: ssl.SSLContext) -> None: - # The OP_NO_SSL* and OP_NO_TLS* become deprecated in favor of - # 'SSLContext.minimum_version' from Python 3.7 onwards, however - # this attribute is not available unless the ssl module is compiled - # with OpenSSL 1.1.0g or newer. - # https://docs.python.org/3.10/library/ssl.html#ssl.SSLContext.minimum_version - # https://docs.python.org/3.7/library/ssl.html#ssl.SSLContext.minimum_version - context.minimum_version = ssl.TLSVersion.TLSv1_2 - -else: - - def set_minimum_tls_version_1_2(context: ssl.SSLContext) -> None: - # If 'minimum_version' isn't available, we configure these options with - # the older deprecated variants. - context.options |= ssl.OP_NO_SSLv2 - context.options |= ssl.OP_NO_SSLv3 - context.options |= ssl.OP_NO_TLSv1 - context.options |= ssl.OP_NO_TLSv1_1 - - -__all__ = ["brotli", "set_minimum_tls_version_1_2"] +__all__ = ["brotli", "zstd"] diff --git a/httpx/_config.py b/httpx/_config.py index ff1f61590a..f701a331ca 100644 --- a/httpx/_config.py +++ b/httpx/_config.py @@ -1,42 +1,18 @@ from __future__ import annotations -import logging import os import ssl +import sys import typing -from pathlib import Path import certifi -from ._compat import set_minimum_tls_version_1_2 from ._models import Headers -from ._types import CertTypes, HeaderTypes, TimeoutTypes, VerifyTypes +from ._types import HeaderTypes, TimeoutTypes from ._urls import URL __all__ = ["Limits", "Proxy", "SSLContext", "Timeout"] -DEFAULT_CIPHERS = ":".join( - [ - "ECDHE+AESGCM", - "ECDHE+CHACHA20", - "DHE+AESGCM", - "DHE+CHACHA20", - "ECDH+AESGCM", - "DH+AESGCM", - "ECDH+AES", - "DH+AES", - "RSA+AESGCM", - "RSA+AES", - "!aNULL", - "!eNULL", - "!MD5", - "!DSS", - ] -) - - -logger = logging.getLogger("httpx") - class UnsetType: pass # pragma: no cover @@ -46,92 +22,44 @@ class UnsetType: class SSLContext(ssl.SSLContext): - DEFAULT_CA_BUNDLE_PATH = Path(certifi.where()) - def __init__( self, - verify: VerifyTypes = True, - cert: CertTypes | None = None, + verify: bool = True, ) -> None: - self.verify = verify - set_minimum_tls_version_1_2(self) - self.options |= ssl.OP_NO_COMPRESSION - self.set_ciphers(DEFAULT_CIPHERS) - - keylogfile = os.environ.get("SSLKEYLOGFILE") - if keylogfile: - self.keylog_filename = keylogfile - - logger.debug( - "load_ssl_context verify=%r cert=%r", - verify, - cert, - ) - + # ssl.SSLContext sets OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_COMPRESSION, + # OP_CIPHER_SERVER_PREFERENCE, OP_SINGLE_DH_USE and OP_SINGLE_ECDH_USE + # by default. (from `ssl.create_default_context`) + super().__init__() + self._verify = verify + + # Our SSL setup here is similar to the stdlib `ssl.create_default_context()` + # implementation, except with `certifi` used for certificate verification. if not verify: self.check_hostname = False self.verify_mode = ssl.CERT_NONE - self._load_client_certs(cert) return - if isinstance(verify, bool): - ca_bundle_path = self.DEFAULT_CA_BUNDLE_PATH - elif Path(verify).exists(): - ca_bundle_path = Path(verify) - else: - raise IOError( - "Could not find a suitable TLS CA certificate bundle, " - "invalid path: {}".format(verify) - ) - self.verify_mode = ssl.CERT_REQUIRED self.check_hostname = True - # Signal to server support for PHA in TLS 1.3. Raises an - # AttributeError if only read-only access is implemented. - try: - self.post_handshake_auth = True - except AttributeError: # pragma: no cover - pass - - # Disable using 'commonName' for SSLContext.check_hostname - # when the 'subjectAltName' extension isn't available. - try: - self.hostname_checks_common_name = False - except AttributeError: # pragma: no cover - pass - - if ca_bundle_path.is_file(): - cafile = str(ca_bundle_path) - logger.debug("load_verify_locations cafile=%r", cafile) - self.load_verify_locations(cafile=cafile) - elif ca_bundle_path.is_dir(): - capath = str(ca_bundle_path) - logger.debug("load_verify_locations capath=%r", capath) - self.load_verify_locations(capath=capath) - - self._load_client_certs(cert) - - def _load_client_certs(self, cert: typing.Optional[CertTypes] = None) -> None: - """ - Loads client certificates into our SSLContext object - """ - if cert is not None: - if isinstance(cert, str): - self.load_cert_chain(certfile=cert) - elif isinstance(cert, tuple) and len(cert) == 2: - self.load_cert_chain(certfile=cert[0], keyfile=cert[1]) - elif isinstance(cert, tuple) and len(cert) == 3: - self.load_cert_chain( - certfile=cert[0], - keyfile=cert[1], - password=cert[2], - ) + # Use stricter verify flags where possible. + if hasattr(ssl, "VERIFY_X509_PARTIAL_CHAIN"): # pragma: nocover + self.verify_flags |= ssl.VERIFY_X509_PARTIAL_CHAIN + if hasattr(ssl, "VERIFY_X509_STRICT"): # pragma: nocover + self.verify_flags |= ssl.VERIFY_X509_STRICT + + # Default to `certifi` for certificiate verification. + self.load_verify_locations(cafile=certifi.where()) + + # OpenSSL keylog file support. + if hasattr(self, "keylog_filename"): + keylogfile = os.environ.get("SSLKEYLOGFILE") + if keylogfile and not sys.flags.ignore_environment: + self.keylog_filename = keylogfile def __repr__(self) -> str: class_name = self.__class__.__name__ - - return f"{class_name}(verify={self.verify!r})" + return f"<{class_name}(verify={self._verify!r})>" def __new__( cls, diff --git a/httpx/_transports/default.py b/httpx/_transports/default.py index e4243c0f2b..f20e2a37e9 100644 --- a/httpx/_transports/default.py +++ b/httpx/_transports/default.py @@ -129,7 +129,6 @@ def __init__( http1: bool = True, http2: bool = False, limits: Limits = DEFAULT_LIMITS, - trust_env: bool = True, proxy: ProxyTypes | None = None, uds: str | None = None, local_address: str | None = None, diff --git a/httpx/_types.py b/httpx/_types.py index 6c60f93207..edd00da1bc 100644 --- a/httpx/_types.py +++ b/httpx/_types.py @@ -52,15 +52,6 @@ CookieTypes = Union["Cookies", CookieJar, Dict[str, str], List[Tuple[str, str]]] -CertTypes = Union[ - # certfile - str, - # (certfile, keyfile) - Tuple[str, Optional[str]], - # (certfile, keyfile, password) - Tuple[str, Optional[str], Optional[str]], -] -VerifyTypes = Union[str, bool] TimeoutTypes = Union[ Optional[float], Tuple[Optional[float], Optional[float], Optional[float], Optional[float]], diff --git a/tests/test_config.py b/tests/test_config.py index 9b9bd64ec2..9f86f83936 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -14,9 +14,10 @@ def test_load_ssl_config(): assert context.check_hostname is True -def test_load_ssl_config_verify_non_existing_path(): +def test_load_ssl_config_verify_non_existing_file(): with pytest.raises(IOError): - httpx.SSLContext(verify="/path/to/nowhere") + context = httpx.SSLContext() + context.load_verify_locations(cafile="/path/to/nowhere") def test_load_ssl_with_keylog(monkeypatch: typing.Any) -> None: @@ -26,20 +27,22 @@ def test_load_ssl_with_keylog(monkeypatch: typing.Any) -> None: def test_load_ssl_config_verify_existing_file(): - context = httpx.SSLContext(verify=certifi.where()) + context = httpx.SSLContext() + context.load_verify_locations(capath=certifi.where()) assert context.verify_mode == ssl.VerifyMode.CERT_REQUIRED assert context.check_hostname is True def test_load_ssl_config_verify_directory(): - path = Path(certifi.where()).parent - context = httpx.SSLContext(verify=str(path)) + context = httpx.SSLContext() + context.load_verify_locations(capath=Path(certifi.where()).parent) assert context.verify_mode == ssl.VerifyMode.CERT_REQUIRED assert context.check_hostname is True def test_load_ssl_config_cert_and_key(cert_pem_file, cert_private_key_file): - context = httpx.SSLContext(cert=(cert_pem_file, cert_private_key_file)) + context = httpx.SSLContext() + context.load_cert_chain(cert_pem_file, cert_private_key_file) assert context.verify_mode == ssl.VerifyMode.CERT_REQUIRED assert context.check_hostname is True @@ -48,9 +51,8 @@ def test_load_ssl_config_cert_and_key(cert_pem_file, cert_private_key_file): def test_load_ssl_config_cert_and_encrypted_key( cert_pem_file, cert_encrypted_private_key_file, password ): - context = httpx.SSLContext( - cert=(cert_pem_file, cert_encrypted_private_key_file, password) - ) + context = httpx.SSLContext() + context.load_cert_chain(cert_pem_file, cert_encrypted_private_key_file, password) assert context.verify_mode == ssl.VerifyMode.CERT_REQUIRED assert context.check_hostname is True @@ -59,14 +61,16 @@ def test_load_ssl_config_cert_and_key_invalid_password( cert_pem_file, cert_encrypted_private_key_file ): with pytest.raises(ssl.SSLError): - httpx.SSLContext( - cert=(cert_pem_file, cert_encrypted_private_key_file, "password1") + context = httpx.SSLContext() + context.load_cert_chain( + cert_pem_file, cert_encrypted_private_key_file, "password1" ) def test_load_ssl_config_cert_without_key_raises(cert_pem_file): with pytest.raises(ssl.SSLError): - httpx.SSLContext(cert=cert_pem_file) + context = httpx.SSLContext() + context.load_cert_chain(cert_pem_file) def test_load_ssl_config_no_verify(): @@ -76,7 +80,8 @@ def test_load_ssl_config_no_verify(): def test_SSLContext_with_get_request(server, cert_pem_file): - context = httpx.SSLContext(verify=cert_pem_file) + context = httpx.SSLContext() + context.load_verify_locations(cert_pem_file) response = httpx.get(server.url, ssl_context=context) assert response.status_code == 200 @@ -84,11 +89,7 @@ def test_SSLContext_with_get_request(server, cert_pem_file): def test_SSLContext_repr(): ssl_context = httpx.SSLContext() - assert repr(ssl_context) == "SSLContext(verify=True)" - - ssl_context = httpx.SSLContext(verify=certifi.where()) - - assert repr(ssl_context) == "SSLContext(verify='{}')".format(certifi.where()) + assert repr(ssl_context) == "" def test_limits_repr(): diff --git a/tests/test_utils.py b/tests/test_utils.py index 3639ea7710..f7e6c1642a 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -3,7 +3,6 @@ import os import random -import certifi import pytest import httpx @@ -119,26 +118,6 @@ def test_logging_redirect_chain(server, caplog): ] -def test_logging_ssl(caplog): - caplog.set_level(logging.DEBUG) - with httpx.Client(): - pass - - cafile = certifi.where() - assert caplog.record_tuples == [ - ( - "httpx", - logging.DEBUG, - "load_ssl_context verify=True cert=None", - ), - ( - "httpx", - logging.DEBUG, - f"load_verify_locations cafile='{cafile}'", - ), - ] - - @pytest.mark.parametrize( ["environment", "proxies"], [ From 69b0f557c71c59f209fbceb50a8b494608ffc881 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Sat, 12 Oct 2024 18:11:58 +0100 Subject: [PATCH 07/17] Drop `_compat.py` module. (#3343) --- httpx/_compat.py | 38 -------------------------------------- httpx/_decoders.py | 32 +++++++++++++++++++++++++------- pyproject.toml | 2 +- 3 files changed, 26 insertions(+), 46 deletions(-) delete mode 100644 httpx/_compat.py diff --git a/httpx/_compat.py b/httpx/_compat.py deleted file mode 100644 index 00b6e29ad8..0000000000 --- a/httpx/_compat.py +++ /dev/null @@ -1,38 +0,0 @@ -""" -The _compat module is used for code which requires branching between different -Python environments. It is excluded from the code coverage checks. -""" - -import re -from types import ModuleType -from typing import Optional - -# Brotli support is optional -# The C bindings in `brotli` are recommended for CPython. -# The CFFI bindings in `brotlicffi` are recommended for PyPy and everything else. -try: - import brotlicffi as brotli -except ImportError: # pragma: no cover - try: - import brotli - except ImportError: - brotli = None - -# Zstandard support is optional -zstd: Optional[ModuleType] = None -try: - import zstandard as zstd -except (AttributeError, ImportError, ValueError): # Defensive: - zstd = None -else: - # The package 'zstandard' added the 'eof' property starting - # in v0.18.0 which we require to ensure a complete and - # valid zstd stream was fed into the ZstdDecoder. - # See: https://github.com/urllib3/urllib3/pull/2624 - _zstd_version = tuple( - map(int, re.search(r"^([0-9]+)\.([0-9]+)", zstd.__version__).groups()) # type: ignore[union-attr] - ) - if _zstd_version < (0, 18): # Defensive: - zstd = None - -__all__ = ["brotli", "zstd"] diff --git a/httpx/_decoders.py b/httpx/_decoders.py index 62f2c0b911..180898c53f 100644 --- a/httpx/_decoders.py +++ b/httpx/_decoders.py @@ -11,9 +11,27 @@ import typing import zlib -from ._compat import brotli, zstd from ._exceptions import DecodingError +# Brotli support is optional +try: + # The C bindings in `brotli` are recommended for CPython. + import brotli +except ImportError: # pragma: no cover + try: + # The CFFI bindings in `brotlicffi` are recommended for PyPy + # and other environments. + import brotlicffi as brotli + except ImportError: + brotli = None + + +# Zstandard support is optional +try: + import zstandard +except ImportError: # pragma: no cover + zstandard = None # type: ignore + class ContentDecoder: def decode(self, data: bytes) -> bytes: @@ -150,24 +168,24 @@ class ZStandardDecoder(ContentDecoder): # inspired by the ZstdDecoder implementation in urllib3 def __init__(self) -> None: - if zstd is None: # pragma: no cover + if zstandard is None: # pragma: no cover raise ImportError( "Using 'ZStandardDecoder', ..." "Make sure to install httpx using `pip install httpx[zstd]`." ) from None - self.decompressor = zstd.ZstdDecompressor().decompressobj() + self.decompressor = zstandard.ZstdDecompressor().decompressobj() def decode(self, data: bytes) -> bytes: - assert zstd is not None + assert zstandard is not None output = io.BytesIO() try: output.write(self.decompressor.decompress(data)) while self.decompressor.eof and self.decompressor.unused_data: unused_data = self.decompressor.unused_data - self.decompressor = zstd.ZstdDecompressor().decompressobj() + self.decompressor = zstandard.ZstdDecompressor().decompressobj() output.write(self.decompressor.decompress(unused_data)) - except zstd.ZstdError as exc: + except zstandard.ZstdError as exc: raise DecodingError(str(exc)) from exc return output.getvalue() @@ -367,5 +385,5 @@ def flush(self) -> list[str]: if brotli is None: SUPPORTED_DECODERS.pop("br") # pragma: no cover -if zstd is None: +if zstandard is None: SUPPORTED_DECODERS.pop("zstd") # pragma: no cover diff --git a/pyproject.toml b/pyproject.toml index 2b83de5f12..9e67191135 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -128,5 +128,5 @@ markers = [ ] [tool.coverage.run] -omit = ["venv/*", "httpx/_compat.py"] +omit = ["venv/*"] include = ["httpx/*", "tests/*"] From cf042b62eeb0b1ad474a093101bd04e7d2d977cc Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 17 Oct 2024 15:29:25 +0100 Subject: [PATCH 08/17] Cleanup `response.elapsed` implementation. (#3345) --- httpx/_client.py | 28 +++++++++++++--------------- httpx/_utils.py | 17 ----------------- 2 files changed, 13 insertions(+), 32 deletions(-) diff --git a/httpx/_client.py b/httpx/_client.py index 8ba5f123a3..05d0322905 100644 --- a/httpx/_client.py +++ b/httpx/_client.py @@ -4,6 +4,7 @@ import enum import logging import ssl +import time import typing import warnings from contextlib import asynccontextmanager, contextmanager @@ -46,7 +47,6 @@ ) from ._urls import URL, QueryParams from ._utils import ( - Timer, URLPattern, get_environment_proxies, is_https_redirect, @@ -113,19 +113,19 @@ class BoundSyncStream(SyncByteStream): """ def __init__( - self, stream: SyncByteStream, response: Response, timer: Timer + self, stream: SyncByteStream, response: Response, start: float ) -> None: self._stream = stream self._response = response - self._timer = timer + self._start = start def __iter__(self) -> typing.Iterator[bytes]: for chunk in self._stream: yield chunk def close(self) -> None: - seconds = self._timer.sync_elapsed() - self._response.elapsed = datetime.timedelta(seconds=seconds) + elapsed = time.perf_counter() - self._start + self._response.elapsed = datetime.timedelta(seconds=elapsed) self._stream.close() @@ -136,19 +136,19 @@ class BoundAsyncStream(AsyncByteStream): """ def __init__( - self, stream: AsyncByteStream, response: Response, timer: Timer + self, stream: AsyncByteStream, response: Response, start: float ) -> None: self._stream = stream self._response = response - self._timer = timer + self._start = start async def __aiter__(self) -> typing.AsyncIterator[bytes]: async for chunk in self._stream: yield chunk async def aclose(self) -> None: - seconds = await self._timer.async_elapsed() - self._response.elapsed = datetime.timedelta(seconds=seconds) + elapsed = time.perf_counter() - self._start + self._response.elapsed = datetime.timedelta(seconds=elapsed) await self._stream.aclose() @@ -963,8 +963,7 @@ def _send_single_request(self, request: Request) -> Response: Sends a single request, without handling any redirections. """ transport = self._transport_for_url(request.url) - timer = Timer() - timer.sync_start() + start = time.perf_counter() if not isinstance(request.stream, SyncByteStream): raise RuntimeError( @@ -978,7 +977,7 @@ def _send_single_request(self, request: Request) -> Response: response.request = request response.stream = BoundSyncStream( - response.stream, response=response, timer=timer + response.stream, response=response, start=start ) self.cookies.extract_cookies(response) response.default_encoding = self._default_encoding @@ -1666,8 +1665,7 @@ async def _send_single_request(self, request: Request) -> Response: Sends a single request, without handling any redirections. """ transport = self._transport_for_url(request.url) - timer = Timer() - await timer.async_start() + start = time.perf_counter() if not isinstance(request.stream, AsyncByteStream): raise RuntimeError( @@ -1680,7 +1678,7 @@ async def _send_single_request(self, request: Request) -> Response: assert isinstance(response.stream, AsyncByteStream) response.request = request response.stream = BoundAsyncStream( - response.stream, response=response, timer=timer + response.stream, response=response, start=start ) self.cookies.extract_cookies(response) response.default_encoding = self._default_encoding diff --git a/httpx/_utils.py b/httpx/_utils.py index 9256745b83..fcf4b64cb8 100644 --- a/httpx/_utils.py +++ b/httpx/_utils.py @@ -6,7 +6,6 @@ import mimetypes import os import re -import time import typing from urllib.request import getproxies @@ -275,22 +274,6 @@ def peek_filelike_length(stream: typing.Any) -> int | None: return length -class Timer: - def sync_start(self) -> None: - self.started = time.perf_counter() - - async def async_start(self) -> None: - self.started = time.perf_counter() - - def sync_elapsed(self) -> float: - now = time.perf_counter() - return now - self.started - - async def async_elapsed(self) -> float: - now = time.perf_counter() - return now - self.started - - class URLPattern: """ A utility class currently used for making lookups against proxy keys... From 8bceb606b3492aaa1dd670ebc141bdaa524a81c4 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Thu, 17 Oct 2024 18:24:05 +0300 Subject: [PATCH 09/17] Warn with `stacklevel=2` to show caller of deprecation (#3136) --- httpx/_client.py | 4 ++-- httpx/_content.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/httpx/_client.py b/httpx/_client.py index 05d0322905..07fa4ff742 100644 --- a/httpx/_client.py +++ b/httpx/_client.py @@ -767,7 +767,7 @@ def request( "the expected behaviour on cookie persistence is ambiguous. Set " "cookies directly on the client instance instead." ) - warnings.warn(message, DeprecationWarning) + warnings.warn(message, DeprecationWarning, stacklevel=2) request = self.build_request( method=method, @@ -1468,7 +1468,7 @@ async def request( "the expected behaviour on cookie persistence is ambiguous. Set " "cookies directly on the client instance instead." ) - warnings.warn(message, DeprecationWarning) + warnings.warn(message, DeprecationWarning, stacklevel=2) request = self.build_request( method=method, diff --git a/httpx/_content.py b/httpx/_content.py index 786699f38f..6e8ad98d08 100644 --- a/httpx/_content.py +++ b/httpx/_content.py @@ -201,7 +201,7 @@ def encode_request( # `data=` usages. We deal with that case here, treating it # as if `content=<...>` had been supplied instead. message = "Use 'content=<...>' to upload raw bytes/text content." - warnings.warn(message, DeprecationWarning) + warnings.warn(message, DeprecationWarning, stacklevel=2) return encode_content(data) if content is not None: From ce270b9f3459c68372628a6011d1f3177dc2b3ee Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 28 Oct 2024 13:52:50 +0000 Subject: [PATCH 10/17] Allow deprecated `verify=<...>` and `cert=<...>`. (#3366) --- httpx/__init__.py | 1 + httpx/_api.py | 45 +++++++++++++++++++++++++++++++ httpx/_client.py | 39 +++++++++++++++++++++++++++ httpx/_config.py | 51 +++++++++++++++++++++++++++++++++++- httpx/_transports/default.py | 20 +++++++++++--- 5 files changed, 152 insertions(+), 4 deletions(-) diff --git a/httpx/__init__.py b/httpx/__init__.py index 3226d9ad02..dc90b90850 100644 --- a/httpx/__init__.py +++ b/httpx/__init__.py @@ -46,6 +46,7 @@ def main() -> None: # type: ignore "ConnectTimeout", "CookieConflict", "Cookies", + "create_ssl_context", "DecodingError", "delete", "DigestAuth", diff --git a/httpx/_api.py b/httpx/_api.py index 98fc40409f..2d352556f9 100644 --- a/httpx/_api.py +++ b/httpx/_api.py @@ -50,6 +50,9 @@ def request( follow_redirects: bool = False, ssl_context: ssl.SSLContext | None = None, trust_env: bool = True, + # Deprecated in favor of `ssl_context`... + verify: typing.Any = None, + cert: typing.Any = None, ) -> Response: """ Sends an HTTP request. @@ -101,6 +104,8 @@ def request( ssl_context=ssl_context, timeout=timeout, trust_env=trust_env, + verify=verify, + cert=cert, ) as client: return client.request( method=method, @@ -134,6 +139,9 @@ def stream( follow_redirects: bool = False, ssl_context: ssl.SSLContext | None = None, trust_env: bool = True, + # Deprecated in favor of `ssl_context`... + verify: typing.Any = None, + cert: typing.Any = None, ) -> typing.Iterator[Response]: """ Alternative to `httpx.request()` that streams the response body @@ -151,6 +159,8 @@ def stream( ssl_context=ssl_context, timeout=timeout, trust_env=trust_env, + verify=verify, + cert=cert, ) as client: with client.stream( method=method, @@ -179,6 +189,9 @@ def get( ssl_context: ssl.SSLContext | None = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, trust_env: bool = True, + # Deprecated in favor of `ssl_context`... + verify: typing.Any = None, + cert: typing.Any = None, ) -> Response: """ Sends a `GET` request. @@ -200,6 +213,8 @@ def get( ssl_context=ssl_context, timeout=timeout, trust_env=trust_env, + verify=verify, + cert=cert, ) @@ -215,6 +230,9 @@ def options( ssl_context: ssl.SSLContext | None = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, trust_env: bool = True, + # Deprecated in favor of `ssl_context`... + verify: typing.Any = None, + cert: typing.Any = None, ) -> Response: """ Sends an `OPTIONS` request. @@ -236,6 +254,8 @@ def options( ssl_context=ssl_context, timeout=timeout, trust_env=trust_env, + verify=verify, + cert=cert, ) @@ -251,6 +271,9 @@ def head( ssl_context: ssl.SSLContext | None = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, trust_env: bool = True, + # Deprecated in favor of `ssl_context`... + verify: typing.Any = None, + cert: typing.Any = None, ) -> Response: """ Sends a `HEAD` request. @@ -272,6 +295,8 @@ def head( ssl_context=ssl_context, timeout=timeout, trust_env=trust_env, + verify=verify, + cert=cert, ) @@ -291,6 +316,9 @@ def post( ssl_context: ssl.SSLContext | None = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, trust_env: bool = True, + # Deprecated in favor of `ssl_context`... + verify: typing.Any = None, + cert: typing.Any = None, ) -> Response: """ Sends a `POST` request. @@ -313,6 +341,8 @@ def post( ssl_context=ssl_context, timeout=timeout, trust_env=trust_env, + verify=verify, + cert=cert, ) @@ -332,6 +362,9 @@ def put( ssl_context: ssl.SSLContext | None = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, trust_env: bool = True, + # Deprecated in favor of `ssl_context`... + verify: typing.Any = None, + cert: typing.Any = None, ) -> Response: """ Sends a `PUT` request. @@ -354,6 +387,8 @@ def put( ssl_context=ssl_context, timeout=timeout, trust_env=trust_env, + verify=verify, + cert=cert, ) @@ -373,6 +408,9 @@ def patch( ssl_context: ssl.SSLContext | None = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, trust_env: bool = True, + # Deprecated in favor of `ssl_context`... + verify: typing.Any = None, + cert: typing.Any = None, ) -> Response: """ Sends a `PATCH` request. @@ -395,6 +433,8 @@ def patch( ssl_context=ssl_context, timeout=timeout, trust_env=trust_env, + verify=verify, + cert=cert, ) @@ -410,6 +450,9 @@ def delete( timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, ssl_context: ssl.SSLContext | None = None, trust_env: bool = True, + # Deprecated in favor of `ssl_context`... + verify: typing.Any = None, + cert: typing.Any = None, ) -> Response: """ Sends a `DELETE` request. @@ -431,4 +474,6 @@ def delete( ssl_context=ssl_context, timeout=timeout, trust_env=trust_env, + verify=verify, + cert=cert, ) diff --git a/httpx/_client.py b/httpx/_client.py index 07fa4ff742..4700ea7165 100644 --- a/httpx/_client.py +++ b/httpx/_client.py @@ -628,6 +628,9 @@ def __init__( transport: BaseTransport | None = None, trust_env: bool = True, default_encoding: str | typing.Callable[[bytes], str] = "utf-8", + # Deprecated in favor of `ssl_context`... + verify: typing.Any = None, + cert: typing.Any = None, ) -> None: super().__init__( auth=auth, @@ -661,6 +664,10 @@ def __init__( http2=http2, limits=limits, transport=transport, + trust_env=trust_env, + # Deprecated in favor of ssl_context... + verify=verify, + cert=cert, ) self._mounts: dict[URLPattern, BaseTransport | None] = { URLPattern(key): None @@ -671,6 +678,9 @@ def __init__( http1=http1, http2=http2, limits=limits, + # Deprecated in favor of ssl_context... + verify=verify, + cert=cert, ) for key, proxy in proxy_map.items() } @@ -689,6 +699,9 @@ def _init_transport( limits: Limits = DEFAULT_LIMITS, transport: BaseTransport | None = None, trust_env: bool = True, + # Deprecated in favor of `ssl_context`... + verify: typing.Any = None, + cert: typing.Any = None, ) -> BaseTransport: if transport is not None: return transport @@ -698,6 +711,8 @@ def _init_transport( http1=http1, http2=http2, limits=limits, + verify=verify, + cert=cert, ) def _init_proxy_transport( @@ -708,6 +723,9 @@ def _init_proxy_transport( http2: bool = False, limits: Limits = DEFAULT_LIMITS, trust_env: bool = True, + # Deprecated in favor of `ssl_context`... + verify: typing.Any = None, + cert: typing.Any = None, ) -> BaseTransport: return HTTPTransport( ssl_context=ssl_context, @@ -715,6 +733,8 @@ def _init_proxy_transport( http2=http2, limits=limits, proxy=proxy, + verify=verify, + cert=cert, ) def _transport_for_url(self, url: URL) -> BaseTransport: @@ -1330,6 +1350,9 @@ def __init__( transport: AsyncBaseTransport | None = None, trust_env: bool = True, default_encoding: str | typing.Callable[[bytes], str] = "utf-8", + # Deprecated in favor of `ssl_context`... + verify: typing.Any = None, + cert: typing.Any = None, ) -> None: super().__init__( auth=auth, @@ -1363,6 +1386,9 @@ def __init__( http2=http2, limits=limits, transport=transport, + # Deprecated in favor of ssl_context + verify=verify, + cert=cert, ) self._mounts: dict[URLPattern, AsyncBaseTransport | None] = { @@ -1374,6 +1400,9 @@ def __init__( http1=http1, http2=http2, limits=limits, + # Deprecated in favor of `ssl_context`... + verify=verify, + cert=cert, ) for key, proxy in proxy_map.items() } @@ -1390,6 +1419,9 @@ def _init_transport( http2: bool = False, limits: Limits = DEFAULT_LIMITS, transport: AsyncBaseTransport | None = None, + # Deprecated in favor of `ssl_context`... + verify: typing.Any = None, + cert: typing.Any = None, ) -> AsyncBaseTransport: if transport is not None: return transport @@ -1399,6 +1431,8 @@ def _init_transport( http1=http1, http2=http2, limits=limits, + verify=verify, + cert=cert, ) def _init_proxy_transport( @@ -1408,6 +1442,9 @@ def _init_proxy_transport( http1: bool = True, http2: bool = False, limits: Limits = DEFAULT_LIMITS, + # Deprecated in favor of `ssl_context`... + verify: typing.Any = None, + cert: typing.Any = None, ) -> AsyncBaseTransport: return AsyncHTTPTransport( ssl_context=ssl_context, @@ -1415,6 +1452,8 @@ def _init_proxy_transport( http2=http2, limits=limits, proxy=proxy, + verify=verify, + cert=cert, ) def _transport_for_url(self, url: URL) -> AsyncBaseTransport: diff --git a/httpx/_config.py b/httpx/_config.py index f701a331ca..d384f80301 100644 --- a/httpx/_config.py +++ b/httpx/_config.py @@ -4,6 +4,7 @@ import ssl import sys import typing +import warnings import certifi @@ -11,7 +12,7 @@ from ._types import HeaderTypes, TimeoutTypes from ._urls import URL -__all__ = ["Limits", "Proxy", "SSLContext", "Timeout"] +__all__ = ["Limits", "Proxy", "SSLContext", "Timeout", "create_ssl_context"] class UnsetType: @@ -21,6 +22,54 @@ class UnsetType: UNSET = UnsetType() +def create_ssl_context( + verify: typing.Any = None, + cert: typing.Any = None, + trust_env: bool = True, + http2: bool = False, +) -> ssl.SSLContext: # pragma: nocover + if isinstance(verify, bool): + ssl_context: ssl.SSLContext = SSLContext(verify=verify) + warnings.warn( + "The verify= parameter is deprecated since 0.28.0. " + "Use `ssl_context=httpx.SSLContext(verify=)`." + ) + elif isinstance(verify, str): + warnings.warn( + "The verify= parameter is deprecated since 0.28.0. " + "Use `ssl_context=httpx.SSLContext()` and `.load_verify_locations()`." + ) + ssl_context = SSLContext() + if os.path.isfile(verify): + ssl_context.load_verify_locations(cafile=verify) + elif os.path.isdir(verify): + ssl_context.load_verify_locations(capath=verify) + elif isinstance(verify, ssl.SSLContext): + warnings.warn( + "The verify= parameter is deprecated since 0.28.0. " + "Use `ssl_context = httpx.SSLContext()`." + ) + ssl_context = verify + else: + warnings.warn( + "`create_ssl_context()` is deprecated since 0.28.0." + "Use `ssl_context = httpx.SSLContext()`." + ) + ssl_context = SSLContext() + + if cert is not None: + warnings.warn( + "The `cert=<...>` parameter is deprecated since 0.28.0. " + "Use `ssl_context = httpx.SSLContext()` and `.load_cert_chain()`." + ) + if isinstance(cert, str): + ssl_context.load_cert_chain(cert) + else: + ssl_context.load_cert_chain(*cert) + + return ssl_context + + class SSLContext(ssl.SSLContext): def __init__( self, diff --git a/httpx/_transports/default.py b/httpx/_transports/default.py index f20e2a37e9..a1978c5ae9 100644 --- a/httpx/_transports/default.py +++ b/httpx/_transports/default.py @@ -33,7 +33,7 @@ import httpcore -from .._config import DEFAULT_LIMITS, Limits, Proxy, SSLContext +from .._config import DEFAULT_LIMITS, Limits, Proxy, SSLContext, create_ssl_context from .._exceptions import ( ConnectError, ConnectTimeout, @@ -134,9 +134,16 @@ def __init__( local_address: str | None = None, retries: int = 0, socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + # Deprecated... + verify: typing.Any = None, + cert: typing.Any = None, ) -> None: proxy = Proxy(url=proxy) if isinstance(proxy, (str, URL)) else proxy - ssl_context = ssl_context or SSLContext() + if verify is not None or cert is not None: # pragma: nocover + # Deprecated... + ssl_context = create_ssl_context(verify, cert) + else: + ssl_context = ssl_context or SSLContext() if proxy is None: self._pool = httpcore.ConnectionPool( @@ -273,9 +280,16 @@ def __init__( local_address: str | None = None, retries: int = 0, socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + # Deprecated... + verify: typing.Any = None, + cert: typing.Any = None, ) -> None: proxy = Proxy(url=proxy) if isinstance(proxy, (str, URL)) else proxy - ssl_context = ssl_context or SSLContext() + if verify is not None or cert is not None: # pragma: nocover + # Deprecated... + ssl_context = create_ssl_context(verify, cert) + else: + ssl_context = ssl_context or SSLContext() if proxy is None: self._pool = httpcore.AsyncConnectionPool( From f3dafec561c83f0d568dbe64f39a70ab90fd2cd2 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 28 Oct 2024 13:54:07 +0000 Subject: [PATCH 11/17] Update .github/CONTRIBUTING.md --- .github/CONTRIBUTING.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index f8a8ac10e0..2cbd58004b 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -211,7 +211,8 @@ this is where our previously generated `client.pem` comes in: ``` import httpx -ssl_context = httpx.SSLContext(verify="/path/to/client.pem")) +ssl_context = httpx.SSLContext() +ssl_context.load_verify_locations("/path/to/client.pem") with httpx.Client(proxy="http://127.0.0.1:8080/", ssl_context=ssl_context) as client: response = client.get("https://example.org") From 140f61dafc8554be06bab114fa7acc27192553bf Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 28 Oct 2024 13:56:37 +0000 Subject: [PATCH 12/17] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 208cb1692d..e1401a427c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased * Added `httpx.SSLContext` class and `ssl_context` argument. (#3022) -* Removed `cert` and `verify` arguments, you should use the `ssl_context=...` instead. (#3022) +* The `verify` and `cert` arguments continue to function, but are deprecated and will raise warnings. Use the `ssl_context = httpx.SSLContext()` API instead. (#3022, #3335) * The deprecated `proxies` argument has now been removed. * The deprecated `app` argument has now been removed. * The `URL.raw` property has now been removed. From f0b7c0dc9a0b0d914f71450c34610f2838d2a05e Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 28 Oct 2024 13:57:06 +0000 Subject: [PATCH 13/17] Update CHANGELOG.md --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e1401a427c..4c22049a7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). * The deprecated `proxies` argument has now been removed. * The deprecated `app` argument has now been removed. * The `URL.raw` property has now been removed. -* The `sniffio` project dependency has now been removed. ## 0.27.2 (27th August, 2024) From 1d93ad80872499d0914424e797730e0c2a991d85 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 28 Oct 2024 14:13:22 +0000 Subject: [PATCH 14/17] Version 0.28. (#3370) Co-authored-by: T-256 <132141463+T-256@users.noreply.github.com> --- CHANGELOG.md | 8 +++++--- docs/advanced/ssl.md | 38 ++++---------------------------------- httpx/__version__.py | 2 +- httpx/_config.py | 2 ++ 4 files changed, 12 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c22049a7c..122fa74749 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,10 +4,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). -## Unreleased +## 0.28.0 (28th October, 2024) -* Added `httpx.SSLContext` class and `ssl_context` argument. (#3022) -* The `verify` and `cert` arguments continue to function, but are deprecated and will raise warnings. Use the `ssl_context = httpx.SSLContext()` API instead. (#3022, #3335) +Version 0.28.0 introduces an `httpx.SSLContext()` class and `ssl_context` parameter. + +* Added `httpx.SSLContext` class and `ssl_context` parameter. (#3022, #3335) +* The `verify` and `cert` arguments have been deprecated and will now raise warnings. (#3022, #3335) * The deprecated `proxies` argument has now been removed. * The deprecated `app` argument has now been removed. * The `URL.raw` property has now been removed. diff --git a/docs/advanced/ssl.md b/docs/advanced/ssl.md index 9e0b2e9955..0c91a38700 100644 --- a/docs/advanced/ssl.md +++ b/docs/advanced/ssl.md @@ -9,7 +9,7 @@ By default httpx will verify HTTPS connections, and raise an error for invalid S httpx.ConnectError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:997) ``` -You can configure the verification using `httpx.SSLContext()`. +You can create and use new SSL context other than the default. ```pycon >>> context = httpx.SSLContext() @@ -31,7 +31,7 @@ You can use this to disable verification completely and allow insecure requests. ### Configuring client instances -If you're using a `Client()` instance you should pass any SSL settings when instantiating the client. +If you're using a `Client()` instance you should pass any SSL context when instantiating the client. ```pycon >>> context = httpx.SSLContext() @@ -40,42 +40,12 @@ If you're using a `Client()` instance you should pass any SSL settings when inst The `client.get(...)` method and other request methods on a `Client` instance *do not* support changing the SSL settings on a per-request basis. -If you need different SSL settings in different cases you should use more that one client instance, with different settings on each. Each client will then be using an isolated connection pool with a specific fixed SSL configuration on all connections within that pool. +If you need different SSL settings in different cases you should use more than one client instance, with different settings on each. Each client will then be using an isolated connection pool with a specific fixed SSL configuration on all connections within that pool. -### Changing the verification defaults +### Configuring certificate stores By default, HTTPX uses the CA bundle provided by [Certifi](https://pypi.org/project/certifi/). -The following all have the same behaviour... - -Using the default SSL context. - -```pycon ->>> client = httpx.Client() ->>> client.get("https://www.example.com") - -``` - -Using the default SSL context, but specified explicitly. - -```pycon ->>> context = httpx.SSLContext() ->>> client = httpx.Client(ssl_context=context) ->>> client.get("https://www.example.com") - -``` - -Using the default SSL context, with `verify=True` specified explicitly. - -```pycon ->>> context = httpx.SSLContext(verify=True) ->>> client = httpx.Client(ssl_context=context) ->>> client.get("https://www.example.com") - -``` - -### Configuring certificate verification - You can load additional certificate verification using the [`.load_verify_locations()`](https://docs.python.org/3/library/ssl.html#ssl.SSLContext.load_verify_locations) API: ```pycon diff --git a/httpx/__version__.py b/httpx/__version__.py index 5eaaddbac9..0a684ac3a9 100644 --- a/httpx/__version__.py +++ b/httpx/__version__.py @@ -1,3 +1,3 @@ __title__ = "httpx" __description__ = "A next generation HTTP client, for Python 3." -__version__ = "0.27.2" +__version__ = "0.28.0" diff --git a/httpx/_config.py b/httpx/_config.py index d384f80301..2c9634a666 100644 --- a/httpx/_config.py +++ b/httpx/_config.py @@ -28,6 +28,8 @@ def create_ssl_context( trust_env: bool = True, http2: bool = False, ) -> ssl.SSLContext: # pragma: nocover + # The `create_ssl_context` helper function is now deprecated + # in favour of `httpx.SSLContext()`. if isinstance(verify, bool): ssl_context: ssl.SSLContext = SSLContext(verify=verify) warnings.warn( From d43a10e2eddb65b2ac2de8052f08311b9efb7943 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 28 Oct 2024 14:16:53 +0000 Subject: [PATCH 15/17] Update CHANGELOG.md --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 122fa74749..e419ed8c43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). -## 0.28.0 (28th October, 2024) +## Unreleased -Version 0.28.0 introduces an `httpx.SSLContext()` class and `ssl_context` parameter. +Version <...> introduces an `httpx.SSLContext()` class and `ssl_context` parameter. * Added `httpx.SSLContext` class and `ssl_context` parameter. (#3022, #3335) * The `verify` and `cert` arguments have been deprecated and will now raise warnings. (#3022, #3335) From 1cb3fda5bb7b5916eddeb4021864596212aa2292 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 28 Oct 2024 14:22:01 +0000 Subject: [PATCH 16/17] Update docs/advanced/ssl.md --- docs/advanced/ssl.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/advanced/ssl.md b/docs/advanced/ssl.md index 0c91a38700..57553515ab 100644 --- a/docs/advanced/ssl.md +++ b/docs/advanced/ssl.md @@ -9,7 +9,7 @@ By default httpx will verify HTTPS connections, and raise an error for invalid S httpx.ConnectError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:997) ``` -You can create and use new SSL context other than the default. +Verification is configured through [the SSL Context API](https://docs.python.org/3/library/ssl.html#ssl-contexts). ```pycon >>> context = httpx.SSLContext() From e83069830fa016b509377dcc4af713615fe44f4f Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 28 Oct 2024 14:25:03 +0000 Subject: [PATCH 17/17] Apply suggestions from code review --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e419ed8c43..1ec91e3b75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). -## Unreleased +## Version 0.28.0 -Version <...> introduces an `httpx.SSLContext()` class and `ssl_context` parameter. +Version 0.28.0 introduces an `httpx.SSLContext()` class and `ssl_context` parameter. * Added `httpx.SSLContext` class and `ssl_context` parameter. (#3022, #3335) * The `verify` and `cert` arguments have been deprecated and will now raise warnings. (#3022, #3335)