From 018f8ff58c71ffdfc2a29e06136efa8b0e3e4597 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Wed, 4 Dec 2024 10:51:00 +0000 Subject: [PATCH 01/11] Initial implementation of uploading with trusted publishing authentication --- pyproject.toml | 1 + twine/auth.py | 36 +++++++++++++++++++++++++++++++++++- twine/settings.py | 11 +++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 582ccaa9..450ca2c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,6 +62,7 @@ register = "twine.commands.register:main" [project.optional-dependencies] keyring = ["keyring >= 15.1"] +oidc = ["id"] [project.scripts] twine = "twine.__main__:main" diff --git a/twine/auth.py b/twine/auth.py index 59e2e21d..916b050f 100644 --- a/twine/auth.py +++ b/twine/auth.py @@ -2,6 +2,9 @@ import getpass import logging from typing import TYPE_CHECKING, Callable, Optional, Type, cast +from urllib.parse import urlparse + +import requests # keyring has an indirect dependency on PyCA cryptography, which has no # pre-built wheels for ppc64le and s390x, see #1158. @@ -28,9 +31,12 @@ def __init__( class Resolver: - def __init__(self, config: utils.RepositoryConfig, input: CredentialInput) -> None: + def __init__( + self, config: utils.RepositoryConfig, input: CredentialInput, oidc: bool = False + ) -> None: self.config = config self.input = input + self.oidc = oidc @classmethod def choose(cls, interactive: bool) -> Type["Resolver"]: @@ -53,6 +59,25 @@ def username(self) -> Optional[str]: @property @functools.lru_cache() def password(self) -> Optional[str]: + if self.oidc: + # Trusted publishing (OpenID Connect): get one token from the CI + # system, and exchange that for a PyPI token. + from id import detect_credential + + repository_domain = urlparse(self.system).netloc + audience = self._oidc_audience(repository_domain) + oidc_token = detect_credential(audience) + + token_exchange_url = f'https://{repository_domain}/_/oidc/mint-token' + + mint_token_resp = requests.post( + token_exchange_url, + json={'token': oidc_token}, + timeout=5, # S113 wants a timeout + ) + mint_token_resp.raise_for_status() + return mint_token_resp.json()['token'] + return utils.get_userpass_value( self.input.password, self.config, @@ -60,6 +85,15 @@ def password(self) -> Optional[str]: prompt_strategy=self.password_from_keyring_or_prompt, ) + @staticmethod + def _oidc_audience(repository_domain): + # Indices are expected to support `https://{domain}/_/oidc/audience`, + # which tells OIDC exchange clients which audience to use. + audience_url = f'https://{repository_domain}/_/oidc/audience' + resp = requests.get(audience_url, timeout=5) + resp.raise_for_status() + return resp.json()['audience'] + @property def system(self) -> Optional[str]: return self.config["repository"] diff --git a/twine/settings.py b/twine/settings.py index 5aa4d964..2dc27591 100644 --- a/twine/settings.py +++ b/twine/settings.py @@ -51,6 +51,7 @@ def __init__( identity: Optional[str] = None, username: Optional[str] = None, password: Optional[str] = None, + trusted_publish: bool = False, non_interactive: bool = False, comment: Optional[str] = None, config_file: str = utils.DEFAULT_CONFIG_FILE, @@ -128,6 +129,7 @@ def __init__( self.auth = auth.Resolver.choose(not non_interactive)( self.repository_config, auth.CredentialInput(username, password), + oidc=trusted_publish, ) @property @@ -222,6 +224,15 @@ def register_argparse_arguments(parser: argparse.ArgumentParser) -> None: "(package index) with. (Can also be set via " "%(env)s environment variable.)", ) + parser.add_argument( + "--trusted-publish", + default=False, + required=False, + action="store_true", + help="Upload from CI using trusted publishing. Use this without " + "specifying username & password. Requires an optional extra " + "dependency (install twine[oidc]).", + ) parser.add_argument( "--non-interactive", action=utils.EnvironmentFlag, From 5ffbfa0e028f76a2df47eaa02b63a8b7987a1f3d Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Wed, 4 Dec 2024 12:38:24 +0100 Subject: [PATCH 02/11] Autoformat --- twine/auth.py | 10 +++++----- twine/settings.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/twine/auth.py b/twine/auth.py index 916b050f..819119a4 100644 --- a/twine/auth.py +++ b/twine/auth.py @@ -68,15 +68,15 @@ def password(self) -> Optional[str]: audience = self._oidc_audience(repository_domain) oidc_token = detect_credential(audience) - token_exchange_url = f'https://{repository_domain}/_/oidc/mint-token' + token_exchange_url = f"https://{repository_domain}/_/oidc/mint-token" mint_token_resp = requests.post( token_exchange_url, - json={'token': oidc_token}, + json={"token": oidc_token}, timeout=5, # S113 wants a timeout ) mint_token_resp.raise_for_status() - return mint_token_resp.json()['token'] + return mint_token_resp.json()["token"] return utils.get_userpass_value( self.input.password, @@ -89,10 +89,10 @@ def password(self) -> Optional[str]: def _oidc_audience(repository_domain): # Indices are expected to support `https://{domain}/_/oidc/audience`, # which tells OIDC exchange clients which audience to use. - audience_url = f'https://{repository_domain}/_/oidc/audience' + audience_url = f"https://{repository_domain}/_/oidc/audience" resp = requests.get(audience_url, timeout=5) resp.raise_for_status() - return resp.json()['audience'] + return resp.json()["audience"] @property def system(self) -> Optional[str]: diff --git a/twine/settings.py b/twine/settings.py index 2dc27591..d8c79fde 100644 --- a/twine/settings.py +++ b/twine/settings.py @@ -231,7 +231,7 @@ def register_argparse_arguments(parser: argparse.ArgumentParser) -> None: action="store_true", help="Upload from CI using trusted publishing. Use this without " "specifying username & password. Requires an optional extra " - "dependency (install twine[oidc]).", + "dependency (install twine[oidc]).", ) parser.add_argument( "--non-interactive", From 93843544abef0186542ca14dc9c168510905ebf2 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Wed, 4 Dec 2024 12:55:27 +0100 Subject: [PATCH 03/11] Satisfy the type checks --- tox.ini | 1 + twine/auth.py | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tox.ini b/tox.ini index e5c71aab..b36ef5d1 100644 --- a/tox.ini +++ b/tox.ini @@ -87,6 +87,7 @@ deps = lxml >= 5.2.0, != 5.2.1 # required for more thorough type declarations keyring >= 22.3 + id # consider replacing with `mypy --install-types` when # https://github.com/python/mypy/issues/10600 is resolved types-requests diff --git a/twine/auth.py b/twine/auth.py index 819119a4..d19ba01b 100644 --- a/twine/auth.py +++ b/twine/auth.py @@ -62,9 +62,9 @@ def password(self) -> Optional[str]: if self.oidc: # Trusted publishing (OpenID Connect): get one token from the CI # system, and exchange that for a PyPI token. - from id import detect_credential + from id import detect_credential # type: ignore - repository_domain = urlparse(self.system).netloc + repository_domain = cast(str, urlparse(self.system).netloc) audience = self._oidc_audience(repository_domain) oidc_token = detect_credential(audience) @@ -76,7 +76,7 @@ def password(self) -> Optional[str]: timeout=5, # S113 wants a timeout ) mint_token_resp.raise_for_status() - return mint_token_resp.json()["token"] + return cast(str, mint_token_resp.json()["token"]) return utils.get_userpass_value( self.input.password, @@ -86,13 +86,13 @@ def password(self) -> Optional[str]: ) @staticmethod - def _oidc_audience(repository_domain): + def _oidc_audience(repository_domain: str) -> str: # Indices are expected to support `https://{domain}/_/oidc/audience`, # which tells OIDC exchange clients which audience to use. audience_url = f"https://{repository_domain}/_/oidc/audience" resp = requests.get(audience_url, timeout=5) resp.raise_for_status() - return resp.json()["audience"] + return cast(str, resp.json()["audience"]) @property def system(self) -> Optional[str]: From eb0faf4d9460c703c21d85fb61a4e29c03d6bdc1 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Wed, 4 Dec 2024 15:00:21 +0100 Subject: [PATCH 04/11] Rename oidc -> trusted_publishing --- pyproject.toml | 2 +- twine/auth.py | 9 ++++++--- twine/settings.py | 4 ++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 450ca2c7..146e8b03 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,7 +62,7 @@ register = "twine.commands.register:main" [project.optional-dependencies] keyring = ["keyring >= 15.1"] -oidc = ["id"] +trusted-publishing = ["id"] [project.scripts] twine = "twine.__main__:main" diff --git a/twine/auth.py b/twine/auth.py index d19ba01b..90bcb182 100644 --- a/twine/auth.py +++ b/twine/auth.py @@ -32,11 +32,14 @@ def __init__( class Resolver: def __init__( - self, config: utils.RepositoryConfig, input: CredentialInput, oidc: bool = False + self, + config: utils.RepositoryConfig, + input: CredentialInput, + trusted_publishing: bool = False, ) -> None: self.config = config self.input = input - self.oidc = oidc + self.trusted_publishing = trusted_publishing @classmethod def choose(cls, interactive: bool) -> Type["Resolver"]: @@ -59,7 +62,7 @@ def username(self) -> Optional[str]: @property @functools.lru_cache() def password(self) -> Optional[str]: - if self.oidc: + if self.trusted_publishing: # Trusted publishing (OpenID Connect): get one token from the CI # system, and exchange that for a PyPI token. from id import detect_credential # type: ignore diff --git a/twine/settings.py b/twine/settings.py index d8c79fde..9033ed3b 100644 --- a/twine/settings.py +++ b/twine/settings.py @@ -129,7 +129,7 @@ def __init__( self.auth = auth.Resolver.choose(not non_interactive)( self.repository_config, auth.CredentialInput(username, password), - oidc=trusted_publish, + trusted_publishing=trusted_publish, ) @property @@ -231,7 +231,7 @@ def register_argparse_arguments(parser: argparse.ArgumentParser) -> None: action="store_true", help="Upload from CI using trusted publishing. Use this without " "specifying username & password. Requires an optional extra " - "dependency (install twine[oidc]).", + "dependency (install twine[trusted-publishing]).", ) parser.add_argument( "--non-interactive", From 09198e0d79f8338d80a897178053be3eebedb4d0 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Wed, 4 Dec 2024 20:22:49 +0100 Subject: [PATCH 05/11] Try trusted publishing on upload to PyPI with no token --- twine/auth.py | 59 +++++++++++++++++++++++++++++------------------ twine/settings.py | 11 --------- 2 files changed, 36 insertions(+), 34 deletions(-) diff --git a/twine/auth.py b/twine/auth.py index 90bcb182..09d24bca 100644 --- a/twine/auth.py +++ b/twine/auth.py @@ -16,6 +16,11 @@ except ModuleNotFoundError: # pragma: no cover keyring = None +try: + from id import detect_credential # type: ignore +except ModuleNotFoundError: # pragma: no cover + detect_credential = None + from twine import exceptions from twine import utils @@ -35,11 +40,9 @@ def __init__( self, config: utils.RepositoryConfig, input: CredentialInput, - trusted_publishing: bool = False, ) -> None: self.config = config self.input = input - self.trusted_publishing = trusted_publishing @classmethod def choose(cls, interactive: bool) -> Type["Resolver"]: @@ -62,24 +65,16 @@ def username(self) -> Optional[str]: @property @functools.lru_cache() def password(self) -> Optional[str]: - if self.trusted_publishing: - # Trusted publishing (OpenID Connect): get one token from the CI - # system, and exchange that for a PyPI token. - from id import detect_credential # type: ignore - - repository_domain = cast(str, urlparse(self.system).netloc) - audience = self._oidc_audience(repository_domain) - oidc_token = detect_credential(audience) - - token_exchange_url = f"https://{repository_domain}/_/oidc/mint-token" - - mint_token_resp = requests.post( - token_exchange_url, - json={"token": oidc_token}, - timeout=5, # S113 wants a timeout + if ( + self.is_pypi() + and self.username == "__token__" + and self.input.password is None + and detect_credential is not None + ): + logger.info( + "Trying to use trusted publishing (no token was explicitly provided)" ) - mint_token_resp.raise_for_status() - return cast(str, mint_token_resp.json()["token"]) + return self.make_trusted_publishing_token() return utils.get_userpass_value( self.input.password, @@ -88,14 +83,32 @@ def password(self) -> Optional[str]: prompt_strategy=self.password_from_keyring_or_prompt, ) - @staticmethod - def _oidc_audience(repository_domain: str) -> str: + def make_trusted_publishing_token(self) -> str: + # Trusted publishing (OpenID Connect): get one token from the CI + # system, and exchange that for a PyPI token. + repository_domain = cast(str, urlparse(self.system).netloc) + session = requests.Session() # TODO: user agent & retries + # Indices are expected to support `https://{domain}/_/oidc/audience`, # which tells OIDC exchange clients which audience to use. audience_url = f"https://{repository_domain}/_/oidc/audience" - resp = requests.get(audience_url, timeout=5) + resp = session.get(audience_url, timeout=5) resp.raise_for_status() - return cast(str, resp.json()["audience"]) + audience = cast(str, resp.json()["audience"]) + + oidc_token = detect_credential(audience) + logger.debug("Got OIDC token for audience %s", audience) + + token_exchange_url = f"https://{repository_domain}/_/oidc/mint-token" + + mint_token_resp = session.post( + token_exchange_url, + json={"token": oidc_token}, + timeout=5, # S113 wants a timeout + ) + mint_token_resp.raise_for_status() + logger.debug("Minted upload token for trusted publishing") + return cast(str, mint_token_resp.json()["token"]) @property def system(self) -> Optional[str]: diff --git a/twine/settings.py b/twine/settings.py index 9033ed3b..5aa4d964 100644 --- a/twine/settings.py +++ b/twine/settings.py @@ -51,7 +51,6 @@ def __init__( identity: Optional[str] = None, username: Optional[str] = None, password: Optional[str] = None, - trusted_publish: bool = False, non_interactive: bool = False, comment: Optional[str] = None, config_file: str = utils.DEFAULT_CONFIG_FILE, @@ -129,7 +128,6 @@ def __init__( self.auth = auth.Resolver.choose(not non_interactive)( self.repository_config, auth.CredentialInput(username, password), - trusted_publishing=trusted_publish, ) @property @@ -224,15 +222,6 @@ def register_argparse_arguments(parser: argparse.ArgumentParser) -> None: "(package index) with. (Can also be set via " "%(env)s environment variable.)", ) - parser.add_argument( - "--trusted-publish", - default=False, - required=False, - action="store_true", - help="Upload from CI using trusted publishing. Use this without " - "specifying username & password. Requires an optional extra " - "dependency (install twine[trusted-publishing]).", - ) parser.add_argument( "--non-interactive", action=utils.EnvironmentFlag, From ea780457e13bd9026168d98e6d5293bf34714319 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Wed, 4 Dec 2024 21:00:39 +0100 Subject: [PATCH 06/11] Make id a full dependency --- pyproject.toml | 2 +- twine/auth.py | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 146e8b03..e976d56d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ dependencies = [ "rfc3986 >= 1.4.0", "rich >= 12.0.0", "packaging", + "id", ] dynamic = ["version"] @@ -62,7 +63,6 @@ register = "twine.commands.register:main" [project.optional-dependencies] keyring = ["keyring >= 15.1"] -trusted-publishing = ["id"] [project.scripts] twine = "twine.__main__:main" diff --git a/twine/auth.py b/twine/auth.py index 09d24bca..1e48380b 100644 --- a/twine/auth.py +++ b/twine/auth.py @@ -5,6 +5,7 @@ from urllib.parse import urlparse import requests +from id import detect_credential # type: ignore # keyring has an indirect dependency on PyCA cryptography, which has no # pre-built wheels for ppc64le and s390x, see #1158. @@ -16,11 +17,6 @@ except ModuleNotFoundError: # pragma: no cover keyring = None -try: - from id import detect_credential # type: ignore -except ModuleNotFoundError: # pragma: no cover - detect_credential = None - from twine import exceptions from twine import utils @@ -69,7 +65,6 @@ def password(self) -> Optional[str]: self.is_pypi() and self.username == "__token__" and self.input.password is None - and detect_credential is not None ): logger.info( "Trying to use trusted publishing (no token was explicitly provided)" From 2a8a126e70ee5b93783295634d562319ee5e7204 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Wed, 4 Dec 2024 21:57:49 +0100 Subject: [PATCH 07/11] Try trusted publishing at lower priority than keyring & .pypirc files --- twine/auth.py | 62 ++++++++++++++++++++++++++++++++------------- twine/exceptions.py | 6 +++++ 2 files changed, 51 insertions(+), 17 deletions(-) diff --git a/twine/auth.py b/twine/auth.py index 1e48380b..c5dc4dfb 100644 --- a/twine/auth.py +++ b/twine/auth.py @@ -1,11 +1,13 @@ import functools import getpass +import json import logging from typing import TYPE_CHECKING, Callable, Optional, Type, cast from urllib.parse import urlparse import requests -from id import detect_credential # type: ignore +from id import AmbientCredentialError # type: ignore +from id import detect_credential # keyring has an indirect dependency on PyCA cryptography, which has no # pre-built wheels for ppc64le and s390x, see #1158. @@ -61,24 +63,14 @@ def username(self) -> Optional[str]: @property @functools.lru_cache() def password(self) -> Optional[str]: - if ( - self.is_pypi() - and self.username == "__token__" - and self.input.password is None - ): - logger.info( - "Trying to use trusted publishing (no token was explicitly provided)" - ) - return self.make_trusted_publishing_token() - return utils.get_userpass_value( self.input.password, self.config, key="password", - prompt_strategy=self.password_from_keyring_or_prompt, + prompt_strategy=self.password_from_keyring_or_trusted_publishing_or_prompt, ) - def make_trusted_publishing_token(self) -> str: + def make_trusted_publishing_token(self) -> Optional[str]: # Trusted publishing (OpenID Connect): get one token from the CI # system, and exchange that for a PyPI token. repository_domain = cast(str, urlparse(self.system).netloc) @@ -91,7 +83,20 @@ def make_trusted_publishing_token(self) -> str: resp.raise_for_status() audience = cast(str, resp.json()["audience"]) - oidc_token = detect_credential(audience) + try: + oidc_token = detect_credential(audience) + except AmbientCredentialError as e: + # If we get here, we're on a supported CI platform for trusted + # publishing, and we have not been given any token, so we can error. + raise exceptions.TrustedPublishingFailure( + "Unable to retrieve an OIDC token from the CI platform for " + f"trusted publishing {e}" + ) + + if oidc_token is None: + logger.debug("This environment is not supported for trusted publishing") + return None # Fall back to prompting for a token (if possible) + logger.debug("Got OIDC token for audience %s", audience) token_exchange_url = f"https://{repository_domain}/_/oidc/mint-token" @@ -101,9 +106,25 @@ def make_trusted_publishing_token(self) -> str: json={"token": oidc_token}, timeout=5, # S113 wants a timeout ) - mint_token_resp.raise_for_status() + try: + mint_token_payload = mint_token_resp.json() + except json.JSONDecodeError: + raise exceptions.TrustedPublishingFailure( + "The token-minting request returned invalid JSON" + ) + + if not mint_token_resp.ok: + reasons = "\n".join( + f'* `{error["code"]}`: {error["description"]}' + for error in mint_token_payload["errors"] + ) + raise exceptions.TrustedPublishingFailure( + "The token request failed; the index server gave the following " + f"reasons:\n\n{reasons}" + ) + logger.debug("Minted upload token for trusted publishing") - return cast(str, mint_token_resp.json()["token"]) + return cast(str, mint_token_payload["token"]) @property def system(self) -> Optional[str]: @@ -147,12 +168,19 @@ def username_from_keyring_or_prompt(self) -> str: return self.prompt("username", input) - def password_from_keyring_or_prompt(self) -> str: + def password_from_keyring_or_trusted_publishing_or_prompt(self) -> str: password = self.get_password_from_keyring() if password: logger.info("password set from keyring") return password + if self.is_pypi() and self.username == "__token__": + logger.debug( + "Trying to use trusted publishing (no token was explicitly provided)" + ) + if (token := self.make_trusted_publishing_token()) is not None: + return token + # Prompt for API token when required. what = "API token" if self.is_pypi() else "password" diff --git a/twine/exceptions.py b/twine/exceptions.py index b87b938b..29b7d8a1 100644 --- a/twine/exceptions.py +++ b/twine/exceptions.py @@ -116,6 +116,12 @@ class NonInteractive(TwineException): pass +class TrustedPublishingFailure(TwineException): + """Raised if we expected to use trusted publishing but couldn't.""" + + pass + + class InvalidPyPIUploadURL(TwineException): """Repository configuration tries to use PyPI with an incorrect URL. From d166096b612b487e30f47652869c4c96a6b1f4ae Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Fri, 6 Dec 2024 14:50:49 +0100 Subject: [PATCH 08/11] Add id to requirements shown with --version --- twine/cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/twine/cli.py b/twine/cli.py index 6ff60d4d..f44426e3 100644 --- a/twine/cli.py +++ b/twine/cli.py @@ -81,6 +81,7 @@ def list_dependencies_and_versions() -> List[Tuple[str, str]]: "requests", "requests-toolbelt", "urllib3", + "id", ] if sys.version_info < (3, 10): deps.append("importlib-metadata") From 60b33277deea4a1cbc4fd2fc2343062c5f347b6c Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Sat, 7 Dec 2024 09:49:15 +0100 Subject: [PATCH 09/11] id no longer needs to be listed in twine --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index b36ef5d1..e5c71aab 100644 --- a/tox.ini +++ b/tox.ini @@ -87,7 +87,6 @@ deps = lxml >= 5.2.0, != 5.2.1 # required for more thorough type declarations keyring >= 22.3 - id # consider replacing with `mypy --install-types` when # https://github.com/python/mypy/issues/10600 is resolved types-requests From 7eb4947febbe791e14d16b6f9a41c1bcbd87a17f Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Sun, 8 Dec 2024 10:18:12 +0100 Subject: [PATCH 10/11] Smaller message when keyring installed but finds no backend --- twine/auth.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/twine/auth.py b/twine/auth.py index c5dc4dfb..1131c9c0 100644 --- a/twine/auth.py +++ b/twine/auth.py @@ -13,11 +13,14 @@ # pre-built wheels for ppc64le and s390x, see #1158. if TYPE_CHECKING: import keyring + from keyring.errors import NoKeyringError else: try: import keyring + from keyring.errors import NoKeyringError except ModuleNotFoundError: # pragma: no cover keyring = None + NoKeyringError = None from twine import exceptions from twine import utils @@ -156,6 +159,8 @@ def get_password_from_keyring(self) -> Optional[str]: username = cast(str, self.username) logger.info("Querying keyring for password") return cast(str, keyring.get_password(system, username)) + except NoKeyringError: + logger.info("No keyring backend found") except Exception as exc: logger.warning("Error getting password from keyring", exc_info=exc) return None From ec859fbfd285284e800461c45d22187f7948a275 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Tue, 10 Dec 2024 10:00:36 +0100 Subject: [PATCH 11/11] Set user-agent & retries for OIDC requests --- twine/auth.py | 3 +-- twine/repository.py | 34 +++------------------------------- twine/utils.py | 26 ++++++++++++++++++++++++++ 3 files changed, 30 insertions(+), 33 deletions(-) diff --git a/twine/auth.py b/twine/auth.py index 1131c9c0..49ab8987 100644 --- a/twine/auth.py +++ b/twine/auth.py @@ -5,7 +5,6 @@ from typing import TYPE_CHECKING, Callable, Optional, Type, cast from urllib.parse import urlparse -import requests from id import AmbientCredentialError # type: ignore from id import detect_credential @@ -77,7 +76,7 @@ def make_trusted_publishing_token(self) -> Optional[str]: # Trusted publishing (OpenID Connect): get one token from the CI # system, and exchange that for a PyPI token. repository_domain = cast(str, urlparse(self.system).netloc) - session = requests.Session() # TODO: user agent & retries + session = utils.make_requests_session() # Indices are expected to support `https://{domain}/_/oidc/audience`, # which tells OIDC exchange clients which audience to use. diff --git a/twine/repository.py b/twine/repository.py index b512bfb3..f04f9dbc 100644 --- a/twine/repository.py +++ b/twine/repository.py @@ -12,18 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -from typing import Any, Dict, List, Optional, Set, Tuple, cast +from typing import Any, Dict, List, Optional, Set, Tuple import requests import requests_toolbelt import rich.progress -import urllib3 -from requests import adapters -from requests_toolbelt.utils import user_agent from rich import print -import twine from twine import package as package_file +from twine.utils import make_requests_session KEYWORDS_TO_NOT_FLATTEN = {"gpg_signature", "attestations", "content"} @@ -47,7 +44,7 @@ def __init__( ) -> None: self.url = repository_url - self.session = requests.session() + self.session = make_requests_session() # requests.Session.auth should be Union[None, Tuple[str, str], ...] # But username or password could be None # See TODO for utils.RepositoryConfig @@ -57,35 +54,10 @@ def __init__( logger.info(f"username: {username if username else ''}") logger.info(f"password: <{'hidden' if password else 'empty'}>") - self.session.headers["User-Agent"] = self._make_user_agent_string() - for scheme in ("http://", "https://"): - self.session.mount(scheme, self._make_adapter_with_retries()) - # Working around https://github.com/python/typing/issues/182 self._releases_json_data: Dict[str, Dict[str, Any]] = {} self.disable_progress_bar = disable_progress_bar - @staticmethod - def _make_adapter_with_retries() -> adapters.HTTPAdapter: - retry = urllib3.Retry( - allowed_methods=["GET"], - connect=5, - total=10, - status_forcelist=[500, 501, 502, 503], - ) - - return adapters.HTTPAdapter(max_retries=retry) - - @staticmethod - def _make_user_agent_string() -> str: - user_agent_string = ( - user_agent.UserAgentBuilder("twine", twine.__version__) - .include_implementation() - .build() - ) - - return cast(str, user_agent_string) - def close(self) -> None: self.session.close() diff --git a/twine/utils.py b/twine/utils.py index 10e374e7..2bc56d0b 100644 --- a/twine/utils.py +++ b/twine/utils.py @@ -25,7 +25,11 @@ import requests import rfc3986 +import urllib3 +from requests.adapters import HTTPAdapter +from requests_toolbelt.utils import user_agent +import twine from twine import exceptions # Shim for input to allow testing. @@ -304,6 +308,28 @@ def get_userpass_value( get_clientcert = functools.partial(get_userpass_value, key="client_cert") +def make_requests_session() -> requests.Session: + """Prepare a requests Session with retries & twine's user-agent string.""" + s = requests.Session() + + retry = urllib3.Retry( + allowed_methods=["GET"], + connect=5, + total=10, + status_forcelist=[500, 501, 502, 503], + ) + + for scheme in ("http://", "https://"): + s.mount(scheme, HTTPAdapter(max_retries=retry)) + + s.headers["User-Agent"] = ( + user_agent.UserAgentBuilder("twine", twine.__version__) + .include_implementation() + .build() + ) + return s + + class EnvironmentDefault(argparse.Action): """Get values from environment variable."""