add --keyring-provider flag to configure keyring-based authenticati…

…on (#2592)

Add a `--keyring-provider` flag to allow Pex callers to configure Pip to
use a keyring provider without the need to loosen the Pip isolated mode
entirely as would be done if `--use-pip-config` were passed.

pex/cli/commands/lock.py

@@ -1756,6 +1756,7 @@ def _sync(self):
                 pip_version=pip_configuration.version,
                 use_pip_config=pip_configuration.use_pip_config,
                 extra_pip_requirements=pip_configuration.extra_requirements,
+                keyring_provider=pip_configuration.keyring_provider,
                 result_type=InstallableType.INSTALLED_WHEEL_CHROOT,
             )
         )

pex/pip/tool.py

@@ -10,9 +10,10 @@
 import re
 import subprocess
 import sys
+import textwrap
 from collections import deque
 
-from pex import targets
+from pex import pex_warnings, targets
 from pex.atomic_directory import atomic_directory
 from pex.auth import PasswordEntry
 from pex.cache.dirs import PipPexDir
@@ -151,6 +152,7 @@ def create(
         password_entries=(),  # type: Iterable[PasswordEntry]
         use_pip_config=False,  # type: bool
         extra_pip_requirements=(),  # type: Tuple[Requirement, ...]
+        keyring_provider=None,  # type: Optional[str]
     ):
         # type: (...) -> PackageIndexConfiguration
         resolver_version = resolver_version or ResolverVersion.default(pip_version)
@@ -174,6 +176,7 @@ def create(
             use_pip_config=use_pip_config,
             extra_pip_requirements=extra_pip_requirements,
             password_entries=password_entries,
+            keyring_provider=keyring_provider,
         )
 
     def __init__(
@@ -186,6 +189,7 @@ def __init__(
         password_entries=(),  # type: Iterable[PasswordEntry]
         pip_version=None,  # type: Optional[PipVersionValue]
         extra_pip_requirements=(),  # type: Tuple[Requirement, ...]
+        keyring_provider=None,  # type: Optional[str]
     ):
         # type: (...) -> None
         self.resolver_version = resolver_version  # type: ResolverVersion.Value
@@ -196,6 +200,7 @@ def __init__(
         self.password_entries = password_entries  # type: Iterable[PasswordEntry]
         self.pip_version = pip_version  # type: Optional[PipVersionValue]
         self.extra_pip_requirements = extra_pip_requirements  # type: Tuple[Requirement, ...]
+        self.keyring_provider = keyring_provider  # type: Optional[str]
 
 
 if TYPE_CHECKING:
@@ -393,6 +398,35 @@ def _spawn_pip_isolated(
             # `~/.config/pip/pip.conf`.
             pip_args.append("--isolated")
 
+        # Configure a keychain provider if so configured and the version of Pip supports the option.
+        # Warn the user if Pex cannot pass the `--keyring-provider` option and suggest a solution.
+        if package_index_configuration and package_index_configuration.keyring_provider:
+            if self.version.version >= PipVersion.v23_1.version:
+                pip_args.append("--keyring-provider")
+                pip_args.append(package_index_configuration.keyring_provider)
+            else:
+                warn_msg = textwrap.dedent(
+                    """
+                    The --keyring-provider option is set to `{PROVIDER}`, but Pip v{THIS_VERSION} does not support the
+                    `--keyring-provider` option (which is only available in Pip v{VERSION_23_1} and later versions).
+                    Consequently, Pex is ignoring the --keyring-provider option for this particular Pip invocation.
+
+                    Note: If this Pex invocation fails, it may be because Pex is trying to use its vendored Pip v{VENDORED_VERSION}
+                    to bootstrap a newer Pip version which does support `--keyring-provider`, but you configured Pex/Pip
+                    to use a Python package index which is not available without additional authentication.
+
+                    In that case, you might wish to consider manually creating a `find-links` directory with that newer version
+                    of Pip, so that Pex will still be able to install the newer version of Pip from the `find-links` directory
+                    (which does not require authentication).
+                    """.format(
+                        PROVIDER=package_index_configuration.keyring_provider,
+                        THIS_VERSION=self.version.version,
+                        VERSION_23_1=PipVersion.v23_1,
+                        VENDORED_VERSION=PipVersion.VENDORED.version,
+                    )
+                )
+                pex_warnings.warn(warn_msg)
+
         if log:
             pip_args.append("--log")
             pip_args.append(log)

pex/resolve/configured_resolve.py

@@ -64,6 +64,7 @@ def resolve(
                     pip_version=lock.pip_version,
                     use_pip_config=pip_configuration.use_pip_config,
                     extra_pip_requirements=pip_configuration.extra_requirements,
+                    keyring_provider=pip_configuration.keyring_provider,
                     result_type=result_type,
                     dependency_configuration=dependency_configuration,
                 )
@@ -130,6 +131,7 @@ def resolve(
                 resolver=ConfiguredResolver(pip_configuration=resolver_configuration),
                 use_pip_config=resolver_configuration.use_pip_config,
                 extra_pip_requirements=resolver_configuration.extra_requirements,
+                keyring_provider=resolver_configuration.keyring_provider,
                 result_type=result_type,
                 dependency_configuration=dependency_configuration,
             )

pex/resolve/configured_resolver.py

@@ -77,6 +77,7 @@ def resolve_lock(
                 pip_version=pip_version or self.pip_configuration.version,
                 use_pip_config=self.pip_configuration.use_pip_config,
                 extra_pip_requirements=self.pip_configuration.extra_requirements,
+                keyring_provider=self.pip_configuration.keyring_provider,
                 result_type=result_type,
             )
         )
@@ -113,5 +114,6 @@ def resolve_requirements(
                 if extra_resolver_requirements is not None
                 else self.pip_configuration.extra_requirements
             ),
+            keyring_provider=self.pip_configuration.keyring_provider,
             result_type=result_type,
         )

pex/resolve/lock_downloader.py

@@ -86,6 +86,7 @@ def __init__(
         resolver=None,  # type: Optional[Resolver]
         use_pip_config=False,  # type: bool
         extra_pip_requirements=(),  # type: Tuple[Requirement, ...]
+        keyring_provider=None,  # type: Optional[str]
     ):
         super(VCSArtifactDownloadManager, self).__init__(
             pex_root=pex_root, file_lock_style=file_lock_style
@@ -108,6 +109,7 @@ def __init__(
         self._resolver = resolver
         self._use_pip_config = use_pip_config
         self._extra_pip_requirements = extra_pip_requirements
+        self._keyring_provider = keyring_provider
 
     def save(
         self,
@@ -134,6 +136,7 @@ def save(
             resolver=self._resolver,
             use_pip_config=self._use_pip_config,
             extra_pip_requirements=self._extra_pip_requirements,
+            keyring_provider=self._keyring_provider,
         )
         if len(downloaded_vcs.local_distributions) != 1:
             return Error(
@@ -217,6 +220,7 @@ def create(
         build_configuration=BuildConfiguration(),  # type: BuildConfiguration
         use_pip_config=False,  # type: bool
         extra_pip_requirements=(),  # type: Tuple[Requirement, ...]
+        keyring_provider=None,  # type: Optional[str]
     ):
         # type: (...) -> LockDownloader
 
@@ -245,6 +249,7 @@ def create(
                         ),
                         use_pip_config=use_pip_config,
                         extra_pip_requirements=extra_pip_requirements,
+                        keyring_provider=keyring_provider,
                     ),
                     max_parallel_jobs=max_parallel_jobs,
                 ),
@@ -266,6 +271,7 @@ def create(
                 resolver=resolver,
                 use_pip_config=use_pip_config,
                 extra_pip_requirements=extra_pip_requirements,
+                keyring_provider=keyring_provider,
             )
             for target in targets
         }

pex/resolve/lock_resolver.py

@@ -47,6 +47,7 @@ def resolve_from_lock(
     pip_version=None,  # type: Optional[PipVersionValue]
     use_pip_config=False,  # type: bool
     extra_pip_requirements=(),  # type: Tuple[Requirement, ...]
+    keyring_provider=None,  # type: Optional[str]
     result_type=InstallableType.INSTALLED_WHEEL_CHROOT,  # type: InstallableType.Value
     dependency_configuration=DependencyConfiguration(),  # type: DependencyConfiguration
 ):
@@ -88,6 +89,7 @@ def resolve_from_lock(
         build_configuration=build_configuration,
         use_pip_config=use_pip_config,
         extra_pip_requirements=extra_pip_requirements,
+        keyring_provider=keyring_provider,
     )
     with TRACER.timed(
         "Downloading {url_count} distributions to satisfy {requirement_count} requirements".format(
@@ -142,6 +144,7 @@ def resolve_from_lock(
                 password_entries=PasswordDatabase.from_netrc().append(password_entries).entries,
                 use_pip_config=use_pip_config,
                 extra_pip_requirements=extra_pip_requirements,
+                keyring_provider=keyring_provider,
             ),
             compile=compile,
             build_configuration=build_configuration,

pex/resolve/lockfile/create.py

@@ -381,6 +381,7 @@ def create(
         ),
         use_pip_config=pip_configuration.use_pip_config,
         extra_pip_requirements=pip_configuration.extra_requirements,
+        keyring_provider=pip_configuration.keyring_provider,
     )
 
     configured_resolver = ConfiguredResolver(pip_configuration=pip_configuration)
@@ -429,6 +430,7 @@ def create(
             resolver=configured_resolver,
             use_pip_config=pip_configuration.use_pip_config,
             extra_pip_requirements=pip_configuration.extra_requirements,
+            keyring_provider=pip_configuration.keyring_provider,
             dependency_configuration=dependency_configuration,
         )
     except resolvers.ResolveError as e:

pex/resolve/pre_resolved_resolver.py

@@ -102,6 +102,7 @@ def resolve_from_dists(
                 password_entries=pip_configuration.repos_configuration.password_entries,
                 use_pip_config=pip_configuration.use_pip_config,
                 extra_pip_requirements=pip_configuration.extra_requirements,
+                keyring_provider=pip_configuration.keyring_provider,
             )
             build_and_install = BuildAndInstallRequest(
                 build_requests=[

pex/resolve/resolver_configuration.py

@@ -203,6 +203,7 @@ class PipConfiguration(object):
     allow_version_fallback = attr.ib(default=True)  # type: bool
     use_pip_config = attr.ib(default=False)  # type: bool
     extra_requirements = attr.ib(default=())  # type Tuple[Requirement, ...]
+    keyring_provider = attr.ib(default=None)  # type: Optional[str]
 
 
 @attr.s(frozen=True)

pex/resolve/resolver_options.py

@@ -135,8 +135,25 @@ def register(
             "See: https://pip.pypa.io/en/stable/topics/authentication/#keyring-support"
         ),
     )
+
     register_use_pip_config(parser)
 
+    parser.add_argument(
+        "--keyring-provider",
+        metavar="PROVIDER",
+        dest="keyring_provider",
+        type=str,
+        default=None,
+        help=(
+            "Configure Pip to use the given keyring provider to obtain authentication for package indexes. "
+            "Please note that keyring support is only available in Pip v23.1 and later versions. "
+            "There is obviously a bootstrap issue here if your only available index is secured; "
+            "so you may need to use an additional --find-links repo or --index that is not "
+            "secured in order to bootstrap a version of Pip which supports keyring. "
+            "See: https://pip.pypa.io/en/stable/topics/authentication/#keyring-support"
+        ),
+    )
+
     register_repos_options(parser)
     register_network_options(parser)
 
@@ -681,6 +698,7 @@ def create_pip_configuration(
         allow_version_fallback=options.allow_pip_version_fallback,
         use_pip_config=get_use_pip_config_value(options),
         extra_requirements=tuple(options.extra_pip_requirements),
+        keyring_provider=options.keyring_provider,
     )
 
 

pex/resolver.py

@@ -1070,6 +1070,7 @@ def resolve(
     resolver=None,  # type: Optional[Resolver]
     use_pip_config=False,  # type: bool
     extra_pip_requirements=(),  # type: Tuple[Requirement, ...]
+    keyring_provider=None,  # type: Optional[str]
     result_type=InstallableType.INSTALLED_WHEEL_CHROOT,  # type: InstallableType.Value
     dependency_configuration=DependencyConfiguration(),  # type: DependencyConfiguration
 ):
@@ -1155,6 +1156,7 @@ def resolve(
         password_entries=password_entries,
         use_pip_config=use_pip_config,
         extra_pip_requirements=extra_pip_requirements,
+        keyring_provider=keyring_provider,
     )
 
     if not build_configuration.allow_wheels:
@@ -1323,6 +1325,7 @@ def download(
     resolver=None,  # type: Optional[Resolver]
     use_pip_config=False,  # type: bool
     extra_pip_requirements=(),  # type: Tuple[Requirement, ...]
+    keyring_provider=None,  # type: Optional[str]
     dependency_configuration=DependencyConfiguration(),  # type: DependencyConfiguration
 ):
     # type: (...) -> Downloaded
@@ -1369,6 +1372,7 @@ def download(
         password_entries=password_entries,
         use_pip_config=use_pip_config,
         extra_pip_requirements=extra_pip_requirements,
+        keyring_provider=keyring_provider,
     )
     build_requests, download_results = _download_internal(
         targets=targets,

tests/integration/test_keyring_support.py

@@ -210,6 +210,7 @@ def download_pip_requirements(
 
 @skip_if_required_keyring_version_not_supported
 @keyring_provider_pip_versions
+@pytest.mark.parametrize("use_keyring_provider_option", [False, True])
 def test_subprocess_provider(
     proxy,  # type: Proxy
     pip_version,  # type: PipVersionValue
@@ -218,6 +219,7 @@ def test_subprocess_provider(
     index_reverse_proxy_target,  # type: str
     devpi_clean_env,  # type: Mapping[str, Any]
     tmpdir,  # type: Any
+    use_keyring_provider_option,  # type: bool
 ):
     # type: (...) -> None
 
@@ -241,6 +243,15 @@ def test_subprocess_provider(
                 ),
             ).geturl()
         )
+
+        # If we are testing the `--keyring-provider`option, then do not put the option into the environment
+        # since it will be passed on the command-line.
+        new_path = os.pathsep.join((keyring_venv.path_element, os.environ.get("PATH", os.defpath)))
+        if use_keyring_provider_option:
+            env = make_env(PATH=new_path, **devpi_clean_env)
+        else:
+            env = make_env(PIP_KEYRING_PROVIDER="subprocess", PATH=new_path, **devpi_clean_env)
+
         run_pex_command(
             args=[
                 "--pex-root",
@@ -254,25 +265,22 @@ def test_subprocess_provider(
                 find_links,
                 "--pip-version",
                 str(pip_version),
-                "--use-pip-config",
+                "--keyring-provider=subprocess"
+                if use_keyring_provider_option
+                else "--use-pip-config",
                 "cowsay==5.0",
                 "-c",
                 "cowsay",
                 "--",
                 "Subprocess Auth!",
             ],
-            env=make_env(
-                PIP_KEYRING_PROVIDER="subprocess",
-                PATH=os.pathsep.join(
-                    (keyring_venv.path_element, os.environ.get("PATH", os.defpath))
-                ),
-                **devpi_clean_env
-            ),
+            env=env,
         ).assert_success(expected_output_re=r"^.*\| Subprocess Auth! \|.*$", re_flags=re.DOTALL)
 
 
 @skip_if_required_keyring_version_not_supported
 @keyring_provider_pip_versions
+@pytest.mark.parametrize("use_keyring_provider_option", [False, True])
 def test_import_provider(
     proxy,  # type: Proxy
     pip_version,  # type: PipVersionValue
@@ -281,6 +289,7 @@ def test_import_provider(
     index_reverse_proxy_target,  # type: str
     devpi_clean_env,  # type: Mapping[str, Any]
     tmpdir,  # type: Any
+    use_keyring_provider_option,  # type: bool
 ):
     # type: (...) -> None
 
@@ -306,6 +315,14 @@ def test_import_provider(
                 netloc="localhost:{port}".format(port=port),
             ).geturl()
         )
+
+        # If we are testing the `--keyring-provider`option, then do not put the option into the environment
+        # since it will be passed on the command-line.
+        if use_keyring_provider_option:
+            env = make_env(**devpi_clean_env)
+        else:
+            env = make_env(PIP_KEYRING_PROVIDER="import", **devpi_clean_env)
+
         run_pex_command(
             args=[
                 "--pex-root",
@@ -321,12 +338,12 @@ def test_import_provider(
                 str(keyring_venv.backend.project_name),
                 "--pip-version",
                 str(pip_version),
-                "--use-pip-config",
+                "--keyring-provider=import" if use_keyring_provider_option else "--use-pip-config",
                 "cowsay==5.0",
                 "-c",
                 "cowsay",
                 "--",
                 "Import Auth!",
             ],
-            env=make_env(PIP_KEYRING_PROVIDER="import", **devpi_clean_env),
+            env=env,
         ).assert_success(expected_output_re=r"^.*\| Import Auth! \|.*$", re_flags=re.DOTALL)