Delay import of ssl in pex.fetcher. (#2417)

This acts as an affordance for users of the gevent monkey patch system,
avoiding warnings about potential imports of `ssl` pre-monkey-patching.

Fixes #2415

CHANGES.md

@@ -1,5 +1,14 @@
 # Release Notes
 
+## 2.3.2
+
+This release fixes a regression for users of gevent monkey patching. The
+fix in #2356 released in Pex 2.1.163 lead to these users receiving
+spurious warnings from the gevent monkey patch system about ssl being
+patched too late.
+
+* Delay import of ssl in `pex.fetcher`. (#2417)
+
 ## 2.3.1
 
 This release fixes Pex to respect lock file interpreter constraints and

pex/fetcher.py

@@ -5,12 +5,10 @@
 
 import contextlib
 import os
-import ssl
 import sys
 import threading
 import time
 from contextlib import closing, contextmanager
-from ssl import SSLContext
 
 from pex import asserts
 from pex.auth import PasswordDatabase, PasswordEntry
@@ -31,6 +29,7 @@
 from pex.version import __version__
 
 if TYPE_CHECKING:
+    from ssl import SSLContext
     from typing import BinaryIO, Dict, Iterable, Iterator, Mapping, Optional, Text
 
     import attr  # vendor:skip
@@ -112,6 +111,13 @@ def create_ssl_context(self):
             ),
         )
         with guard_stdout():
+            # We import ssl lazily as an affordance to PEXes that use gevent SSL monkeypatching,
+            # which requires (and checks) that the `ssl` module is not imported priory to the
+            # `from gevent import monkey; monkey.patch_all()` call.
+            #
+            # See: https://github.com/pex-tool/pex/issues/2415
+            import ssl
+
             ssl_context = ssl.create_default_context(cafile=self.cert)
             if self.client_cert:
                 ssl_context.load_cert_chain(self.client_cert)

pex/version.py

@@ -1,4 +1,4 @@
 # Copyright 2015 Pex project contributors.
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-__version__ = "2.3.1"
+__version__ = "2.3.2"

tests/integration/test_issue_2183.py

@@ -57,7 +57,12 @@ def run():
     lambda_zip = os.path.join(str(tmpdir), "lambda.zip")
     run_pex_command(
         args=[
-            "lambdex",
+            # The Lambdex 0.2.0 final release changed from a top-level pex.third_party import to a
+            # lazy one which foils scrubbing under Python<3.8. Instead of releasing a Lambdex 0.2.1
+            # with a fix for this eagerly, we just pin this test dep low to avoid the issue and see
+            # if any bug report ever comes in, which seems unlikely since Python 3.7 is EOL and
+            # Lambdex is as well.
+            "lambdex<0.2.0",
             "-c",
             "lambdex",
             "--",

tests/integration/test_issue_2415.py

@@ -0,0 +1,145 @@
+# Copyright 2024 Pex project contributors.
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+import atexit
+import os.path
+import re
+import subprocess
+import threading
+from textwrap import dedent
+from threading import Event
+from typing import Optional
+
+import pytest
+
+from pex.common import safe_open
+from pex.fetcher import URLFetcher
+from pex.typing import TYPE_CHECKING
+from testing import IS_PYPY, PY_VER, data, run_pex_command
+
+if TYPE_CHECKING:
+    from typing import Any
+
+    import attr  # vendor:skip
+else:
+    from pex.third_party import attr
+
+
+@pytest.mark.skipif(
+    IS_PYPY or PY_VER < (3, 8) or PY_VER >= (3, 13),
+    reason=(
+        "The lock file for this test only supports CPythons >=3.8,<3.13 which were the officially "
+        "supported CPythons at the time issue 2415 was reported."
+    ),
+)
+def test_gevent_monkeypatch(tmpdir):
+    # type: (Any) -> None
+
+    with safe_open(os.path.join(str(tmpdir), "app.py"), "w") as app_fp:
+        app_fp.write(
+            dedent(
+                """\
+                from gevent import monkey
+                monkey.patch_all()
+
+                from flask import Flask
+
+
+                app = Flask(__name__)
+
+
+                @app.route("/")
+                def hello_world():
+                    return "Hello, World!"
+                """
+            )
+        )
+
+    pex_root = os.path.join(str(tmpdir), "pex_root")
+    pex = os.path.join(str(tmpdir), "pex")
+
+    # N.B.: Created with the following, where gevent 1.3.4 was picked as a lower bound since it
+    # 1st introduced the `from gevent import monkey; monkey.patch_all()` ssl check warning that is
+    # the subject of issue 2415:
+    #
+    # pex3 lock create \
+    #   --resolver-version pip-2020-resolver \
+    #   --pip-version latest \
+    #   --style universal \
+    #   --interpreter-constraint ">=3.8,<3.13" \
+    #   --indent 2 \
+    #   flask \
+    #   "gevent>=1.3.4" \
+    #   gunicorn
+    lock = data.path("locks", "issue-2415.lock.json")
+
+    run_pex_command(
+        args=[
+            "--pex-root",
+            pex_root,
+            "--runtime-pex-root",
+            pex_root,
+            "--lock",
+            lock,
+            "-M",
+            "app",
+            "-c",
+            "gunicorn",
+            "--inject-args",
+            "app:app",
+            "-o",
+            pex,
+        ],
+        cwd=str(tmpdir),
+    ).assert_success()
+
+    log = os.path.join(str(tmpdir), "log")
+    os.mkfifo(log)
+
+    @attr.s
+    class LogScanner(object):
+        port_seen = attr.ib(factory=Event, init=False)  # type: Event
+        _port = attr.ib(default=None)  # type: Optional[int]
+
+        def scan_log(self):
+            # type: () -> None
+
+            with open(log) as log_fp:
+                for line in log_fp:
+                    if self._port is None:
+                        match = re.search(r"Listening at: http://127.0.0.1:(?P<port>\d{1,5})", line)
+                        if match:
+                            self._port = int(match.group("port"))
+                            self.port_seen.set()
+
+        @property
+        def port(self):
+            # type: () -> int
+            self.port_seen.wait()
+            assert self._port is not None
+            return self._port
+
+    log_scanner = LogScanner()
+    log_scan_thread = threading.Thread(target=log_scanner.scan_log)
+    log_scan_thread.daemon = True
+    log_scan_thread.start()
+
+    with open(os.path.join(str(tmpdir), "stderr"), "wb+") as stderr_fp:
+        gunicorn = subprocess.Popen(
+            args=[pex, "--bind", "127.0.0.1:0", "--log-file", log], stderr=stderr_fp
+        )
+        atexit.register(gunicorn.kill)
+
+        with URLFetcher().get_body_stream(
+            "http://127.0.0.1:{port}".format(port=log_scanner.port)
+        ) as http_fp:
+            assert b"Hello, World!" == http_fp.read().strip()
+
+        gunicorn.kill()
+        log_scan_thread.join()
+        stderr_fp.flush()
+        stderr_fp.seek(0)
+        stderr = stderr_fp.read()
+        assert b"MonkeyPatchWarning: Monkey-patching ssl after ssl " not in stderr, stderr.decode(
+            "utf-8"
+        )

tests/integration/test_shebang_length_limit.py

@@ -6,16 +6,16 @@
 import errno
 import json
 import os
+import shutil
 import subprocess
 from textwrap import dedent
 
 import colors
 import pytest
 
-from pex.common import chmod_plus_x, touch
+from pex.common import chmod_plus_x, safe_open, touch
 from pex.typing import TYPE_CHECKING
-from pex.version import __version__
-from testing import IS_PYPY, run_pex_command
+from testing import IS_PYPY, make_project, run_pex_command
 from testing.cli import run_pex3
 
 if TYPE_CHECKING:
@@ -260,14 +260,19 @@ def test_shebang_length_limit_buildtime_resolve(
 
 def test_shebang_length_limit_buildtime_lock_local_project(
     tmpdir,  # type: Any
-    pex_project_dir,  # type: str
     too_deep_pex_root,  # type: str
 ):
     # type: (...) -> None
 
+    local_project_dir = os.path.join(str(tmpdir), "foo")
+    with make_project(name="foo") as td:
+        with safe_open(os.path.join(td, "foo", "version.py"), "w") as fp:
+            fp.write("__version__ = 42")
+        shutil.move(td, local_project_dir)
+
     lock = os.path.realpath(os.path.join(str(tmpdir), "lock.json"))
     # N.B.: This runs the vendored PEP 517 / 518 sdist build tool in a dogfood venv to create an
-    # sdist for the local Pex project that can be consistently hashed.
+    # sdist for the local foo project that can be consistently hashed.
     run_pex3(
         "lock",
         "create",
@@ -278,7 +283,7 @@ def test_shebang_length_limit_buildtime_lock_local_project(
         "--indent",
         "2",
         "ansicolors==1.1.8",
-        pex_project_dir,
+        local_project_dir,
     ).assert_success()
 
     pex = os.path.realpath(os.path.join(str(tmpdir), "pex"))
@@ -296,7 +301,7 @@ def test_shebang_length_limit_buildtime_lock_local_project(
     ).assert_success()
 
     assert (
-        colors.yellow(__version__)
+        colors.yellow("42")
         == subprocess.check_output(
             args=[
                 pex,
@@ -305,7 +310,7 @@ def test_shebang_length_limit_buildtime_lock_local_project(
                     """\
                     import colors
 
-                    from pex.version import __version__
+                    from foo.version import __version__
 
 
                     print(colors.yellow(__version__))

tox.ini

@@ -18,10 +18,19 @@ download = true
 # N.B.: We configure tox to disable its build sdist, then install sdist in venv scheme for all envs
 # with `skip_install = false` (the default). As such, we use a custom `install_command` for all
 # envs that need Pex installed. The install command is tox's default, with the one addition of
-# `{toxinidir}`, which adds `.` to the requirement list handed to Pip to install.
+# `{toxinidir}`, which adds `-e .` to the requirement list handed to Pip to install.
 install_command =
-    docs,check,typecheck,py{py27,py35,py36,py37,py38,py39,py310,27,35,36,37,38,39,310,311,312,313}: \
-        python -m pip install {opts} {toxinidir} {packages}
+    # N.B.: The latest Pip supported by Python 2.7 and 3.5 can't install pyproject.toml projects in
+    # editable mode.
+    # TODO(John Sirois): Consider moving back to the setuptools.build_meta backend and using a
+    # minimal setup.py since hatchling is buying us less and less and getting in the way more and
+    # more.
+    py{py27,py35,27,35}: python -m pip install {opts} {toxinidir} {packages}
+
+    docs,check,typecheck,py{py36,py37,py38,py39,py310,36,37,38,39,310,311,312,313}: \
+        python -m pip install {opts} -e {toxinidir} {packages}
+
+
 commands =
     !integration: python testing/bin/run_tests.py {posargs:-vvs}
     integration: python testing/bin/run_tests.py --it {posargs:-vvs}
@@ -162,7 +171,7 @@ basepython = python3.8
 skip_install = true
 deps =
     ansicolors==1.1.8
-    pip==20.2.4
+    pip==24.0
     redbaron==0.9.2
     setuptools==50.3.2
     wheel==0.35.1