From 39e87780ca60332337b39ce3a8eed43e188ccb95 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Thu, 6 Feb 2025 17:14:17 -0800 Subject: [PATCH] Windows Build Fixes. (#2664) Re-jigger configuration for tox and git to support Windows dev / CI. As part of this, fix `pex.pyenv` to work with `pyenv-win`. More work towards #2658. CF: https://github.com/tox-dev/tox/issues/3136#issuecomment-2638029551 --- .gitattributes | 1 + .gitignore | 1 + compatible-tox-hack/pyproject.toml | 13 +++++++++++ pex/pyenv.py | 37 +++++++++++++++++++++++------- scripts/format.py | 3 ++- tox.ini | 6 +++-- 6 files changed, 50 insertions(+), 11 deletions(-) create mode 100644 .gitattributes create mode 100644 compatible-tox-hack/pyproject.toml diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..212566614 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto \ No newline at end of file diff --git a/.gitignore b/.gitignore index b50b7e33d..bf26a7624 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ __pycache__/ /.mypy_cache/ /.tox/ +/compatible-tox-hack/compatible_tox_hack.egg-info/ /dist/ /docs/_static_dynamic/ diff --git a/compatible-tox-hack/pyproject.toml b/compatible-tox-hack/pyproject.toml new file mode 100644 index 000000000..1f188b763 --- /dev/null +++ b/compatible-tox-hack/pyproject.toml @@ -0,0 +1,13 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "compatible-tox-hack" +version = "0.1.0" +dependencies = [ + "tox<4; platform_system != 'Windows'", + "tox; platform_system == 'Windows'", + "virtualenv<20.16; platform_system != 'Windows'", + "virtualenv; platform_system == 'Windows'", +] diff --git a/pex/pyenv.py b/pex/pyenv.py index 589abd0cf..3b5620b01 100644 --- a/pex/pyenv.py +++ b/pex/pyenv.py @@ -7,7 +7,8 @@ import re import subprocess -from pex.os import is_exe +from pex.os import WINDOWS, is_exe +from pex.sysconfig import SCRIPT_DIR, script_name from pex.tracer import TRACER from pex.typing import TYPE_CHECKING @@ -31,9 +32,20 @@ def find(cls): pyenv_root = os.environ.get("PYENV_ROOT", "") if not pyenv_root: for path_entry in os.environ.get("PATH", "").split(os.pathsep): - pyenv_exe = os.path.join(path_entry, "pyenv") + pyenv_exe = os.path.join(path_entry, "pyenv.bat" if WINDOWS else "pyenv") if is_exe(pyenv_exe): - process = subprocess.Popen(args=[pyenv_exe, "root"], stdout=subprocess.PIPE) + try: + process = subprocess.Popen( + args=[pyenv_exe, "root"], stdout=subprocess.PIPE + ) + except OSError as e: + TRACER.log( + "Skipping {pyenv_exe} executable. Not executable: {err}".format( + pyenv_exe=pyenv_exe, err=e + ), + V=6, + ) + continue stdout, _ = process.communicate() if process.returncode == 0: pyenv_root = str(stdout).strip() @@ -74,6 +86,10 @@ class Shim(object): [a-z]? )? )? + (?: + # Under pyenv-win. + \.bat + )? $ """, flags=re.VERBOSE, @@ -107,7 +123,7 @@ def select_version(self, search_dir=None): with TRACER.timed("Calculating active version for {}...".format(self), V=6): active_versions = self.pyenv.active_versions(search_dir=search_dir) if active_versions: - binary_name = os.path.basename(self.path) + binary_name, _ = os.path.splitext(os.path.basename(self.path)) if self.name == "python" and not self.major and not self.minor: for pyenv_version in active_versions: if pyenv_version[0] in self._PYENV_CPYTHON_VERSION_LEADING_CHARS: @@ -204,9 +220,14 @@ def python( # N.B.: Pyenv creates a 'python' symlink for both the CPython and PyPy versions it installs; # so counting on 'python' is OK here. We do resolve the symlink though to return a canonical # direct path to the python binary. - binary_name = binary_name or "python" + binary_name = script_name(binary_name or "python") - python = os.path.realpath( - os.path.join(self.root, "versions", pyenv_version, "bin", binary_name) - ) + if WINDOWS: + python = os.path.realpath( + os.path.join(self.root, "versions", pyenv_version, binary_name) + ) + else: + python = os.path.realpath( + os.path.join(self.root, "versions", pyenv_version, SCRIPT_DIR, binary_name) + ) return python if is_exe(python) else None diff --git a/scripts/format.py b/scripts/format.py index d64f3305d..44206c084 100755 --- a/scripts/format.py +++ b/scripts/format.py @@ -15,7 +15,7 @@ def filtered_stream(exclude: str, dest: TextIO) -> Iterator[int]: read_fd, write_fd = os.pipe() def filter_stream() -> None: - with os.fdopen(read_fd, "r") as fh: + with os.fdopen(read_fd, "r", encoding="utf-8") as fh: for line in fh: if not re.search(exclude, line): dest.write(line) @@ -50,6 +50,7 @@ def run_black(*args: str) -> None: ], stdout=out_fd, stderr=subprocess.STDOUT, + encoding="utf-8", check=True, ) diff --git a/tox.ini b/tox.ini index 4428b5b3e..1deff7852 100644 --- a/tox.ini +++ b/tox.ini @@ -3,8 +3,7 @@ skip_missing_interpreters = true minversion = 3.25.1 requires = # Ensure tox and virtualenv compatible back through Python 2.7. - tox<4 - virtualenv<20.16 + compatible-tox-hack @ file://{toxinidir}/compatible-tox-hack [testenv] # N.B.: We need modern setuptools downloaded out of band by virtualenv to work with Python>=3.12. @@ -192,6 +191,8 @@ deps = setuptools==50.3.2 wheel==0.35.1 {[testenv:format-run]deps} +setenv = + PYTHONUTF8=1 commands = python -m pex.vendor {posargs} {[testenv:format-run]commands} @@ -202,6 +203,7 @@ skip_install = true deps = tox httpx==0.23.0 +setenv = {[testenv:vendor]setenv} commands = tox -e vendor -- --no-update python scripts/embed-virtualenv.py