Windows Build Fixes.

Re-jigger tox and git to support Windows development / CI. As part of
this, fix `pex.pyenv` to work with `pyenv-win`.

More work towards pex-tool#2658.

.gitattributes

@@ -0,0 +1 @@
+* text=auto

.gitignore

@@ -6,6 +6,7 @@ __pycache__/
 
 /.mypy_cache/
 /.tox/
+/compatible-tox-hack/compatible_tox_hack.egg-info/
 /dist/
 /docs/_static_dynamic/
 

compatible-tox-hack/pyproject.toml

@@ -0,0 +1,13 @@
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "compatible-tox-hack"
+version = "0.1.0"
+dependencies = [
+    "tox<4; platform_system != 'Windows'",
+    "tox; platform_system == 'Windows'",
+    "virtualenv<20.16; platform_system != 'Windows'",
+    "virtualenv; platform_system == 'Windows'",
+]

pex/pyenv.py

@@ -7,7 +7,8 @@
 import re
 import subprocess
 
-from pex.os import is_exe
+from pex.os import WINDOWS, is_exe
+from pex.sysconfig import SCRIPT_DIR, script_name
 from pex.tracer import TRACER
 from pex.typing import TYPE_CHECKING
 
@@ -31,9 +32,20 @@ def find(cls):
             pyenv_root = os.environ.get("PYENV_ROOT", "")
             if not pyenv_root:
                 for path_entry in os.environ.get("PATH", "").split(os.pathsep):
-                    pyenv_exe = os.path.join(path_entry, "pyenv")
+                    pyenv_exe = os.path.join(path_entry, "pyenv.bat" if WINDOWS else "pyenv")
                     if is_exe(pyenv_exe):
-                        process = subprocess.Popen(args=[pyenv_exe, "root"], stdout=subprocess.PIPE)
+                        try:
+                            process = subprocess.Popen(
+                                args=[pyenv_exe, "root"], stdout=subprocess.PIPE
+                            )
+                        except OSError as e:
+                            TRACER.log(
+                                "Skipping {pyenv_exe} executable. Not executable: {err}".format(
+                                    pyenv_exe=pyenv_exe, err=e
+                                ),
+                                V=6,
+                            )
+                            continue
                         stdout, _ = process.communicate()
                         if process.returncode == 0:
                             pyenv_root = str(stdout).strip()
@@ -74,6 +86,10 @@ class Shim(object):
                     [a-z]?
                 )?
             )?
+            (?:
+                # Under pyenv-win.
+                \.bat
+            )?
             $
             """,
             flags=re.VERBOSE,
@@ -107,7 +123,7 @@ def select_version(self, search_dir=None):
             with TRACER.timed("Calculating active version for {}...".format(self), V=6):
                 active_versions = self.pyenv.active_versions(search_dir=search_dir)
                 if active_versions:
-                    binary_name = os.path.basename(self.path)
+                    binary_name, _ = os.path.splitext(os.path.basename(self.path))
                     if self.name == "python" and not self.major and not self.minor:
                         for pyenv_version in active_versions:
                             if pyenv_version[0] in self._PYENV_CPYTHON_VERSION_LEADING_CHARS:
@@ -204,9 +220,14 @@ def python(
         # N.B.: Pyenv creates a 'python' symlink for both the CPython and PyPy versions it installs;
         # so counting on 'python' is OK here. We do resolve the symlink though to return a canonical
         # direct path to the python binary.
-        binary_name = binary_name or "python"
+        binary_name = script_name(binary_name or "python")
 
-        python = os.path.realpath(
-            os.path.join(self.root, "versions", pyenv_version, "bin", binary_name)
-        )
+        if WINDOWS:
+            python = os.path.realpath(
+                os.path.join(self.root, "versions", pyenv_version, binary_name)
+            )
+        else:
+            python = os.path.realpath(
+                os.path.join(self.root, "versions", pyenv_version, SCRIPT_DIR, binary_name)
+            )
         return python if is_exe(python) else None

scripts/format.py

@@ -15,7 +15,7 @@ def filtered_stream(exclude: str, dest: TextIO) -> Iterator[int]:
     read_fd, write_fd = os.pipe()
 
     def filter_stream() -> None:
-        with os.fdopen(read_fd, "r") as fh:
+        with os.fdopen(read_fd, "r", encoding="utf-8") as fh:
             for line in fh:
                 if not re.search(exclude, line):
                     dest.write(line)
@@ -50,6 +50,7 @@ def run_black(*args: str) -> None:
             ],
             stdout=out_fd,
             stderr=subprocess.STDOUT,
+            encoding="utf-8",
             check=True,
         )
 

tox.ini

@@ -3,8 +3,7 @@ skip_missing_interpreters = true
 minversion = 3.25.1
 requires =
     # Ensure tox and virtualenv compatible back through Python 2.7.
-    tox<4
-    virtualenv<20.16
+    compatible-tox-hack @ file://{toxinidir}/compatible-tox-hack
 
 [testenv]
 # N.B.: We need modern setuptools downloaded out of band by virtualenv to work with Python>=3.12.
@@ -192,6 +191,8 @@ deps =
     setuptools==50.3.2
     wheel==0.35.1
     {[testenv:format-run]deps}
+setenv =
+    PYTHONUTF8=1
 commands =
     python -m pex.vendor {posargs}
     {[testenv:format-run]commands}
@@ -202,6 +203,7 @@ skip_install = true
 deps =
     tox
     httpx==0.23.0
+setenv = {[testenv:vendor]setenv}
 commands =
     tox -e vendor -- --no-update
     python scripts/embed-virtualenv.py