diff --git a/pex/atomic_directory.py b/pex/atomic_directory.py index cd7ba5459..ab5c04069 100644 --- a/pex/atomic_directory.py +++ b/pex/atomic_directory.py @@ -4,7 +4,6 @@ from __future__ import absolute_import import errno -import fcntl import hashlib import os import threading @@ -13,8 +12,9 @@ from pex import pex_warnings from pex.common import safe_mkdir, safe_rmtree -from pex.enum import Enum -from pex.typing import TYPE_CHECKING, cast +from pex.fs import lock, safe_rename +from pex.fs.lock import FileLockStyle +from pex.typing import TYPE_CHECKING if TYPE_CHECKING: from typing import Callable, Dict, Iterator, Optional @@ -161,7 +161,7 @@ def finalize(self, source=None): # # We have satisfied the single filesystem constraint by arranging the `work_dir` to be a # sibling of the `target_dir`. - os.rename(source, self._target_dir) + safe_rename(source, self._target_dir) except OSError as e: if e.errno not in (errno.EEXIST, errno.ENOTEMPTY): raise e @@ -173,17 +173,6 @@ def cleanup(self): safe_rmtree(self._work_dir) -class FileLockStyle(Enum["FileLockStyle.Value"]): - class Value(Enum.Value): - pass - - BSD = Value("bsd") - POSIX = Value("posix") - - -FileLockStyle.seal() - - def _is_bsd_lock(lock_style=None): # type: (Optional[FileLockStyle.Value]) -> bool @@ -209,29 +198,15 @@ class _FileLock(object): def acquire(self): # type: () -> Callable[[], None] self._in_process_lock.acquire() - - # N.B.: We don't actually write anything to the lock file but the fcntl file locking - # operations only work on files opened for at least write. safe_mkdir(os.path.dirname(self._path)) - lock_fd = os.open(self._path, os.O_CREAT | os.O_WRONLY) - - lock_api = cast( - "Callable[[int, int], None]", - fcntl.flock if _is_bsd_lock(self._style) else fcntl.lockf, - ) - - # N.B.: Since lockf and flock operate on an open file descriptor and these are - # guaranteed to be closed by the operating system when the owning process exits, - # this lock is immune to staleness. - lock_api(lock_fd, fcntl.LOCK_EX) # A blocking write lock. + file_lock = lock.acquire(self._path, FileLockStyle.BSD if _is_bsd_lock(self._style) else FileLockStyle.POSIX) def release(): # type: () -> None try: - lock_api(lock_fd, fcntl.LOCK_UN) + file_lock.release() finally: - os.close(lock_fd) - self._in_process_lock.release() + self._in_process_lock.release() return release diff --git a/pex/cache/access.py b/pex/cache/access.py index 1fa1c0275..1e06ea31b 100644 --- a/pex/cache/access.py +++ b/pex/cache/access.py @@ -3,12 +3,14 @@ from __future__ import absolute_import, print_function -import fcntl import itertools import os from contextlib import contextmanager from pex.common import safe_mkdir, touch +from pex.fs import lock +from pex.fs.lock import FileLockStyle +from pex.os import WINDOWS from pex.typing import TYPE_CHECKING from pex.variables import ENV @@ -64,21 +66,14 @@ def _lock(exclusive): existing_exclusive, lock_fd, existing_lock_file = _LOCK if existing_exclusive == exclusive: return existing_lock_file + elif WINDOWS: + # Windows shared locks are not re-entrant; so this is the best we can do. + lock.release(lock_fd) lock_file = os.path.join(ENV.PEX_ROOT, "access.lck") - if lock_fd is None: - # N.B.: We don't actually write anything to the lock file but the fcntl file locking - # operations only work on files opened for at least write. - safe_mkdir(os.path.dirname(lock_file)) - lock_fd = os.open(lock_file, os.O_CREAT | os.O_WRONLY) - - # N.B.: Since flock operates on an open file descriptor and these are - # guaranteed to be closed by the operating system when the owning process exits, - # this lock is immune to staleness. - fcntl.flock(lock_fd, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH) - - _LOCK = exclusive, lock_fd, lock_file + file_lock = lock.acquire(lock_file, exclusive=exclusive, style=FileLockStyle.BSD, fd=lock_fd) + _LOCK = exclusive, file_lock.fd, lock_file return lock_file diff --git a/pex/cache/dirs.py b/pex/cache/dirs.py index d10753544..290ba5de3 100644 --- a/pex/cache/dirs.py +++ b/pex/cache/dirs.py @@ -336,6 +336,9 @@ def iter_all(cls, pex_root=ENV): for venv_short_dir_symlink in glob.glob( CacheDir.VENVS.path("s", "*", cls.SHORT_SYMLINK_NAME, pex_root=pex_root) ): + # TODO(John Sirois): XXX: Explain or limit this Windows hack. + # See: https://github.com/pex-tool/pex/issues/2660#issuecomment-2635441311 + venv_short_dir_symlink = os.path.realpath(venv_short_dir_symlink) if not os.path.isdir(venv_short_dir_symlink): continue diff --git a/pex/cache/prunable.py b/pex/cache/prunable.py index ad63195b1..b2571dd87 100644 --- a/pex/cache/prunable.py +++ b/pex/cache/prunable.py @@ -4,6 +4,7 @@ from __future__ import absolute_import import os.path +import sys from collections import OrderedDict from datetime import datetime @@ -54,7 +55,13 @@ def scan(cls, pex_dirs_by_hash): # extra requirements installed does not affect cache management. pip_caches_to_prune = OrderedDict() # type: OrderedDict[PipVersionValue, Pip] for pip in iter_all_pips(record_access=False): - pex_dir, prunable = pex_dirs_by_hash[pip.pex_hash] + try: + pex_dir, prunable = pex_dirs_by_hash[pip.pex_hash] + except KeyError: + print("pex_dirs_by_hash:", file=sys.stderr) + for hash, pex_dirs in pex_dirs_by_hash.items(): + print(f"{hash=} {pex_dirs}", file=sys.stderr) + raise if prunable: pips_to_prune[pip] = False else: diff --git a/pex/cli/commands/lock.py b/pex/cli/commands/lock.py index 2b240e014..4b97cb8af 100644 --- a/pex/cli/commands/lock.py +++ b/pex/cli/commands/lock.py @@ -33,6 +33,7 @@ from pex.executables import is_exe from pex.interpreter import PythonInterpreter from pex.orderedset import OrderedSet +from pex.os import safe_execv from pex.pep_376 import InstalledWheel, Record from pex.pep_427 import InstallableType from pex.pep_440 import Version @@ -344,7 +345,7 @@ def sync( if self.command: try: - os.execv(self.command[0], self.command) + safe_execv(self.command) except OSError as e: return Error("Failed to execute {exe}: {err}".format(exe=self.command[0], err=e)) diff --git a/pex/common.py b/pex/common.py index 0c239ecfa..b2e599a37 100644 --- a/pex/common.py +++ b/pex/common.py @@ -24,6 +24,7 @@ from pex.enum import Enum from pex.executables import chmod_plus_x +from pex.fs import safe_rename from pex.typing import TYPE_CHECKING, cast if TYPE_CHECKING: @@ -119,7 +120,7 @@ def do_copy(): # type: () -> None temp_dest = dest + uuid4().hex shutil.copy(source, temp_dest) - os.rename(temp_dest, dest) + safe_rename(temp_dest, dest) # If the platform supports hard-linking, use that and fall back to copying. # Windows does not support hard-linking. diff --git a/pex/compatibility.py b/pex/compatibility.py index 7d375abe5..0ceab59ce 100644 --- a/pex/compatibility.py +++ b/pex/compatibility.py @@ -178,8 +178,6 @@ def cpu_count(): from Queue import Queue as Queue -WINDOWS = os.name == "nt" - # Universal newlines is the default in Python 3. MODE_READ_UNIVERSAL_NEWLINES = "rU" if PY2 else "r" diff --git a/pex/dist_metadata.py b/pex/dist_metadata.py index e1dc2ea76..8f4242f61 100644 --- a/pex/dist_metadata.py +++ b/pex/dist_metadata.py @@ -376,6 +376,10 @@ def iter_metadata_files( ): # type: (...) -> Iterator[MetadataFiles] + # TODO(John Sirois): XXX: Explain or limit this Windows hack. + # See: https://github.com/pex-tool/pex/issues/2660#issuecomment-2635441311 + location = os.path.realpath(location) + files = [] for metadata_type in restrict_types_to or MetadataType.values(): key = MetadataKey(metadata_type=metadata_type, location=location) diff --git a/pex/finders.py b/pex/finders.py index fb80c5227..767d2fa60 100644 --- a/pex/finders.py +++ b/pex/finders.py @@ -17,6 +17,7 @@ from pex.executables import is_python_script from pex.pep_376 import InstalledWheel from pex.pep_503 import ProjectName +from pex.sysconfig import SCRIPT_DIR, script_name from pex.typing import TYPE_CHECKING, cast from pex.wheel import Wheel @@ -38,14 +39,14 @@ def find( ): # type: (...) -> Optional[DistributionScript] if dist.type is DistributionType.WHEEL: - script_path = Wheel.load(dist.location).data_path("scripts", name) + script_path = Wheel.load(dist.location).data_path("scripts", script_name(name)) with open_zip(dist.location) as zfp: try: zfp.getinfo(script_path) except KeyError: return None elif dist.type is DistributionType.INSTALLED: - script_path = InstalledWheel.load(dist.location).stashed_path("bin", name) + script_path = InstalledWheel.load(dist.location).stashed_path(SCRIPT_DIR, script_name(name)) if not os.path.isfile(script_path): return None else: diff --git a/pex/fs/__init__.py b/pex/fs/__init__.py new file mode 100644 index 000000000..ac0a67418 --- /dev/null +++ b/pex/fs/__init__.py @@ -0,0 +1,126 @@ +# Copyright 2025 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import os +import sys +from typing import Callable + +from pex.os import WINDOWS +from pex.typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Text, Optional + +if WINDOWS and not hasattr(os, "replace"): + _MOVEFILE_REPLACE_EXISTING = 0x1 + + _MF = None + + def safe_rename( + src, # type: Text + dst, # type: Text + ): + # type: (...) -> None + + import ctypes + from ctypes.wintypes import BOOL, DWORD, LPCWSTR + + global _MF + if _MF is None: + mf = ctypes.windll.kernel32.MoveFileExW + mf.argtypes = ( + # lpExistingFileName + LPCWSTR, + # lpNewFileName + LPCWSTR, + # dwFlags + DWORD, + ) + mf.restype = BOOL + _MF = mf + + # See: https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-movefileexw + if not _MF(src, dst, _MOVEFILE_REPLACE_EXISTING): + raise ctypes.WinError() + +else: + safe_rename = getattr(os, "replace", os.rename) + + +# N.B.: Python 3.7 has os.symlink on Windows, but the implementation does not pass the +# _SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE flag. +if WINDOWS and (not hasattr(os, "symlink") or sys.version_info[:2] < (3, 8)): + _SYMBOLIC_LINK_FLAG_FILE = 0x0 + _SYMBOLIC_LINK_FLAG_DIRECTORY = 0x1 + _SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE = 0x2 + + _CSL = None + + def safe_symlink( + src, # type: Text + dst, # type: Text + ): + # type: (...) -> None + + import ctypes + from ctypes.wintypes import BOOLEAN, DWORD, LPCWSTR + + global _CSL + if _CSL is None: + csl = ctypes.windll.kernel32.CreateSymbolicLinkW + csl.argtypes = ( + # lpSymlinkFileName + LPCWSTR, + # lpTargetFileName + LPCWSTR, + # dwFlags + DWORD, + ) + csl.restype = BOOLEAN + _CSL = csl + + # See: https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createsymboliclinkw + flags = _SYMBOLIC_LINK_FLAG_DIRECTORY if os.path.isdir(src) else _SYMBOLIC_LINK_FLAG_FILE + flags |= _SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE + if not _CSL(dst, src, flags): + raise ctypes.WinError() + +else: + safe_realpath = os.path.realpath + safe_symlink = getattr(os, "symlink") + + +if WINDOWS and not hasattr(os, "link"): + _CHL = None + + def safe_link( + src, # type: Text + dst, # type: Text + ): + # type: (...) -> None + + import ctypes + from ctypes.wintypes import BOOL, LPCWSTR, LPVOID + + global _CHL + if _CHL is None: + # See: https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createhardlinkw + chl = ctypes.windll.kernel32.CreateHardLinkW + chl.argtypes = ( + # lpFileName + LPCWSTR, + # lpExistingFileName + LPCWSTR, + # lpSecurityAttributes (Reserved; must be NULL) + LPVOID, + ) + chl.restype = BOOL + _CHL = chl + + if not _CHL(dst, src, None): + raise ctypes.WinError() + +else: + safe_link = getattr(os, "link") diff --git a/pex/fs/_posix.py b/pex/fs/_posix.py new file mode 100644 index 000000000..59480dcc2 --- /dev/null +++ b/pex/fs/_posix.py @@ -0,0 +1,44 @@ +# Copyright 2025 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import fcntl + +from pex.fs.lock import FileLock, FileLockStyle +from pex.typing import TYPE_CHECKING, cast + +if TYPE_CHECKING: + from typing import Callable + + +class PosixFileLock(FileLock): + @staticmethod + def _lock_api(style): + # type: (FileLockStyle.Value) -> Callable[[int, int], None] + + return cast( + "Callable[[int, int], None]", fcntl.flock if style is FileLockStyle.BSD else fcntl.lockf + ) + + @classmethod + def acquire( + cls, + fd, # type: int + exclusive, # type: bool + style, # type: FileLockStyle.Value + ): + # type: (...) -> PosixFileLock + + cls._lock_api(style)(fd, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH) + return cls(locked_fd=fd, unlock=lambda: cls.release_lock(fd, style=style)) + + @classmethod + def release_lock( + cls, + fd, # type: int + style, # type: FileLockStyle.Value + ): + # type: (...) -> None + + cls._lock_api(style)(fd, fcntl.LOCK_UN) diff --git a/pex/fs/_windows.py b/pex/fs/_windows.py new file mode 100644 index 000000000..a302275c6 --- /dev/null +++ b/pex/fs/_windows.py @@ -0,0 +1,108 @@ +# Copyright 2025 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import ctypes +import msvcrt +from ctypes.wintypes import BOOL, DWORD, HANDLE, PULONG, LPVOID, ULONG + +from pex.fs.lock import FileLock + + +class Offset(ctypes.Structure): + _fields_ = [ + ("Offset", DWORD), + ("OffsetHigh", DWORD), + ] + + +class OffsetUnion(ctypes.Union): + _fields_ = [ + ("Offset", Offset), + ("Pointer", LPVOID) + ] + + +# See: https://learn.microsoft.com/en-us/windows/win32/api/minwinbase/ns-minwinbase-overlapped +class Overlapped(ctypes.Structure): + @classmethod + def ignored(cls): + # type: () -> Overlapped + return cls(PULONG(ULONG(0)), PULONG(ULONG(0)), OffsetUnion(Offset(0, 0)), HANDLE(0)) + + _fields_ = [ + ("Internal", PULONG), + ("InternalHigh", PULONG), + ("OffsetUnion", OffsetUnion), + ("hEvent", HANDLE) + ] + + +# See: https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-lockfileex +_LockFileEx = ctypes.windll.kernel32.LockFileEx +_LockFileEx.argtypes = ( + HANDLE, # hFile + DWORD, # dwFlags + DWORD, # dwReserved + DWORD, # nNumberOfBytesToLockLow + DWORD, # nNumberOfBytesToLockHigh + Overlapped, # lpOverlapped +) +_LockFileEx.restype = BOOL +_LOCKFILE_EXCLUSIVE_LOCK = 0x2 + + +# See: https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-unlockfileex +_UnlockFileEx = ctypes.windll.kernel32.UnlockFileEx +_UnlockFileEx.argtypes = ( + HANDLE, # hFile + DWORD, # dwReserved + DWORD, # nNumberOfBytesToLockLow + DWORD, # nNumberOfBytesToLockHigh + Overlapped, # lpOverlapped +) +_UnlockFileEx.restype = BOOL + + +class WindowsFileLock(FileLock): + @classmethod + def acquire( + cls, + fd, # type: int + exclusive, # type: bool + ): + # type: (...) -> WindowsFileLock + + mode = 0 # The default is a shared lock. + if exclusive: + mode |= _LOCKFILE_EXCLUSIVE_LOCK + + overlapped = Overlapped.ignored() + if not _LockFileEx( + HANDLE(msvcrt.get_osfhandle(fd)), # hFile + DWORD(mode), # dwFlags + DWORD(0), # dwReserved + DWORD(1), # nNumberOfBytesToLockLow + DWORD(0), # nNumberOfBytesToLockHigh + overlapped, # lpOverlapped + ): + raise ctypes.WinError() + return cls(locked_fd=fd, unlock=lambda: cls.release_lock(fd, overlapped=overlapped)) + + @classmethod + def release_lock( + cls, + fd, # type: int + overlapped=None, # type: Optional[Overlapped] + ): + # type: (int) -> None + + if not _UnlockFileEx( + HANDLE(msvcrt.get_osfhandle(fd)), # hFile + DWORD(0), # dwReserved + DWORD(1), # nNumberOfBytesToLockLow + DWORD(0), # nNumberOfBytesToLockHigh + overlapped or Overlapped.ignored(), # lpOverlapped + ): + raise ctypes.WinError() diff --git a/pex/fs/lock.py b/pex/fs/lock.py new file mode 100644 index 000000000..266ad1265 --- /dev/null +++ b/pex/fs/lock.py @@ -0,0 +1,89 @@ +# Copyright 2025 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import os + +from pex.common import safe_mkdir +from pex.enum import Enum +from pex.os import WINDOWS +from pex.typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any, Callable, Optional + + import attr # vendor: skip +else: + from pex.third_party import attr + + +class FileLockStyle(Enum["FileLockStyle.Value"]): + class Value(Enum.Value): + pass + + BSD = Value("bsd") + POSIX = Value("posix") + + +FileLockStyle.seal() + + +@attr.s(frozen=True) +class FileLock(object): + _locked_fd = attr.ib() # type: int + _unlock = attr.ib() # type: Callable[[], Any] + + @property + def fd(self): + # type: () -> int + return self._locked_fd + + def release(self): + # type: () -> None + try: + self._unlock() + finally: + os.close(self._locked_fd) + + +def acquire( + path, # type: str + exclusive=True, # type: bool + style=FileLockStyle.POSIX, # type: FileLockStyle.Value + fd=None, # type: Optional[int] +): + # type: (...) -> FileLock + + if fd: + lock_fd = fd + else: + # N.B.: We don't actually write anything to the lock file but the fcntl file locking + # operations only work on files opened for at least write. + safe_mkdir(os.path.dirname(path)) + lock_fd = os.open(path, os.O_CREAT | os.O_WRONLY) + + if WINDOWS: + from pex.fs._windows import WindowsFileLock + + return WindowsFileLock.acquire(lock_fd, exclusive=exclusive) + else: + from pex.fs._posix import PosixFileLock + + return PosixFileLock.acquire(lock_fd, exclusive=exclusive, style=style) + + +def release( + fd, # type: int + style=FileLockStyle.POSIX, # type: FileLockStyle.Value +): + # type: (...) -> None + + if WINDOWS: + from pex.fs._windows import WindowsFileLock + + WindowsFileLock.release_lock(fd) + else: + from pex.fs._posix import PosixFileLock + + return PosixFileLock.release_lock(fd, style=style) \ No newline at end of file diff --git a/pex/interpreter.py b/pex/interpreter.py index 833022e91..4cd5856c2 100644 --- a/pex/interpreter.py +++ b/pex/interpreter.py @@ -22,6 +22,7 @@ from pex.executor import Executor from pex.jobs import Job, Retain, SpawnedJob, execute_parallel from pex.orderedset import OrderedSet +from pex.os import WINDOWS from pex.pep_425 import CompatibilityTags from pex.pep_508 import MarkerEnvironment from pex.platforms import Platform @@ -29,6 +30,8 @@ from pex.pyenv import Pyenv from pex.third_party.packaging import __version__ as packaging_version from pex.third_party.packaging import tags + +from pex.sysconfig import EXE_EXTENSION, script_name, SCRIPT_DIR from pex.tracer import TRACER from pex.typing import TYPE_CHECKING, cast, overload @@ -140,8 +143,7 @@ def _adjust_to_final_path(path): # type: (str) -> str for current_path, final_path in _PATH_MAPPINGS.items(): if path.startswith(current_path): - prefix_pattern = re.escape(current_path) - return re.sub(prefix_pattern, final_path, path) + return final_path + path[len(current_path):] return path @@ -149,8 +151,7 @@ def _adjust_to_current_path(path): # type: (str) -> str for current_path, final_path in _PATH_MAPPINGS.items(): if path.startswith(final_path): - prefix_pattern = re.escape(final_path) - return re.sub(prefix_pattern, current_path, path) + return current_path + path[len(final_path):] return path @@ -798,7 +799,7 @@ def include_system_site_packages(self): class PythonInterpreter(object): _REGEXEN = ( # NB: OSX ships python binaries named Python with a capital-P; so we allow for this. - re.compile(r"^Python$"), + re.compile(r"^Python{extension}$".format(extension=re.escape(EXE_EXTENSION))), re.compile( r""" ^ @@ -818,8 +819,9 @@ class PythonInterpreter(object): [a-z]? )? )? + {extension} $ - """, + """.format(extension=re.escape(EXE_EXTENSION)), flags=re.VERBOSE, ), ) @@ -1423,11 +1425,17 @@ def resolve_base_interpreter(self): prefix = "pypy" if self.is_pypy else "python" suffixes = ("{}.{}".format(version[0], version[1]), str(version[0]), "") - candidate_binaries = tuple("{}{}".format(prefix, suffix) for suffix in suffixes) + candidate_binaries = tuple(script_name("{}{}".format(prefix, suffix)) for suffix in suffixes) def iter_base_candidate_binary_paths(interpreter): # type: (PythonInterpreter) -> Iterator[str] - bin_dir = os.path.join(interpreter._identity.base_prefix, "bin") + + # TODO(John Sirois): XXX: Is this always right on Windows? + bin_dir = ( + interpreter._identity.base_prefix + if WINDOWS else + os.path.join(interpreter._identity.base_prefix, SCRIPT_DIR) + ) for candidate_binary in candidate_binaries: candidate_binary_path = os.path.join(bin_dir, candidate_binary) if is_exe(candidate_binary_path): diff --git a/pex/os.py b/pex/os.py new file mode 100644 index 000000000..34e66e311 --- /dev/null +++ b/pex/os.py @@ -0,0 +1,83 @@ +# Copyright 2025 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import os +import sys + +from pex.typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import List, NoReturn, Text + +# N.B.: Python 2.7 uses "linux2". +LINUX = sys.platform.startswith("linux") +MAC = sys.platform == "darwin" +WINDOWS = sys.platform == "win32" + + +HOME_ENV_VAR = "USERPROFILE" if WINDOWS else "HOME" + + +if WINDOWS: + + def safe_execv(argv): + # type: (List[str]) -> NoReturn + import subprocess + import sys + + sys.exit(subprocess.call(args=argv)) + +else: + + def safe_execv(argv): + # type: (List[str]) -> NoReturn + os.execv(argv[0], argv) + + +if WINDOWS: + _GBT = None + + def is_exe(path): + # type: (Text) -> bool + + if not os.path.isfile(path): + return False + + from pex.sysconfig import EXE_EXTENSIONS + _, ext = os.path.splitext(path) + if ext.lower() in EXE_EXTENSIONS: + return True + + import ctypes + from ctypes.wintypes import BOOL, DWORD, LPCWSTR, LPDWORD + + global _GBT + if _GBT is None: + gbt = ctypes.windll.kernel32.GetBinaryTypeW + gbt.argtypes = ( + # lpApplicationName + LPCWSTR, + # lpBinaryType + LPDWORD, + ) + gbt.restype = BOOL + _GBT = gbt + + # See: https://learn.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-getbinarytypew + # N.B.: We don't care about the binary type, just the bool which tells us it is or is not an + # executable. + _binary_type = DWORD(0) + return bool(_GBT(path, ctypes.byref(_binary_type))) + +else: + + def is_exe(path): + # type: (Text) -> bool + """Determines if the given path is a file executable by the current user. + + :param path: The path to check. + :return: `True if the given path is a file executable by the current user. + """ + return os.path.isfile(path) and os.access(path, os.R_OK | os.X_OK) diff --git a/pex/pep_427.py b/pex/pep_427.py index 291598476..d1c66cec2 100644 --- a/pex/pep_427.py +++ b/pex/pep_427.py @@ -22,6 +22,7 @@ from pex.interpreter import PythonInterpreter from pex.pep_376 import InstalledFile, InstalledWheel, Record from pex.pep_503 import ProjectName +from pex.sysconfig import SCRIPT_DIR from pex.typing import TYPE_CHECKING, cast from pex.wheel import Wheel @@ -74,7 +75,7 @@ def chroot( purelib=destination, platlib=destination, headers=os.path.join(base, "include", "site", "pythonX.Y", project_name.raw), - scripts=os.path.join(base, "bin"), + scripts=os.path.join(base, SCRIPT_DIR), data=base, ) diff --git a/pex/pex.py b/pex/pex.py index 2ce7c478a..8a9177f7f 100644 --- a/pex/pex.py +++ b/pex/pex.py @@ -23,6 +23,7 @@ from pex.interpreter import PythonIdentity, PythonInterpreter from pex.layout import Layout from pex.orderedset import OrderedSet +from pex.os import safe_execv from pex.pex_info import PexInfo from pex.targets import LocalInterpreter from pex.tracer import TRACER @@ -719,7 +720,7 @@ def execute_with_options( for arg in python_options ): os.environ["PYTHONINSPECT"] = "1" - os.execv(python, cmdline) + safe_execv(cmdline) def execute_script(self, script_name): # type: (str) -> Any diff --git a/pex/pex_boot.py b/pex/pex_boot.py index fa7d5755e..6974b9e56 100644 --- a/pex/pex_boot.py +++ b/pex/pex_boot.py @@ -53,11 +53,13 @@ def __re_exec__( ): # type: (...) -> NoReturn + from pex.os import safe_execv + argv = [python] argv.extend(python_args) argv.extend(extra_python_args) - os.execv(python, argv + sys.argv[1:]) + safe_execv(argv + sys.argv[1:]) __SHOULD_EXECUTE__ = __name__ == "__main__" @@ -116,6 +118,7 @@ def __maybe_run_venv__( # type: (...) -> Optional[str] from pex.executables import is_exe + from pex.sysconfig import SCRIPT_DIR, script_name from pex.tracer import TRACER from pex.variables import venv_dir @@ -133,7 +136,7 @@ def __maybe_run_venv__( return venv_root_dir TRACER.log("Executing venv PEX for {pex} at {venv_pex}".format(pex=pex, venv_pex=venv_pex)) - venv_python = os.path.join(venv_root_dir, "bin", "python") + venv_python = os.path.join(venv_root_dir, SCRIPT_DIR, script_name("python")) if hermetic_venv_scripts: __re_exec__(venv_python, python_args, "-sE", venv_pex) else: diff --git a/pex/pex_bootstrapper.py b/pex/pex_bootstrapper.py index d4fcda49a..484a7ba47 100644 --- a/pex/pex_bootstrapper.py +++ b/pex/pex_bootstrapper.py @@ -21,7 +21,9 @@ ) from pex.layout import Layout from pex.orderedset import OrderedSet +from pex.os import safe_execv from pex.pex_info import PexInfo +from pex.sysconfig import SCRIPT_DIR, script_name from pex.targets import LocalInterpreter from pex.tracer import TRACER from pex.typing import TYPE_CHECKING, cast @@ -451,7 +453,7 @@ def maybe_reexec_pex( # Avoid a re-run through compatibility_constraint checking. os.environ[current_interpreter_blessed_env_var] = "1" - os.execv(target_binary, cmdline) + safe_execv(cmdline) def _bootstrap(entry_point): @@ -471,7 +473,7 @@ class VenvPex(object): def bin_file(self, name): # type: (str) -> str - return os.path.join(self.venv_dir, "bin", name) + return os.path.join(self.venv_dir, SCRIPT_DIR, script_name(name)) def __attrs_post_init__(self): # type: () -> None @@ -597,7 +599,7 @@ def ensure_venv( python=os.path.join( venv_dirs.short_dir, "venv", - "bin", + SCRIPT_DIR, os.path.basename(virtualenv.interpreter.binary), ), collisions_ok=collisions_ok, @@ -707,7 +709,7 @@ def _activate_venv_dir( venv_python = None if venv_dir: - python = os.path.join(venv_dir, "bin", "python") + python = os.path.join(venv_dir, SCRIPT_DIR, script_name("python")) if os.path.exists(python): venv_python = python diff --git a/pex/pex_builder.py b/pex/pex_builder.py index a3a205afd..d647d52ae 100644 --- a/pex/pex_builder.py +++ b/pex/pex_builder.py @@ -32,6 +32,7 @@ from pex.enum import Enum from pex.executables import chmod_plus_x, create_sh_python_redirector_shebang from pex.finders import get_entry_point_from_console_script, get_script_from_distributions +from pex.fs import safe_rename from pex.interpreter import PythonInterpreter from pex.layout import Layout from pex.orderedset import OrderedSet @@ -551,7 +552,7 @@ def _prepare_bootstrap(self): ) bootstrap_digest = hashlib.sha1() - bootstrap_packages = ["cache", "repl", "third_party", "venv"] + bootstrap_packages = ["cache", "fs", "repl", "third_party", "venv"] if self._pex_info.includes_tools: bootstrap_packages.extend(["commands", "tools"]) @@ -671,7 +672,7 @@ def build( main_fp.readline() # Throw away shebang line. shutil.copyfileobj(main_fp, script_fp) chmod_plus_x(pex_script) - os.rename(pex_script, main_py) + safe_rename(pex_script, main_py) os.symlink("__main__.py", pex_script) if os.path.isdir(path): @@ -679,7 +680,7 @@ def build( elif os.path.isdir(tmp_pex): safe_delete(path) check.perform_check(layout, tmp_pex) - os.rename(tmp_pex, path) + safe_rename(tmp_pex, path) def set_sh_boot_script( self, diff --git a/pex/pex_info.py b/pex/pex_info.py index b90fc899a..d944bd7e3 100644 --- a/pex/pex_info.py +++ b/pex/pex_info.py @@ -10,10 +10,11 @@ from pex import layout, pex_warnings, variables from pex.cache import root as cache_root from pex.common import can_write_dir, open_zip, safe_mkdtemp -from pex.compatibility import PY2, WINDOWS +from pex.compatibility import PY2 from pex.compatibility import string as compatibility_string from pex.inherit_path import InheritPath from pex.orderedset import OrderedSet +from pex.os import WINDOWS from pex.typing import TYPE_CHECKING, cast from pex.variables import ENV, Variables from pex.venv.bin_path import BinPath diff --git a/pex/scie/configure-binding.py b/pex/scie/configure-binding.py index 0f4b74081..386b0ecf7 100644 --- a/pex/scie/configure-binding.py +++ b/pex/scie/configure-binding.py @@ -7,6 +7,8 @@ import sys from argparse import ArgumentParser +from pex.sysconfig import SCRIPT_DIR + # When running under MyPy, this will be set to True for us automatically; so we can use it as a # typing module import guard to protect Python 2 imports of typing - which is not normally available # in Python 2. @@ -27,7 +29,7 @@ def write_bindings( print("PYTHON=" + sys.executable, file=fp) print("PEX=" + pex, file=fp) if venv_dir: - print("VENV_BIN_DIR_PLUS_SEP=" + os.path.join(venv_dir, "bin") + os.path.sep, file=fp) + print("VENV_BIN_DIR_PLUS_SEP=" + os.path.join(venv_dir, SCRIPT_DIR) + os.path.sep, file=fp) if __name__ == "__main__": diff --git a/pex/sysconfig.py b/pex/sysconfig.py new file mode 100644 index 000000000..7e4fa672d --- /dev/null +++ b/pex/sysconfig.py @@ -0,0 +1,29 @@ +# Copyright 2025 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import os.path +from sysconfig import get_config_var + +from pex.os import WINDOWS + +EXE_EXTENSION = get_config_var("EXE") or "" +EXE_EXTENSIONS = ( + tuple(ext.lower() for ext in os.environ.get("PATHEXT", EXE_EXTENSION).split(os.pathsep)) + if EXE_EXTENSION + else () +) + + +def script_name(name): + # type: (str) -> str + if not EXE_EXTENSION: + return name + stem, ext = os.path.splitext(name) + return name if (ext and ext.lower() in EXE_EXTENSIONS) else name + EXE_EXTENSION + + +# TODO(John Sirois): XXX: Use sysconfig.get_path("scripts", expand=False) + +# sysconfig.get_config_vars() +SCRIPT_DIR = "Scripts" if WINDOWS else "bin" diff --git a/pex/venv/installer.py b/pex/venv/installer.py index f30e56fd6..2eabe3154 100644 --- a/pex/venv/installer.py +++ b/pex/venv/installer.py @@ -17,11 +17,13 @@ from pex.environment import PEXEnvironment from pex.executables import chmod_plus_x from pex.orderedset import OrderedSet +from pex.os import WINDOWS from pex.pep_376 import InstalledWheel from pex.pep_440 import Version from pex.pep_503 import ProjectName from pex.pex import PEX from pex.result import Error +from pex.sysconfig import SCRIPT_DIR from pex.tracer import TRACER from pex.typing import TYPE_CHECKING from pex.util import CacheHelper @@ -304,7 +306,7 @@ def populate_venv_distributions( venv=venv, distributions=distributions, venv_python=provenance.target_python, - copy_mode=copy_mode, + copy_mode=CopyMode.LINK if copy_mode is CopyMode.SYMLINK and WINDOWS else copy_mode, hermetic_scripts=hermetic_scripts, top_level_source_packages=top_level_source_packages, ) @@ -345,7 +347,7 @@ def populate_venv_sources( def _iter_top_level_packages( path, # type: str - excludes=("bin", "__pycache__"), # type: Container[str] + excludes=(SCRIPT_DIR, "__pycache__"), # type: Container[str] ): # type: (...) -> Iterator[str] for name in os.listdir(path): @@ -418,11 +420,11 @@ def _populate_legacy_dist( # just 1 top-level module, we keep .pyc anchored to their associated dists when shared # and accept the cost of re-compiling top-level modules in each venv that uses them. for src, dst in iter_copytree( - src=dist.location, dst=dest_dir, exclude=("bin", "__pycache__"), copy_mode=copy_mode + src=dist.location, dst=dest_dir, exclude=(SCRIPT_DIR, "__pycache__"), copy_mode=copy_mode ): yield src, dst - dist_bin_dir = os.path.join(dist.location, "bin") + dist_bin_dir = os.path.join(dist.location, SCRIPT_DIR) if os.path.isdir(dist_bin_dir): for src, dst in iter_copytree(src=dist_bin_dir, dst=bin_dir, copy_mode=copy_mode): yield src, dst @@ -592,8 +594,8 @@ def _populate_first_party( print( "import os, sys; " "sys.path.extend(" - "entry for entry in os.environ.get('{env_var}', '').split(':') if entry" - ")".format(env_var=env_var), + "entry for entry in os.environ.get('{env_var}', '').split({pathsep!r}) if entry" + ")".format(env_var=env_var, pathsep=os.pathsep), file=fp, ) @@ -613,6 +615,7 @@ def _populate_first_party( if __name__ == "__main__": boot( shebang_python={shebang_python!r}, + venv_bin_dir={venv_bin_dir!r}, bin_path={bin_path!r}, strip_pex_env={strip_pex_env!r}, inject_env={inject_env!r}, @@ -626,6 +629,7 @@ def _populate_first_party( shebang=shebang, code=inspect.getsource(venv_pex).strip(), shebang_python=venv_python, + venv_bin_dir=SCRIPT_DIR, bin_path=bin_path, strip_pex_env=pex_info.strip_pex_env, inject_env=tuple(pex_info.inject_env.items()), diff --git a/pex/venv/venv_pex.py b/pex/venv/venv_pex.py index 613494eb6..2242df411 100644 --- a/pex/venv/venv_pex.py +++ b/pex/venv/venv_pex.py @@ -10,7 +10,7 @@ if TYPE_CHECKING: from types import CodeType - from typing import Any, Dict, Iterable, List, Optional, Tuple + from typing import Any, Dict, Iterable, List, Optional, Tuple, NoReturn _PY2_EXEC_FUNCTION = """ def exec_function(ast, globals_map): @@ -40,8 +40,21 @@ def exec_function( return locals_map +if sys.platform == "win32": + def safe_execv(argv): + # type: (List[str]) -> NoReturn + import subprocess + + sys.exit(subprocess.call(args=argv)) +else: + def safe_execv(argv): + # type: (List[str]) -> NoReturn + os.execv(argv[0], argv) + + def boot( shebang_python, # type: str + venv_bin_dir, # type: str bin_path, # type: str strip_pex_env, # type: bool inject_env, # type: Iterable[Tuple[str, str]] @@ -53,7 +66,7 @@ def boot( # type: (...) -> None venv_dir = os.path.abspath(os.path.dirname(__file__)) - venv_bin_dir = os.path.join(venv_dir, "bin") + venv_bin_dir = os.path.join(venv_dir, venv_bin_dir) python = os.path.join(venv_bin_dir, os.path.basename(shebang_python)) def iter_valid_venv_pythons(): @@ -96,7 +109,7 @@ def maybe_log(*message): if hermetic_re_exec: argv.append("-sE") argv.extend(sys.argv) - os.execv(python, argv) + safe_execv(argv) pex_file = os.environ.get("PEX", None) if pex_file: @@ -233,7 +246,7 @@ def maybe_log(*message): pex_script = pex_overrides.get("PEX_SCRIPT") if pex_overrides else script if pex_script: script_path = os.path.join(venv_bin_dir, pex_script) - os.execv(script_path, [script_path] + sys.argv[1:]) + safe_execv([script_path] + sys.argv[1:]) pex_interpreter = pex_overrides.get("PEX_INTERPRETER", "").lower() in ("1", "true") entry_point = None if pex_interpreter else pex_overrides.get("PEX_MODULE", entry_point) @@ -279,7 +292,7 @@ def maybe_log(*message): "Re-executing with Python interpreter options: " "cmdline={cmdline!r}".format(cmdline=" ".join(cmdline)) ) - os.execv(python, cmdline) + safe_execv(cmdline) arg = args[0] if arg == "-m": diff --git a/pex/venv/virtualenv.py b/pex/venv/virtualenv.py index de7d1f7da..bf6678e63 100644 --- a/pex/venv/virtualenv.py +++ b/pex/venv/virtualenv.py @@ -31,6 +31,7 @@ create_shebang, ) from pex.orderedset import OrderedSet +from pex.sysconfig import SCRIPT_DIR, script_name from pex.tracer import TRACER from pex.typing import TYPE_CHECKING, cast from pex.util import named_temporary_file @@ -393,8 +394,8 @@ def __init__( # type: (...) -> None self._venv_dir = venv_dir self._custom_prompt = custom_prompt - self._bin_dir = os.path.join(venv_dir, "bin") - python_exe_path = os.path.join(self._bin_dir, python_exe_name) + self._bin_dir = os.path.join(venv_dir, SCRIPT_DIR) + python_exe_path = os.path.join(self._bin_dir, script_name(python_exe_name)) try: self._interpreter = PythonInterpreter.from_binary(python_exe_path) except PythonInterpreter.Error as e: @@ -451,7 +452,7 @@ def join_path(self, *components): def bin_path(self, *components): # type: (*str) -> str - return os.path.join(self._bin_dir, *components) + return script_name(os.path.join(self._bin_dir, *components)) @property def bin_dir(self): diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 62819e2b3..66b4b2507 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -20,13 +20,14 @@ from pex import targets from pex.cache.dirs import CacheDir, InterpreterDir from pex.common import environment_as, safe_mkdir, safe_open, safe_rmtree, temporary_dir, touch -from pex.compatibility import WINDOWS, commonpath +from pex.compatibility import commonpath from pex.dist_metadata import Distribution, Requirement, is_wheel from pex.executables import is_exe from pex.fetcher import URLFetcher from pex.interpreter import PythonInterpreter from pex.layout import Layout from pex.network_configuration import NetworkConfiguration +from pex.os import WINDOWS from pex.pep_427 import InstallableType from pex.pex_info import PexInfo from pex.pip.version import PipVersion diff --git a/tests/test_pex.py b/tests/test_pex.py index 5a2725b60..604516bd3 100644 --- a/tests/test_pex.py +++ b/tests/test_pex.py @@ -17,9 +17,10 @@ from pex import resolver from pex.common import environment_as, safe_mkdir, safe_open, temporary_dir -from pex.compatibility import PY2, WINDOWS, commonpath, to_bytes +from pex.compatibility import PY2, commonpath, to_bytes from pex.dist_metadata import Distribution from pex.interpreter import PythonIdentity, PythonInterpreter +from pex.os import WINDOWS from pex.pex import PEX, IsolatedSysPath from pex.pex_builder import PEXBuilder from pex.pex_info import PexInfo