Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use Windows scripts stubs. #2668

Merged
merged 2 commits into from
Feb 9, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions pex/cache/dirs.py
Original file line number Diff line number Diff line change
Expand Up @@ -336,6 +336,10 @@ def iter_all(cls, pex_root=ENV):
for venv_short_dir_symlink in glob.glob(
CacheDir.VENVS.path("s", "*", cls.SHORT_SYMLINK_NAME, pex_root=pex_root)
):
# TODO(John Sirois): Explain or limit this Windows hack.
# See: https://github.com/pex-tool/pex/issues/2660#issuecomment-2635441311
venv_short_dir_symlink = os.path.realpath(venv_short_dir_symlink)

if not os.path.isdir(venv_short_dir_symlink):
continue

Expand Down
4 changes: 4 additions & 0 deletions pex/dist_metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -376,6 +376,10 @@ def iter_metadata_files(
):
# type: (...) -> Iterator[MetadataFiles]

# TODO(John Sirois): Explain or limit this Windows hack.
# See: https://github.com/pex-tool/pex/issues/2660#issuecomment-2635441311
location = os.path.realpath(location)

files = []
for metadata_type in restrict_types_to or MetadataType.values():
key = MetadataKey(metadata_type=metadata_type, location=location)
Expand Down
9 changes: 5 additions & 4 deletions pex/environment.py
Original file line number Diff line number Diff line change
Expand Up @@ -301,11 +301,12 @@ def iter_distributions(self, result_type_wheel_file=False):
):
with identify_layout(self.source_pex) as layout:
for distribution_name, fingerprint in self._pex_info.distributions.items():
dist_relpath = os.path.join(
self._pex_info.internal_cache, distribution_name
)
yield FingerprintedDistribution(
distribution=Distribution.load(layout.wheel_file_path(dist_relpath)),
distribution=Distribution.load(
layout.wheel_file_path(
(self._pex_info.internal_cache, distribution_name)
)
),
fingerprint=fingerprint,
)
else:
Expand Down
73 changes: 40 additions & 33 deletions pex/layout.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ def dist_strip_prefix(self, dist_name):
@abstractmethod
def dist_size(
self,
dist_relpath, # type: str
dist_relpath_components, # type: Tuple[str, ...]
is_wheel_file, # type: bool
):
# type: (...) -> int
Expand All @@ -126,15 +126,15 @@ def dist_size(
def extract_dist(
self,
dest_dir, # type: str
dist_relpath, # type: str
dist_relpath_components, # type: Tuple[str, ...]
is_wheel_file, # type: bool
):
# type: (...) -> None
raise NotImplementedError()

@abstractmethod
def wheel_file_path(self, dist_relpath):
# type: (str) -> str
def wheel_file_path(self, dist_relpath_components):
# type: (Tuple[str, ...]) -> str
raise NotImplementedError()

@abstractmethod
Expand Down Expand Up @@ -176,7 +176,8 @@ def _install_distribution(
spread_dest = InstalledWheelDir.create(
wheel_name=location, install_hash=sha, pex_root=pex_info.pex_root
)
dist_relpath = os.path.join(DEPS_DIR, location)
dist_relpath_components = (DEPS_DIR, location)
dist_relpath = os.path.join(*dist_relpath_components)
source = None if is_wheel_file else layout.dist_strip_prefix(location)
symlink_src = os.path.relpath(
spread_dest,
Expand All @@ -188,7 +189,7 @@ def _install_distribution(
if not spread_chroot.is_finalized():
layout.extract_dist(
dest_dir=spread_chroot.work_dir,
dist_relpath=dist_relpath,
dist_relpath_components=dist_relpath_components,
is_wheel_file=is_wheel_file,
)

Expand Down Expand Up @@ -239,7 +240,7 @@ def _ensure_distributions_installed_parallel(
verb_past="installed",
max_jobs=max_jobs,
costing_function=lambda item: layout.dist_size(
os.path.join(DEPS_DIR, item[0]), is_wheel_file=pex_info.deps_are_wheel_files
(DEPS_DIR, item[0]), is_wheel_file=pex_info.deps_are_wheel_files
),
)

Expand Down Expand Up @@ -267,7 +268,7 @@ def _ensure_distributions_installed(
install_serial = dist_count == 1 or pex_info.max_install_jobs == 1
if not install_serial and pex_info.max_install_jobs == -1:
total_size = sum(
layout.dist_size(os.path.join(DEPS_DIR, location), pex_info.deps_are_wheel_files)
layout.dist_size((DEPS_DIR, location), pex_info.deps_are_wheel_files)
for location in pex_info.distributions
)
average_distribution_size = total_size // dist_count
Expand Down Expand Up @@ -455,40 +456,42 @@ def dist_strip_prefix(self, dist_name):

def dist_size(
self,
dist_relpath, # type: str
dist_relpath_components, # type: Tuple[str, ...]
is_wheel_file, # type: bool
):
# type: (...) -> int
zip_relpath = "/".join(dist_relpath_components)
if is_wheel_file:
return self.zfp.getinfo(dist_relpath).file_size
return self.zfp.getinfo(zip_relpath).file_size
else:
return sum(
self.zfp.getinfo(name).file_size
for name in self.names
if name.startswith(dist_relpath)
if name.startswith(zip_relpath)
)

def extract_dist(
self,
dest_dir, # type: str
dist_relpath, # type: str
dist_relpath_components, # type: Tuple[str, ...]
is_wheel_file, # type: bool
):
# type: (...) -> None
if is_wheel_file:
from pex.pep_427 import install_wheel_chroot

install_wheel_chroot(self.wheel_file_path(dist_relpath), dest_dir)
install_wheel_chroot(self.wheel_file_path(dist_relpath_components), dest_dir)
else:
zip_relpath = "/".join(dist_relpath_components)
for name in self.names:
if name.startswith(dist_relpath) and not name.endswith("/"):
if name.startswith(zip_relpath) and not name.endswith("/"):
self.zfp.extract(name, dest_dir)

def wheel_file_path(self, dist_relpath):
# type: (str) -> str
def wheel_file_path(self, dist_relpath_components):
# type: (Tuple[str, ...]) -> str
extract_chroot = safe_mkdtemp()
self.zfp.extract(dist_relpath, extract_chroot)
return os.path.join(extract_chroot, dist_relpath)
self.zfp.extract("/".join(dist_relpath_components), extract_chroot)
return os.path.join(extract_chroot, *dist_relpath_components)

def extract_code(self, dest_dir):
# type: (str) -> None
Expand Down Expand Up @@ -526,20 +529,21 @@ def extract_bootstrap(self, dest_dir):

def dist_size(
self,
dist_relpath, # type: str
dist_relpath_components, # type: Tuple[str, ...]
is_wheel_file, # type: bool
):
# type: (...) -> int
return os.path.getsize(os.path.join(self._path, dist_relpath))
return os.path.getsize(os.path.join(self._path, *dist_relpath_components))

def extract_dist(
self,
dest_dir, # type: str
dist_relpath, # type: str
dist_relpath_components, # type: Tuple[str, ...]
is_wheel_file, # type: bool
):
# type: (...) -> None
dist_path = self.wheel_file_path(dist_relpath)
dist_relpath = os.path.join(*dist_relpath_components)
dist_path = self.wheel_file_path(dist_relpath_components)
if is_wheel_file:
from pex.pep_427 import install_wheel_chroot

Expand All @@ -550,9 +554,9 @@ def extract_dist(
with open_zip(dist_path) as zfp:
zfp.extractall(dest_dir)

def wheel_file_path(self, dist_relpath):
# type: (str) -> str
return os.path.join(self._path, dist_relpath)
def wheel_file_path(self, dist_relpath_components):
# type: (Tuple[str, ...]) -> str
return os.path.join(self._path, *dist_relpath_components)

def extract_code(self, dest_dir):
# type: (str) -> None
Expand Down Expand Up @@ -607,31 +611,34 @@ def extract_bootstrap(self, dest_dir):

def dist_size(
self,
dist_relpath, # type: str
dist_relpath_components, # type: Tuple[str, ...]
is_wheel_file, # type: bool
):
assert (
is_wheel_file
), "Expected loose layout install to be skipped when deps are pre-installed wheel chroots."
return os.path.getsize(os.path.join(self._path, dist_relpath))
return os.path.getsize(os.path.join(self._path, *dist_relpath_components))

def extract_dist(
self,
dest_dir,
dist_relpath, # type: str
dist_relpath_components, # type: Tuple[str, ...]
is_wheel_file, # type: bool
):
# type: (...) -> None
assert (
is_wheel_file
), "Expected loose layout install to be skipped when deps are pre-installed wheel chroots."
from pex.pep_427 import install_wheel_chroot

with TRACER.timed("Installing wheel file {}".format(dist_relpath)):
install_wheel_chroot(self.wheel_file_path(dist_relpath), dest_dir)
with TRACER.timed(
"Installing wheel file {}".format(os.path.join(*dist_relpath_components))
):
install_wheel_chroot(self.wheel_file_path(dist_relpath_components), dest_dir)

def wheel_file_path(self, dist_relpath):
# type: (str) -> str
return os.path.join(self._path, dist_relpath)
def wheel_file_path(self, dist_relpath_components):
# type: (Tuple[str, ...]) -> str
return os.path.join(self._path, *dist_relpath_components)

def extract_code(self, dest_dir):
# type: (str) -> None
Expand Down
17 changes: 11 additions & 6 deletions pex/pep_427.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,14 @@
from fileinput import FileInput
from textwrap import dedent

from pex import pex_warnings
from pex import pex_warnings, windows
from pex.common import is_pyc_file, iter_copytree, open_zip, safe_open, touch
from pex.compatibility import commonpath, get_stdout_bytes_buffer
from pex.dist_metadata import CallableEntryPoint, Distribution, ProjectNameAndVersion
from pex.enum import Enum
from pex.executables import chmod_plus_x
from pex.interpreter import PythonInterpreter
from pex.os import WINDOWS
from pex.pep_376 import InstalledFile, InstalledWheel, Record
from pex.pep_503 import ProjectName
from pex.sysconfig import SCRIPT_DIR
Expand Down Expand Up @@ -287,8 +288,9 @@ def record_files(

dist = Distribution(location=dest, metadata=wheel.dist_metadata())
entry_points = dist.get_entry_map()
for named_entry_point in itertools.chain.from_iterable(
entry_points.get(key, {}).values() for key in ("console_scripts", "gui_scripts")
for named_entry_point, gui in itertools.chain.from_iterable(
((value, gui) for value in entry_points.get(key, {}).values())
for key, gui in (("console_scripts", False), ("gui_scripts", True))
):
entry_point = named_entry_point.entry_point
if isinstance(entry_point, CallableEntryPoint):
Expand Down Expand Up @@ -328,9 +330,12 @@ def record_files(
modname=entry_point.module,
)
script_abspath = os.path.join(install_paths.scripts, named_entry_point.name)
with safe_open(script_abspath, "w") as fp:
fp.write(script)
chmod_plus_x(fp.name)
if WINDOWS:
script_abspath = windows.create_script(script_abspath, script, gui=gui)
else:
with safe_open(script_abspath, "w") as fp:
fp.write(script)
chmod_plus_x(fp.name)
installed_files.append(
InstalledWheel.create_installed_file(path=script_abspath, dest_dir=dest)
)
Expand Down
15 changes: 9 additions & 6 deletions pex/scie/configure-binding.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,43 +19,46 @@
def write_bindings(
env_file, # type: str
pex, # type: str
venv_dir=None, # type: Optional[str]
venv_bin_dir=None, # type: Optional[str]
):
# type: (...) -> None

with open(env_file, "a") as fp:
print("PYTHON=" + sys.executable, file=fp)
print("PEX=" + pex, file=fp)
if venv_dir:
print("VENV_BIN_DIR_PLUS_SEP=" + os.path.join(venv_dir, "bin") + os.path.sep, file=fp)
if venv_bin_dir:
print("VENV_BIN_DIR_PLUS_SEP=" + venv_bin_dir + os.path.sep, file=fp)


if __name__ == "__main__":
parser = ArgumentParser()
parser.add_argument(
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument(
"--installed-pex-dir",
help=(
"The final resting install directory of the PEX if it is a zipapp PEX. If left unset, "
"this indicates the PEX is a venv PEX whose resting venv directory should be "
"determined dynamically."
),
)
group.add_argument("--venv-bin-dir", help="The platform-specific venv bin dir name.")
options = parser.parse_args()

if options.installed_pex_dir:
pex = os.path.realpath(options.installed_pex_dir)
venv_dir = None
venv_bin_dir = None # type: Optional[str]
else:
venv_dir = os.path.realpath(
# N.B.: In practice, VIRTUAL_ENV should always be set by the PEX venv __main__.py
# script.
os.environ.get("VIRTUAL_ENV", os.path.dirname(os.path.dirname(sys.executable)))
)
pex = venv_dir
venv_bin_dir = os.path.join(venv_dir, options.venv_bin_dir)

write_bindings(
env_file=os.environ["SCIE_BINDING_ENV"],
pex=pex,
venv_dir=venv_dir,
venv_bin_dir=venv_bin_dir,
)
sys.exit(0)
Loading