Respect OS user cache location conventions. (#2505)

Change the default `PEX_ROOT` cache location from `~/.pex` to
`<OS default user cache dir>/pex`.

Work towards #2201, #1655 and #1176.

docker/user/Dockerfile

@@ -21,9 +21,10 @@ VOLUME /development/pex/.tox
 VOLUME /development/pex_dev
 ENV _PEX_TEST_DEV_ROOT=/development/pex_dev
 
-# This will be a named volume used to persist the Pex cache on the host but isolated from the nost
-# ~/.pex cache.
-VOLUME "/home/${USER}/.pex"
+# This will be a named volume used to persist the Pex cache on the host but isolated from the host
+# Pex cache.
+VOLUME /var/cache/pex
+ENV PEX_ROOT=/var/cache/pex
 
 # This will be a named volume used to persist the pytest tmp tree (/tmp/pytest-of-$USER/) for use \
 # in `./dtox inspect` sessions.

docs/recipes.rst

@@ -52,7 +52,7 @@ You'll see something like this as a result:
 
     $ ./my.pex --foo bar &
     $ ps -o command | grep pex
-    /home/jsirois/.pyenv/versions/3.10.2/bin/python3.10 /home/jsirois/.pex/unzipped_pexes/94790b07dc3768a9926dab999b41a87e399e0aa9 --foo bar
+    /home/jsirois/.pyenv/versions/3.10.2/bin/python3.10 /home/jsirois/.cache/pex/unzipped_pexes/94790b07dc3768a9926dab999b41a87e399e0aa9 --foo bar
 
 The original PEX file is not mentioned anywhere in the ``ps`` output. Worse, if you have many PEX
 processes it will be unclear which process corresponds to which PEX.

dtox.sh

@@ -53,7 +53,7 @@ if [[ -z "$(user_image_id)" ]]; then
   docker run \
     --rm \
     --volume pex-caches:/development/pex_dev \
-    --volume "pex-root:${CONTAINER_HOME}/.pex" \
+    --volume pex-root:/var/cache/pex \
     --volume pex-tmp:/tmp \
     --volume pex-tox:/development/pex/.tox \
     --entrypoint bash \
@@ -62,7 +62,7 @@ if [[ -z "$(user_image_id)" ]]; then
     -c "
       chown -R $(id -un):$(id -gn) \
       /development/pex_dev \
-      ${CONTAINER_HOME}/.pex \
+      /var/cache/pex \
       /tmp \
       /development/pex/.tox
     "
@@ -132,7 +132,7 @@ fi
 exec docker run \
   --rm \
   --volume pex-tmp:/tmp \
-  --volume "pex-root:${CONTAINER_HOME}/.pex" \
+  --volume pex-root:/var/cache/pex \
   --volume pex-caches:/development/pex_dev \
   --volume "${ROOT}:/development/pex" \
   --volume pex-tox:/development/pex/.tox \

mypy.ini

@@ -24,6 +24,9 @@ pretty = True
 [mypy-pex.third_party.*]
 ignore_missing_imports = True
 
+[mypy-appdirs]
+ignore_missing_imports = True
+
 [mypy-coloredlogs]
 ignore_missing_imports = True
 

pex/bin/pex.py

@@ -401,8 +401,11 @@ def configure_clp_pex_options(parser):
         "--runtime-pex-root",
         dest="runtime_pex_root",
         default=None,
-        help="Specify the pex root to be used in the generated .pex file (if unspecified, "
-        "uses ~/.pex).",
+        help=(
+            "Specify the pex root to be used in the generated .pex file (if unspecified, uses a "
+            "pex subdirectory of default user cache directory for the runtime OS; e.g.: "
+            "~/.cache/pex on Linux and ~/Library/Caches/pex on Mac)."
+        ),
     )
 
     group.add_argument(

pex/cache.py

@@ -0,0 +1,32 @@
+# Copyright 2024 Pex project contributors.
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import
+
+import os.path
+
+from pex.compatibility import commonpath
+from pex.typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from typing import Iterable
+
+    import appdirs  # vendor:skip
+else:
+    from pex.third_party import appdirs
+
+
+_USER_DIR = os.path.expanduser("~")
+_CACHE_DIR = appdirs.user_cache_dir(appauthor="pex-tool.org", appname="pex")  # type: str
+
+
+def cache_path(
+    sub_path=(),  # type: Iterable[str]
+    expand_user=True,  # type: bool
+):
+    # type: (...) -> str
+
+    path = os.path.join(_CACHE_DIR, *sub_path)
+    if expand_user or _USER_DIR != commonpath((_USER_DIR, path)):
+        return path
+    return os.path.join("~", os.path.relpath(path, _USER_DIR))

pex/pex_builder.py

@@ -549,7 +549,7 @@ def _prepare_bootstrap(self):
         # NB: We use pip here in the builder, but that's only at build time, and
         # although we don't use pyparsing directly, packaging.markers, which we
         # do use at runtime, does.
-        root_module_names = ["attr", "colors", "packaging", "pkg_resources", "pyparsing"]
+        root_module_names = ["appdirs", "attr", "colors", "packaging", "pkg_resources", "pyparsing"]
         prepared_sources = vendor.vendor_runtime(
             chroot=self._chroot,
             dest_basedir=self._pex_info.bootstrap,
@@ -797,8 +797,8 @@ def _build_zipapp(
                 mode="a",
                 deterministic_timestamp=deterministic_timestamp,
                 # When configured with a `copy_mode` of `CopyMode.SYMLINK`, we symlink distributions
-                # as pointers to installed wheel directories in ~/.pex/installed_wheels/... Since
-                # those installed wheels reside in a shared cache, they can be in-use by other
+                # as pointers to installed wheel directories in <PEX_ROOT>/installed_wheels/...
+                # Since those installed wheels reside in a shared cache, they can be in-use by other
                 # processes and so their code may be in the process of being bytecode compiled as we
                 # attempt to zip up our chroot. Bytecode compilation produces ephemeral temporary
                 # pyc files that we should avoid copying since they are useless and inherently

pex/pex_info.py

@@ -7,7 +7,7 @@
 import os
 import zipfile
 
-from pex import layout, pex_warnings, variables
+from pex import cache, layout, pex_warnings, variables
 from pex.common import can_write_dir, open_zip, safe_mkdtemp
 from pex.compatibility import PY2, WINDOWS
 from pex.compatibility import string as compatibility_string
@@ -45,7 +45,7 @@ class PexInfo(object):
     requirements: list                 # list of requirements for this environment
 
     # Environment options
-    pex_root: string                    # root of all pex-related files eg: ~/.pex
+    pex_root: string                    # root of all pex-related files eg: ~/.cache/pex
     entry_point: string                 # entry point into this pex
     script: string                      # script to execute in this pex environment
                                         # at most one of script/entry_point can be specified
@@ -503,7 +503,7 @@ def distributions(self):
     @property
     def raw_pex_root(self):
         # type: () -> str
-        return cast(str, self._pex_info.get("pex_root", os.path.join("~", ".pex")))
+        return cast(str, self._pex_info.get("pex_root", cache.cache_path(expand_user=False)))
 
     @property
     def pex_root(self):

pex/resolver.py

@@ -418,10 +418,10 @@ def finalize_install(self, install_requests):
         # machine.
         #
         # From a clean cache after building a simple pex this looks like:
-        # $ rm -rf ~/.pex
+        # $ rm -rf ~/.cache/pex
         # $ python -mpex -c pex -o /tmp/pex.pex .
-        # $ tree -L 4 ~/.pex/
-        # /home/jsirois/.pex/
+        # $ tree -L 4 ~/.cache/pex/
+        # /home/jsirois/.cache/pex/
         # ├── built_wheels
         # │ └── 1003685de2c3604dc6daab9540a66201c1d1f718
         # │     └── cp-38-cp38
@@ -433,7 +433,7 @@ def finalize_install(self, install_requests):
         #     │     ├── pex
         #     │     └── pex-2.0.2.dist-info
         #     └── ae13cba3a8e50262f4d730699a11a5b79536e3e1
-        #         └── pex-2.0.2-py2.py3-none-any.whl -> /home/jsirois/.pex/installed_wheels/2a594cef34d2e9109bad847358d57ac4615f81f4/pex-2.0.2-py2.py3-none-any.whl  # noqa
+        #         └── pex-2.0.2-py2.py3-none-any.whl -> /home/jsirois/.cache/pex/installed_wheels/2a594cef34d2e9109bad847358d57ac4615f81f4/pex-2.0.2-py2.py3-none-any.whl  # noqa
         #
         # 11 directories, 1 file
         #
@@ -457,7 +457,7 @@ def finalize_install(self, install_requests):
         # pex:   * /usr/lib/python38.zip
         # pex:     /usr/lib/python3.8
         # pex:     /usr/lib/python3.8/lib-dynload
-        # pex:     /home/jsirois/.pex/installed_wheels/2a594cef34d2e9109bad847358d57ac4615f81f4/pex-2.0.2-py2.py3-none-any.whl  # noqa
+        # pex:     /home/jsirois/.cache/pex/installed_wheels/2a594cef34d2e9109bad847358d57ac4615f81f4/pex-2.0.2-py2.py3-none-any.whl  # noqa
         # pex:   * /tmp/pex.pex/.bootstrap
         # pex:   * - paths that do not exist or will be imported via zipimport
         # pex.pex 2.0.2

pex/variables.py

@@ -49,11 +49,11 @@ class and call the `strip_default` method, passing the instance in question.
     def __init__(
         self,
         func,  # type: Callable[[_O], _P]
-        default,  # type: _P
+        default,  # type: Union[_P, Callable[[], _P]]
     ):
         # type: (...) -> None
         self._func = func  # type: Callable[[_O], _P]
-        self._default = default  # type: _P
+        self._default = default  # type: Union[_P, Callable[[], _P]]
         self._validator = None  # type: Optional[Callable[[_O, _P], _P]]
 
     @overload
@@ -85,7 +85,9 @@ def __get__(
         try:
             return self._validate(instance, self._func(instance))
         except NoValueError:
-            return self._validate(instance, self._default)
+            return self._validate(
+                instance, self._default() if callable(self._default) else self._default
+            )
 
     def strip_default(self, instance):
         # type: (_O) -> Optional[_P]
@@ -140,7 +142,8 @@ def _validate(self, instance, value):
 
 
 def defaulted_property(
-    default,  # type: _P
+    default,  # type: Union[_P, Callable[[], _P]]
+    _type_hint=None,  # type: Optional[Type[_P]]  # N.B.: Can be used to help MyPy along.
 ):
     # type: (...) -> Callable[[Callable[[_O], _P]], DefaultedProperty[_O, _P]]
     """Creates a `@property` with a `default` value.
@@ -156,6 +159,17 @@ def wrapped(func):
     return wrapped
 
 
+def _default_pex_root():
+    # type: () -> str
+
+    # N.B.: We need lazy import gymnastics here since cache uses appdirs which is pex.third_party
+    # and the pex.third_party mechanism uses ENV.PEX_VERBOSE indirectly via TRACER to log certain
+    # third party import lifecycle events.
+    from pex import cache
+
+    return cache.cache_path(expand_user=False)
+
+
 class Variables(object):
     """Environment variables supported by the PEX runtime."""
 
@@ -631,15 +645,12 @@ def PEX_EXTRA_SYS_PATH(self):
         """
         return self._maybe_get_path_tuple("PEX_EXTRA_SYS_PATH") or ()
 
-    @defaulted_property(default=os.path.join("~", ".pex"))
+    @defaulted_property(default=_default_pex_root, _type_hint=str)
     def PEX_ROOT(self):
         # type: () -> str
         """Directory.
 
-        The directory location for PEX to cache any dependencies and code.  PEX must write not-zip-
-        safe eggs and all wheels to disk in order to activate them.
-
-        Default: ~/.pex
+        The directory location for PEX to cache any dependencies and code.
         """
         return self._get_path("PEX_ROOT")
 

pex/vendor/__init__.py

@@ -175,6 +175,7 @@ def iter_vendor_specs(filter_requires_python=None):
         )
 
     yield VendorSpec.pinned("ansicolors", "1.1.8")
+    yield VendorSpec.pinned("appdirs", "1.4.4")
 
     # We use this for a better @dataclass that is also Python2.7 and PyPy compatible.
     # N.B.: The `[testenv:typecheck]` section in `tox.ini` should have its deps list updated to

pex/vendor/_vendored/appdirs/.layout.json

@@ -0,0 +1 @@
+{"fingerprint": "383339ca0d10d6c4144fc79e24b2c65304bdfcfbebabcf10c8ec6e9b73ee64fd", "record_relpath": "appdirs-1.4.4.dist-info/RECORD", "root_is_purelib": true, "stash_dir": ".prefix"}

pex/vendor/_vendored/appdirs/appdirs-1.4.4.dist-info/INSTALLER

@@ -0,0 +1 @@
+pex

pex/vendor/_vendored/appdirs/appdirs-1.4.4.dist-info/LICENSE.txt

@@ -0,0 +1,23 @@
+# This is the MIT license
+
+Copyright (c) 2010 ActiveState Software Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+