Add Windows support in key critical areas. (#2663)

Although Pex still does not support Windows, this is work towards #2658.

In particular, support is added for:
+ Shared and exclusive file locks.
+ Versions of `os.link`, `os.symlink` and `os.rename` that either work
  with Windows or work with most modern Windows setups.
+ Executable file detection that works for Windows.
+ Partial support for Windows venvs by accommodating its `Scripts` dir
  and console script style (`.exe` stubs).

Informal testing shows the following work:
+ `py -mpex cowsay`
+ `py -mpex cowsay -ccowsay {,--venv} -- -t Moo!`
+ `py -mpex cowsay -ccowsay {,--venv} -ocowsay.pex`
+ `py -mpex cowsay -ccowsay -ocowsay.pex --scie {eager,lazy}`
+ `py -mpex.cli cache {prune,purge}`

Important non-working things include:
+ The developer system: `tox ...` needs work most fundamentally.
+ Venv scripts / venv scies: Need support for Windows `.exe` stub
  scripts.

pex/atomic_directory.py

@@ -4,17 +4,18 @@
 from __future__ import absolute_import
 
 import errno
-import fcntl
 import hashlib
 import os
 import threading
 from contextlib import contextmanager
 from uuid import uuid4
 
+import pex
 from pex import pex_warnings
 from pex.common import safe_mkdir, safe_rmtree
-from pex.enum import Enum
-from pex.typing import TYPE_CHECKING, cast
+from pex.fs import lock
+from pex.fs.lock import FileLockStyle
+from pex.typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from typing import Callable, Dict, Iterator, Optional
@@ -161,7 +162,7 @@ def finalize(self, source=None):
             #
             # We have satisfied the single filesystem constraint by arranging the `work_dir` to be a
             # sibling of the `target_dir`.
-            os.rename(source, self._target_dir)
+            pex.fs.safe_rename(source, self._target_dir)
         except OSError as e:
             if e.errno not in (errno.EEXIST, errno.ENOTEMPTY):
                 raise e
@@ -173,19 +174,8 @@ def cleanup(self):
         safe_rmtree(self._work_dir)
 
 
-class FileLockStyle(Enum["FileLockStyle.Value"]):
-    class Value(Enum.Value):
-        pass
-
-    BSD = Value("bsd")
-    POSIX = Value("posix")
-
-
-FileLockStyle.seal()
-
-
-def _is_bsd_lock(lock_style=None):
-    # type: (Optional[FileLockStyle.Value]) -> bool
+def _lock_style(lock_style=None):
+    # type: (Optional[FileLockStyle.Value]) -> FileLockStyle.Value
 
     # The atomic_directory file locking has used POSIX locks since inception. These have maximum
     # compatibility across OSes and stand a decent chance of working over modern NFS. With the
@@ -194,10 +184,9 @@ def _is_bsd_lock(lock_style=None):
     # `lock_style` to atomic_directory. In order to allow experimenting with / debugging possible
     # file locking bugs, we allow a `_PEX_FILE_LOCK_STYLE` back door private ~API to upgrade all
     # locks to BSD style locks. This back door can be removed at any time.
-    file_lock_style = lock_style or FileLockStyle.for_value(
+    return lock_style or FileLockStyle.for_value(
         os.environ.get("_PEX_FILE_LOCK_STYLE", FileLockStyle.POSIX.value)
     )
-    return file_lock_style is FileLockStyle.BSD
 
 
 @attr.s(frozen=True)
@@ -209,29 +198,14 @@ class _FileLock(object):
     def acquire(self):
         # type: () -> Callable[[], None]
         self._in_process_lock.acquire()
-
-        # N.B.: We don't actually write anything to the lock file but the fcntl file locking
-        # operations only work on files opened for at least write.
-        safe_mkdir(os.path.dirname(self._path))
-        lock_fd = os.open(self._path, os.O_CREAT | os.O_WRONLY)
-
-        lock_api = cast(
-            "Callable[[int, int], None]",
-            fcntl.flock if _is_bsd_lock(self._style) else fcntl.lockf,
-        )
-
-        # N.B.: Since lockf and flock operate on an open file descriptor and these are
-        # guaranteed to be closed by the operating system when the owning process exits,
-        # this lock is immune to staleness.
-        lock_api(lock_fd, fcntl.LOCK_EX)  # A blocking write lock.
+        file_lock = lock.acquire(self._path, exclusive=True, style=_lock_style(self._style))
 
         def release():
             # type: () -> None
             try:
-                lock_api(lock_fd, fcntl.LOCK_UN)
+                file_lock.release()
             finally:
-                os.close(lock_fd)
-            self._in_process_lock.release()
+                self._in_process_lock.release()
 
         return release
 

pex/cache/access.py

@@ -3,12 +3,14 @@
 
 from __future__ import absolute_import, print_function
 
-import fcntl
 import itertools
 import os
 from contextlib import contextmanager
 
-from pex.common import safe_mkdir, touch
+from pex.common import touch
+from pex.fs import lock
+from pex.fs.lock import FileLockStyle
+from pex.os import WINDOWS
 from pex.typing import TYPE_CHECKING
 from pex.variables import ENV
 
@@ -64,21 +66,14 @@ def _lock(exclusive):
         existing_exclusive, lock_fd, existing_lock_file = _LOCK
         if existing_exclusive == exclusive:
             return existing_lock_file
+        elif WINDOWS:
+            # Windows shared locks are not re-entrant; so this is the best we can do.
+            lock.release(lock_fd)
 
     lock_file = os.path.join(ENV.PEX_ROOT, "access.lck")
 
-    if lock_fd is None:
-        # N.B.: We don't actually write anything to the lock file but the fcntl file locking
-        # operations only work on files opened for at least write.
-        safe_mkdir(os.path.dirname(lock_file))
-        lock_fd = os.open(lock_file, os.O_CREAT | os.O_WRONLY)
-
-    # N.B.: Since flock operates on an open file descriptor and these are
-    # guaranteed to be closed by the operating system when the owning process exits,
-    # this lock is immune to staleness.
-    fcntl.flock(lock_fd, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH)
-
-    _LOCK = exclusive, lock_fd, lock_file
+    file_lock = lock.acquire(lock_file, exclusive=exclusive, style=FileLockStyle.BSD, fd=lock_fd)
+    _LOCK = exclusive, file_lock.fd, lock_file
     return lock_file
 
 

pex/cache/dirs.py

@@ -10,8 +10,8 @@
 from pex.compatibility import commonpath
 from pex.enum import Enum
 from pex.exceptions import production_assert
-from pex.executables import is_exe
 from pex.orderedset import OrderedSet
+from pex.os import is_exe
 from pex.typing import TYPE_CHECKING, cast
 from pex.variables import ENV, Variables
 

pex/cli/commands/lock.py

@@ -30,9 +30,9 @@
 )
 from pex.enum import Enum
 from pex.exceptions import production_assert
-from pex.executables import is_exe
 from pex.interpreter import PythonInterpreter
 from pex.orderedset import OrderedSet
+from pex.os import is_exe, safe_execv
 from pex.pep_376 import InstalledWheel, Record
 from pex.pep_427 import InstallableType
 from pex.pep_440 import Version
@@ -344,7 +344,7 @@ def sync(
 
         if self.command:
             try:
-                os.execv(self.command[0], self.command)
+                safe_execv(self.command)
             except OSError as e:
                 return Error("Failed to execute {exe}: {err}".format(exe=self.command[0], err=e))
 

pex/common.py

@@ -24,6 +24,7 @@
 
 from pex.enum import Enum
 from pex.executables import chmod_plus_x
+from pex.fs import safe_link, safe_rename, safe_symlink
 from pex.typing import TYPE_CHECKING, cast
 
 if TYPE_CHECKING:
@@ -119,13 +120,13 @@ def do_copy():
         # type: () -> None
         temp_dest = dest + uuid4().hex
         shutil.copy(source, temp_dest)
-        os.rename(temp_dest, dest)
+        safe_rename(temp_dest, dest)
 
     # If the platform supports hard-linking, use that and fall back to copying.
     # Windows does not support hard-linking.
     if hasattr(os, "link"):
         try:
-            os.link(source, dest)
+            safe_link(source, dest)
         except OSError as e:
             if e.errno == errno.EEXIST:
                 # File already exists.  If overwrite=True, write otherwise skip.
@@ -618,7 +619,7 @@ def symlink(
         self._ensure_parent(dst)
         abs_src = os.path.realpath(src)
         abs_dst = os.path.realpath(os.path.join(self.chroot, dst))
-        os.symlink(os.path.relpath(abs_src, os.path.dirname(abs_dst)), abs_dst)
+        safe_symlink(os.path.relpath(abs_src, os.path.dirname(abs_dst)), abs_dst)
 
     def write(
         self,
@@ -779,7 +780,7 @@ def relative_symlink(
     """
     dst_parent = os.path.dirname(dst)
     rel_src = os.path.relpath(src, dst_parent)
-    os.symlink(rel_src, dst)
+    safe_symlink(rel_src, dst)
 
 
 class CopyMode(Enum["CopyMode.Value"]):
@@ -839,7 +840,7 @@ def iter_copytree(
                     # later go missing leaving the dst_entry dangling.
                     if link and not os.path.islink(src_entry):
                         try:
-                            os.link(src_entry, dst_entry)
+                            safe_link(src_entry, dst_entry)
                             continue
                         except OSError as e:
                             if e.errno != errno.EXDEV:

pex/compatibility.py

@@ -178,8 +178,6 @@ def cpu_count():
 
     from Queue import Queue as Queue
 
-WINDOWS = os.name == "nt"
-
 
 # Universal newlines is the default in Python 3.
 MODE_READ_UNIVERSAL_NEWLINES = "rU" if PY2 else "r"

pex/executables.py

@@ -8,6 +8,7 @@
 import stat
 from textwrap import dedent
 
+from pex.os import is_exe
 from pex.typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
@@ -28,16 +29,6 @@ def chmod_plus_x(path):
     os.chmod(path, path_mode)
 
 
-def is_exe(path):
-    # type: (Text) -> bool
-    """Determines if the given path is a file executable by the current user.
-
-    :param path: The path to check.
-    :return: True if the given path is a file executable by the current user.
-    """
-    return os.path.isfile(path) and os.access(path, os.R_OK | os.X_OK)
-
-
 def is_script(
     path,  # type: Text
     pattern=None,  # type: Optional[bytes]

pex/finders.py

@@ -17,6 +17,7 @@
 from pex.executables import is_python_script
 from pex.pep_376 import InstalledWheel
 from pex.pep_503 import ProjectName
+from pex.sysconfig import SCRIPT_DIR, script_name
 from pex.typing import TYPE_CHECKING, cast
 from pex.wheel import Wheel
 
@@ -38,14 +39,16 @@ def find(
     ):
         # type: (...) -> Optional[DistributionScript]
         if dist.type is DistributionType.WHEEL:
-            script_path = Wheel.load(dist.location).data_path("scripts", name)
+            script_path = Wheel.load(dist.location).data_path("scripts", script_name(name))
             with open_zip(dist.location) as zfp:
                 try:
                     zfp.getinfo(script_path)
                 except KeyError:
                     return None
         elif dist.type is DistributionType.INSTALLED:
-            script_path = InstalledWheel.load(dist.location).stashed_path("bin", name)
+            script_path = InstalledWheel.load(dist.location).stashed_path(
+                SCRIPT_DIR, script_name(name)
+            )
             if not os.path.isfile(script_path):
                 return None
         else: