feat: add PyPy macOS arm64 (#1372)

* feat: add PyPy macOS arm64
* fix: use PyPy 7.3.11
* fix: filter-out unsupported PyPy cross-compilation configs
* Add quiet notice about the auto-skipping of arm64 pp builds on x86_64

Signed-off-by: Henry Schreiner <[email protected]>
Co-authored-by: Henry Schreiner <[email protected]>
Co-authored-by: Joe Rickerby <[email protected]>

Author: 3 people

README.md

@@ -32,12 +32,13 @@ What does it do?
 | CPython 3.10  | ✅ | ✅  | ✅  | ✅  | ✅² | ✅ | ✅  | ✅ | ✅  | ✅  |
 | CPython 3.11  | ✅ | ✅  | ✅  | ✅  | ✅² | ✅ | ✅  | ✅ | ✅  | ✅  |
 | PyPy 3.7 v7.3 | ✅ | N/A | ✅  | N/A | N/A | ✅¹ | ✅¹  | ✅¹ | N/A | N/A |
-| PyPy 3.8 v7.3 | ✅ | N/A | ✅  | N/A | N/A | ✅¹ | ✅¹  | ✅¹ | N/A | N/A |
-| PyPy 3.9 v7.3 | ✅ | N/A | ✅  | N/A | N/A | ✅¹ | ✅¹  | ✅¹ | N/A | N/A |
+| PyPy 3.8 v7.3 | ✅ | ✅⁴ | ✅  | N/A | N/A | ✅¹ | ✅¹  | ✅¹ | N/A | N/A |
+| PyPy 3.9 v7.3 | ✅ | ✅⁴ | ✅  | N/A | N/A | ✅¹ | ✅¹  | ✅¹ | N/A | N/A |
 
 <sup>¹ PyPy is only supported for manylinux wheels.</sup><br>
 <sup>² Windows arm64 support is experimental.</sup><br>
 <sup>³ Alpine 3.14 and very briefly 3.15's default python3 [was not able to load](https://github.com/pypa/cibuildwheel/issues/934) musllinux wheels. This has been fixed; please upgrade the python package if using Alpine from before the fix.</sup><br>
+<sup>⁴ Cross-compilation not supported with PyPy - to build these wheels you need to run cibuildwheel on an Apple Silicon machine.</sup><br>
 
 - Builds manylinux, musllinux, macOS 10.9+, and Windows wheels for CPython and PyPy
 - Works on GitHub Actions, Azure Pipelines, Travis CI, AppVeyor, CircleCI, GitLab CI, and Cirrus CI

bin/update_pythons.py

@@ -159,8 +159,8 @@ def update_version_windows(self, spec: Specifier) -> ConfigWinCP:
         )
 
     def update_version_macos(self, spec: Specifier) -> ConfigMacOS:
-        if self.arch != "64":
-            msg = "Other archs not supported yet on macOS"
+        if self.arch not in {"64", "ARM64"}:
+            msg = f"'{self.arch}' arch not supported yet on macOS"
             raise RuntimeError(msg)
 
         releases = [r for r in self.releases if spec.contains(r["python_version"])]
@@ -172,12 +172,14 @@ def update_version_macos(self, spec: Specifier) -> ConfigMacOS:
 
         release = releases[-1]
         version = release["python_version"]
-        identifier = f"pp{version.major}{version.minor}-macosx_x86_64"
+        arch = "x86_64" if self.arch == "64" else self.arch.lower()
+        identifier = f"pp{version.major}{version.minor}-macosx_{arch}"
 
+        arch = "x64" if self.arch == "64" else self.arch.lower()
         (url,) = (
             rf["download_url"]
             for rf in release["files"]
-            if "" in rf["platform"] == "darwin" and rf["arch"] == "x64"
+            if "" in rf["platform"] == "darwin" and rf["arch"] == arch
         )
 
         return ConfigMacOS(
@@ -251,6 +253,7 @@ def __init__(self) -> None:
 
         self.macos_cpython = CPythonVersions()
         self.macos_pypy = PyPyVersions("64")
+        self.macos_pypy_arm64 = PyPyVersions("ARM64")
 
     def update_config(self, config: dict[str, str]) -> None:
         identifier = config["identifier"]
@@ -261,11 +264,14 @@ def update_config(self, config: dict[str, str]) -> None:
         config_update: AnyConfig | None = None
 
         # We need to use ** in update due to MyPy (probably a bug)
-        if "macos" in identifier:
+        if "macosx" in identifier:
             if identifier.startswith("cp"):
                 config_update = self.macos_cpython.update_version_macos(identifier, version, spec)
             elif identifier.startswith("pp"):
-                config_update = self.macos_pypy.update_version_macos(spec)
+                if "macosx_x86_64" in identifier:
+                    config_update = self.macos_pypy.update_version_macos(spec)
+                elif "macosx_arm64" in identifier:
+                    config_update = self.macos_pypy_arm64.update_version_macos(spec)
         elif "win32" in identifier:
             if identifier.startswith("cp"):
                 config_update = self.windows_32.update_version_windows(spec)

cibuildwheel/logger.py

@@ -126,6 +126,10 @@ def step_end_with_error(self, error: BaseException | str) -> None:
         self.step_end(success=False)
         self.error(error)
 
+    def quiet(self, message: str) -> None:
+        c = self.colors
+        print(f"{c.gray}{message}{c.end}", file=sys.stderr)
+
     def notice(self, message: str) -> None:
         if self.fold_mode == "github":
             print(f"::notice::{message}\n", file=sys.stderr)

cibuildwheel/macos.py

@@ -84,7 +84,31 @@ def get_python_configurations(
     ]
 
     # skip builds as required by BUILD/SKIP
-    return [c for c in python_configurations if build_selector(c.identifier)]
+    python_configurations = [c for c in python_configurations if build_selector(c.identifier)]
+
+    # filter-out some cross-compilation configs with PyPy:
+    # can't build arm64 on x86_64
+    # rosetta allows to build x86_64 on arm64
+    if platform.machine() == "x86_64":
+        python_configurations_before = set(python_configurations)
+        python_configurations = [
+            c
+            for c in python_configurations
+            if not (c.identifier.startswith("pp") and c.identifier.endswith("arm64"))
+        ]
+        removed_elements = python_configurations_before - set(python_configurations)
+        if removed_elements:
+            ids = ", ".join(c.identifier for c in removed_elements)
+            log.quiet(
+                unwrap(
+                    f"""
+                    Note: {ids}  {'was' if len(removed_elements) == 1 else 'were'}
+                    selected, but can't be built on x86_64 so will be skipped automatically.
+                    """
+                )
+            )
+
+    return python_configurations
 
 
 def install_cpython(tmp: Path, version: str, url: str) -> Path:

cibuildwheel/resources/build-platforms.toml

@@ -89,7 +89,9 @@ python_configurations = [
   { identifier = "cp311-macosx_universal2", version = "3.11", url = "https://www.python.org/ftp/python/3.11.1/python-3.11.1-macos11.pkg" },
   { identifier = "pp37-macosx_x86_64", version = "3.7", url = "https://downloads.python.org/pypy/pypy3.7-v7.3.9-osx64.tar.bz2" },
   { identifier = "pp38-macosx_x86_64", version = "3.8", url = "https://downloads.python.org/pypy/pypy3.8-v7.3.11-macos_x86_64.tar.bz2" },
+  { identifier = "pp38-macosx_arm64", version = "3.8", url = "https://downloads.python.org/pypy/pypy3.8-v7.3.11-macos_arm64.tar.bz2" },
   { identifier = "pp39-macosx_x86_64", version = "3.9", url = "https://downloads.python.org/pypy/pypy3.9-v7.3.11-macos_x86_64.tar.bz2" },
+  { identifier = "pp39-macosx_arm64", version = "3.9", url = "https://downloads.python.org/pypy/pypy3.9-v7.3.11-macos_arm64.tar.bz2" },
 ]
 
 [windows]

docs/options.md

@@ -227,8 +227,8 @@ When setting the options, you can use shell-style globbing syntax, as per [fnmat
 | Python 3.10  | cp310-macosx_x86_64<br/>cp310-macosx_universal2<br/>cp310-macosx_arm64 | cp310-win_amd64<br/>cp310-win32<br/>cp310-win_arm64 | cp310-manylinux_x86_64<br/>cp310-manylinux_i686<br/>cp310-musllinux_x86_64<br/>cp310-musllinux_i686 | cp310-manylinux_aarch64<br/>cp310-manylinux_ppc64le<br/>cp310-manylinux_s390x<br/>cp310-musllinux_aarch64<br/>cp310-musllinux_ppc64le<br/>cp310-musllinux_s390x |
 | Python 3.11  | cp311-macosx_x86_64<br/>cp311-macosx_universal2<br/>cp311-macosx_arm64 | cp311-win_amd64<br/>cp311-win32<br/>cp311-win_arm64 | cp311-manylinux_x86_64<br/>cp311-manylinux_i686<br/>cp311-musllinux_x86_64<br/>cp311-musllinux_i686 | cp311-manylinux_aarch64<br/>cp311-manylinux_ppc64le<br/>cp311-manylinux_s390x<br/>cp311-musllinux_aarch64<br/>cp311-musllinux_ppc64le<br/>cp311-musllinux_s390x |
 | PyPy3.7 v7.3 | pp37-macosx_x86_64                                                     | pp37-win_amd64                                      | pp37-manylinux_x86_64<br/>pp37-manylinux_i686                                                       | pp37-manylinux_aarch64                                                                                                                                          |
-| PyPy3.8 v7.3 | pp38-macosx_x86_64                                                     | pp38-win_amd64                                      | pp38-manylinux_x86_64<br/>pp38-manylinux_i686                                                       | pp38-manylinux_aarch64                                                                                                                                          |
-| PyPy3.9 v7.3 | pp39-macosx_x86_64                                                     | pp39-win_amd64                                      | pp39-manylinux_x86_64<br/>pp39-manylinux_i686                                                       | pp39-manylinux_aarch64                                                                                                                                          |
+| PyPy3.8 v7.3 | pp38-macosx_x86_64<br/>pp38-macosx_arm64                               | pp38-win_amd64                                      | pp38-manylinux_x86_64<br/>pp38-manylinux_i686                                                       | pp38-manylinux_aarch64                                                                                                                                          |
+| PyPy3.9 v7.3 | pp39-macosx_x86_64<br/>pp39-macosx_arm64                               | pp39-win_amd64                                      | pp39-manylinux_x86_64<br/>pp39-manylinux_i686                                                       | pp39-manylinux_aarch64                                                                                                                                          |
 
 The list of supported and currently selected build identifiers can also be retrieved by passing the `--print-build-identifiers` flag to cibuildwheel.
 The format is `python_tag-platform_tag`, with tags similar to those in [PEP 425](https://www.python.org/dev/peps/pep-0425/#details).

test/test_macos_archs.py

@@ -65,7 +65,7 @@ def test_cross_compiled_test(tmp_path, capfd, build_universal2):
     actual_wheels = utils.cibuildwheel_run(
         project_dir,
         add_env={
-            "CIBW_BUILD": "cp39-*",
+            "CIBW_BUILD": "cp39-*" if build_universal2 else "*p39-*",
             "CIBW_TEST_COMMAND": '''python -c "import platform; print('running tests on ' + platform.machine())"''',
             "CIBW_ARCHS": "universal2" if build_universal2 else "x86_64 arm64",
             "CIBW_BUILD_VERBOSITY": "3",
@@ -76,7 +76,8 @@ def test_cross_compiled_test(tmp_path, capfd, build_universal2):
 
     assert DEPLOYMENT_TARGET_TOO_LOW_WARNING not in captured.err
 
-    if platform.machine() == "x86_64":
+    platform_machine = platform.machine()
+    if platform_machine == "x86_64":
         # ensure that tests were run on only x86_64
         assert "running tests on x86_64" in captured.out
         assert "running tests on arm64" not in captured.out
@@ -89,15 +90,24 @@ def test_cross_compiled_test(tmp_path, capfd, build_universal2):
             assert (
                 "While arm64 wheels can be built on x86_64, they cannot be tested" in captured.err
             )
-    elif platform.machine() == "arm64":
+    elif platform_machine == "arm64":
         # ensure that tests were run on both x86_64 and arm64
         assert "running tests on x86_64" in captured.out
         assert "running tests on arm64" in captured.out
+        assert (
+            "While universal2 wheels can be built on x86_64, the arm64 part of them cannot currently be tested"
+            not in captured.err
+        )
+        assert (
+            "While arm64 wheels can be built on x86_64, they cannot be tested" not in captured.err
+        )
 
     if build_universal2:
         expected_wheels = [w for w in ALL_MACOS_WHEELS if "cp39" in w and "universal2" in w]
     else:
-        expected_wheels = [w for w in ALL_MACOS_WHEELS if "cp39" in w and "universal2" not in w]
+        expected_wheels = [w for w in ALL_MACOS_WHEELS if "p39-" in w and "universal2" not in w]
+        if platform_machine == "x86_64":
+            expected_wheels = [w for w in expected_wheels if not ("pp39" in w and "arm64" in w)]
 
     assert set(actual_wheels) == set(expected_wheels)
 

test/utils.py

@@ -177,7 +177,14 @@ def expected_wheels(
 
         if platform == "macos" and machine_arch == "arm64":
             # arm64 macs are only supported by cp38+
-            python_abi_tags = ["cp38-cp38", "cp39-cp39", "cp310-cp310", "cp311-cp311"]
+            python_abi_tags = [
+                "cp38-cp38",
+                "cp39-cp39",
+                "cp310-cp310",
+                "cp311-cp311",
+                "pp38-pypy38_pp73",
+                "pp39-pypy39_pp73",
+            ]
 
     wheels = []