feat: configuration support (#684)

* feat: configuration support * refactor: include manylinux, minor redesign * "project global" overrides "default platform" This allows overriding `repair-wheel-command` in the `[tool.cibuildwheel.global]` section. * fix: add PyPy images * refactor: remove global * fix: support tables and arrays * docs: add pyproject.toml config to docs * docs: updates based on feedback * fix: only join if sensible * refactor: remove manylinux dict * docs: examples headers and sections on each * feat: support changing the config from the command line * fix: use {package} * tests: add a few tests and mention config one more place * Restyle examples tabs to remove indent * Unrelated docs improvements * Absorb extra content into the tab * Make the tabs smaller and to the right * Copy edits to options * Fix header ids * Use fewer [tool.cibuildwheel] headers * docs: minor additions/fixes * feat: disallow options on some platforms * docs: make the examples a tiny bit more tabby * fix: enforce tables/lists are optionally separate * Some unrelated fixes to the other tab styling * Minor fix to tab styling * Refactor to make code a bit more linear and immutable * Minor docs changes Co-authored-by: mayeut <[email protected]> Co-authored-by: Joe Rickerby <[email protected]>
pypa · Jun 21, 2021 · f62cbc3 · f62cbc3
1 parent f7bdccf
commit f62cbc3
Show file tree

Hide file tree

Showing 9 changed files with 1,114 additions and 286 deletions.
diff --git a/README.md b/README.md
@@ -123,6 +123,7 @@ Options
 |   | [`CIBW_TEST_SKIP`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-skip)  | Skip running tests on some builds |
 | **Other** | [`CIBW_BUILD_VERBOSITY`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-verbosity)  | Increase/decrease the output of pip wheel |
 
+These options can be specified in a pyproject.toml file, as well; see [configuration](https://cibuildwheel.readthedocs.io/en/stable/options/#configuration).
 
 Working examples
 ----------------

diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py
@@ -5,7 +5,7 @@
 import traceback
 from configparser import ConfigParser
 from pathlib import Path
-from typing import Dict, List, Optional, Set, Union, overload
+from typing import Dict, List, Optional, Set, Union
 
 from packaging.specifiers import SpecifierSet
 
@@ -16,6 +16,7 @@
 import cibuildwheel.windows
 from cibuildwheel.architecture import Architecture, allowed_architectures_check
 from cibuildwheel.environment import EnvironmentParseError, parse_environment
+from cibuildwheel.options import ConfigOptions
 from cibuildwheel.projectfiles import get_requires_python_str
 from cibuildwheel.typing import PLATFORMS, PlatformName, assert_never
 from cibuildwheel.util import (
@@ -28,39 +29,16 @@
     resources_dir,
 )
 
-
-@overload
-def get_option_from_environment(
-    option_name: str, *, platform: Optional[str] = None, default: str
-) -> str:
-    ...  # noqa: E704
-
-
-@overload
-def get_option_from_environment(
-    option_name: str, *, platform: Optional[str] = None, default: None = None
-) -> Optional[str]:
-    ...  # noqa: E704 E302
-
-
-def get_option_from_environment(
-    option_name: str, *, platform: Optional[str] = None, default: Optional[str] = None
-) -> Optional[str]:  # noqa: E302
-    """
-    Returns an option from the environment, optionally scoped by the platform.
-
-    Example:
-      get_option_from_environment('CIBW_COLOR', platform='macos')
-
-      This will return the value of CIBW_COLOR_MACOS if it exists, otherwise the value of
-      CIBW_COLOR.
-    """
-    if platform:
-        option = os.environ.get(f"{option_name}_{platform.upper()}")
-        if option is not None:
-            return option
-
-    return os.environ.get(option_name, default)
+MANYLINUX_ARCHS = (
+    "x86_64",
+    "i686",
+    "pypy_x86_64",
+    "aarch64",
+    "ppc64le",
+    "s390x",
+    "pypy_aarch64",
+    "pypy_i686",
+)
 
 
 def main() -> None:
@@ -69,8 +47,9 @@ def main() -> None:
     parser = argparse.ArgumentParser(
         description="Build wheels for all the platforms.",
         epilog="""
-            Most options are supplied via environment variables.
-            See https://github.com/pypa/cibuildwheel#options for info.
+            Most options are supplied via environment variables or in
+            --config-file (pyproject.toml usually). See
+            https://github.com/pypa/cibuildwheel#options for info.
         """,
     )
 
@@ -104,10 +83,17 @@ def main() -> None:
 
     parser.add_argument(
         "--output-dir",
-        default=os.environ.get("CIBW_OUTPUT_DIR", "wheelhouse"),
         help="Destination folder for the wheels.",
     )
 
+    parser.add_argument(
+        "--config-file",
+        help="""
+            TOML config file for cibuildwheel. Defaults to pyproject.toml, but
+            can be overridden with this option.
+        """,
+    )
+
     parser.add_argument(
         "package_dir",
         default=".",
@@ -176,40 +162,40 @@ def main() -> None:
         sys.exit(2)
 
     package_dir = Path(args.package_dir)
-    output_dir = Path(args.output_dir)
 
-    if platform == "linux":
-        repair_command_default = "auditwheel repair -w {dest_dir} {wheel}"
-    elif platform == "macos":
-        repair_command_default = "delocate-listdeps {wheel} && delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}"
-    elif platform == "windows":
-        repair_command_default = ""
-    else:
-        assert_never(platform)
-
-    build_config = os.environ.get("CIBW_BUILD") or "*"
-    skip_config = os.environ.get("CIBW_SKIP", "")
-    test_skip = os.environ.get("CIBW_TEST_SKIP", "")
-    environment_config = get_option_from_environment(
-        "CIBW_ENVIRONMENT", platform=platform, default=""
-    )
-    before_all = get_option_from_environment("CIBW_BEFORE_ALL", platform=platform, default="")
-    before_build = get_option_from_environment("CIBW_BEFORE_BUILD", platform=platform)
-    repair_command = get_option_from_environment(
-        "CIBW_REPAIR_WHEEL_COMMAND", platform=platform, default=repair_command_default
-    )
-    dependency_versions = get_option_from_environment(
-        "CIBW_DEPENDENCY_VERSIONS", platform=platform, default="pinned"
-    )
-    test_command = get_option_from_environment("CIBW_TEST_COMMAND", platform=platform)
-    before_test = get_option_from_environment("CIBW_BEFORE_TEST", platform=platform)
-    test_requires = get_option_from_environment(
-        "CIBW_TEST_REQUIRES", platform=platform, default=""
-    ).split()
-    test_extras = get_option_from_environment("CIBW_TEST_EXTRAS", platform=platform, default="")
-    build_verbosity_str = get_option_from_environment(
-        "CIBW_BUILD_VERBOSITY", platform=platform, default=""
+    manylinux_identifiers = {
+        f"manylinux-{build_platform}-image" for build_platform in MANYLINUX_ARCHS
+    }
+    disallow = {
+        "linux": {"dependency-versions"},
+        "macos": manylinux_identifiers,
+        "windows": manylinux_identifiers,
+    }
+    options = ConfigOptions(package_dir, args.config_file, platform=platform, disallow=disallow)
+    output_dir = Path(
+        args.output_dir
+        if args.output_dir is not None
+        else os.environ.get("CIBW_OUTPUT_DIR", "wheelhouse")
     )
+
+    build_config = options("build", env_plat=False, sep=" ") or "*"
+    skip_config = options("skip", env_plat=False, sep=" ")
+    test_skip = options("test-skip", env_plat=False, sep=" ")
+
+    archs_config_str = args.archs or options("archs", sep=" ")
+
+    environment_config = options("environment", table={"item": '{k}="{v}"', "sep": " "})
+    before_all = options("before-all", sep=" && ")
+    before_build = options("before-build", sep=" && ")
+    repair_command = options("repair-wheel-command", sep=" && ")
+
+    dependency_versions = options("dependency-versions")
+    test_command = options("test-command", sep=" && ")
+    before_test = options("before-test", sep=" && ")
+    test_requires = options("test-requires", sep=" ").split()
+    test_extras = options("test-extras", sep=",")
+    build_verbosity_str = options("build-verbosity")
+
     prerelease_pythons = args.prerelease_pythons or cibuildwheel.util.strtobool(
         os.environ.get("CIBW_PRERELEASE_PYTHONS", "0")
     )
@@ -218,11 +204,11 @@ def main() -> None:
 
     if not any(package_dir.joinpath(name).exists() for name in package_files):
         names = ", ".join(sorted(package_files, reverse=True))
-        print(
-            f"cibuildwheel: Could not find any of {{{names}}} at root of package", file=sys.stderr
-        )
+        msg = f"cibuildwheel: Could not find any of {{{names}}} at root of package"
+        print(msg, file=sys.stderr)
         sys.exit(2)
 
+    # This is not supported in tool.cibuildwheel, as it comes from a standard location.
     # Passing this in as an environment variable will override pyproject.toml, setup.cfg, or setup.py
     requires_python_str: Optional[str] = os.environ.get(
         "CIBW_PROJECT_REQUIRES_PYTHON"
@@ -270,13 +256,6 @@ def main() -> None:
     # This needs to be passed on to the docker container in linux.py
     os.environ["CIBUILDWHEEL"] = "1"
 
-    if args.archs is not None:
-        archs_config_str = args.archs
-    else:
-        archs_config_str = get_option_from_environment(
-            "CIBW_ARCHS", platform=platform, default="auto"
-        )
-
     archs = Architecture.parse_config(archs_config_str, platform=platform)
 
     identifiers = get_build_identifiers(platform, build_selector, archs)
@@ -286,7 +265,7 @@ def main() -> None:
             print(identifier)
         sys.exit(0)
 
-    manylinux_images: Optional[Dict[str, str]] = None
+    manylinux_images: Dict[str, str] = {}
     if platform == "linux":
         pinned_docker_images_file = resources_dir / "pinned_docker_images.cfg"
         all_pinned_docker_images = ConfigParser()
@@ -297,22 +276,10 @@ def main() -> None:
         #   'pypy_x86_64': {'manylinux2010': '...' }
         #   ... }
 
-        manylinux_images = {}
-
-        for build_platform in [
-            "x86_64",
-            "i686",
-            "pypy_x86_64",
-            "aarch64",
-            "ppc64le",
-            "s390x",
-            "pypy_aarch64",
-            "pypy_i686",
-        ]:
+        for build_platform in MANYLINUX_ARCHS:
             pinned_images = all_pinned_docker_images[build_platform]
 
-            config_name = f"CIBW_MANYLINUX_{build_platform.upper()}_IMAGE"
-            config_value = os.environ.get(config_name)
+            config_value = options(f"manylinux-{build_platform}-image")
 
             if config_value is None:
                 # default to manylinux2010 if it's available, otherwise manylinux2014
@@ -340,7 +307,7 @@ def main() -> None:
         repair_command=repair_command,
         environment=environment,
         dependency_constraints=dependency_constraints,
-        manylinux_images=manylinux_images,
+        manylinux_images=manylinux_images or None,
     )
 
     # Python is buffering by default when running on the CI platforms, giving problems interleaving subprocess call output with unflushed calls to 'print'