diff --git a/.github/actions/download.sh b/.github/actions/download.sh index ea16c2784c9..b9eb706ff26 100755 --- a/.github/actions/download.sh +++ b/.github/actions/download.sh @@ -33,7 +33,7 @@ if [ -z "${build_packages}" ]; then else for package in ${build_packages}; do echo "===> Download wheels: ${package}" - make -C spk/${package} wheeldownload + make -C spk/${package} download-wheels done fi diff --git a/mk/spksrc.common.mk b/mk/spksrc.common.mk index 27b27744445..abbb8d8cd85 100644 --- a/mk/spksrc.common.mk +++ b/mk/spksrc.common.mk @@ -32,6 +32,9 @@ PIP ?= pip # System default pip outside from build environment PIP_SYSTEM = $(shell which pip) +# System default pip outside from build environment +PIP_NATIVE = $(WORK_DIR)/../../../native/$(or $(PYTHON_PACKAGE),$(SPK_NAME))/work-native/install/usr/local/bin/pip + # Why ask for the same thing twice? Always cache downloads PIP_CACHE_OPT ?= --find-links $(PIP_DISTRIB_DIR) --cache-dir $(PIP_CACHE_DIR) PIP_WHEEL_ARGS = wheel --disable-pip-version-check --no-binary :all: $(PIP_CACHE_OPT) --no-deps --wheel-dir $(WHEELHOUSE) diff --git a/mk/spksrc.crossenv.mk b/mk/spksrc.crossenv.mk index 900cde068ec..fb146a32c69 100644 --- a/mk/spksrc.crossenv.mk +++ b/mk/spksrc.crossenv.mk @@ -13,7 +13,8 @@ # build_crossenv_target (override with CROSSENV_TARGET) # post_crossenv_target (override with POST_CROSSENV_TARGET) # Variables: -# WHEELS List of wheels to go through +# WHEEL_NAME Name of wheel to process +# WHEEL_VERSION Version of wheel to process (can be empty) # Defined using PYTHON_PACKAGE_WORK_DIR from spksrc.python.mk or use local work directory PYTHON_WORK_DIR = $(or $(wildcard $(PYTHON_PACKAGE_WORK_DIR)),$(wildcard $(WORK_DIR))) @@ -55,13 +56,13 @@ endif ### -# Check for wheel==x.y, then fallback to wheel, then default -ifneq ($(wildcard $(CROSSENV_CONFIG_PATH)/requirements-$(WHEEL).txt),) -CROSSENV_BUILD_WHEEL = $(WHEEL) -CROSSENV_BUILD_REQUIREMENTS = $(CROSSENV_CONFIG_PATH)/requirements-$(CROSSENV_BUILD_WHEEL).txt -else ifneq ($(wildcard $(CROSSENV_CONFIG_PATH)/requirements-$(firstword $(subst -, ,$(WHEEL))).txt),) -CROSSENV_BUILD_WHEEL = $(firstword $(subst -, ,$(WHEEL))) +# Check for -, then fallback to , then default +ifneq ($(wildcard $(CROSSENV_CONFIG_PATH)/requirements-$(WHEEL_NAME)-$(WHEEL_VERSION).txt),) +CROSSENV_BUILD_WHEEL = $(WHEEL_NAME)-$(WHEEL_VERSION) CROSSENV_BUILD_REQUIREMENTS = $(CROSSENV_CONFIG_PATH)/requirements-$(CROSSENV_BUILD_WHEEL).txt +else ifneq ($(wildcard $(CROSSENV_CONFIG_PATH)/requirements-$(WHEEL_NAME).txt),) +CROSSENV_BUILD_WHEEL = $(WHEEL_NAME) +CROSSENV_BUILD_REQUIREMENTS = $(CROSSENV_CONFIG_PATH)/requirements-$(WHEEL_NAME).txt else CROSSENV_BUILD_WHEEL = default CROSSENV_BUILD_REQUIREMENTS = $(CROSSENV_CONFIG_DEFAULT) @@ -73,10 +74,10 @@ CROSSENV_COOKIE = $(WORK_DIR)/.crossenv-$(CROSSENV_BUILD_WHEEL)_done ### # default wheel packages to install in crossenv -ifneq ($(wildcard $(CROSSENV_CONFIG_PATH)/requirements-$(WHEEL).txt $(CROSSENV_CONFIG_DEFAULT)),) -CROSSENV_DEFAULT_PIP_VERSION = $(shell grep -h -E "^pip[<>=]=" $(wildcard $(CROSSENV_CONFIG_PATH)/requirements-$(WHEEL).txt $(CROSSENV_CONFIG_DEFAULT)) | head -1 | sed -E 's/.*[<>=]=//') -CROSSENV_DEFAULT_SETUPTOOLS_VERSION = $(shell grep -h -E "^setuptools[<>=]=" $(wildcard $(CROSSENV_CONFIG_PATH)/requirements-$(WHEEL).txt $(CROSSENV_CONFIG_DEFAULT)) | head -1 | sed -E 's/.*[<>=]=//') -CROSSENV_DEFAULT_WHEEL_VERSION = $(shell grep -h -E "^wheel[<>=]=" $(wildcard $(CROSSENV_CONFIG_PATH)/requirements-$(WHEEL).txt $(CROSSENV_CONFIG_DEFAULT)) | head -1 | sed -E 's/.*[<>=]=//') +ifneq ($(wildcard $(CROSSENV_BUILD_REQUIREMENTS) $(CROSSENV_CONFIG_DEFAULT)),) +CROSSENV_DEFAULT_PIP_VERSION = $(shell grep -h -E "^pip[<>=]=" $(wildcard $(CROSSENV_BUILD_REQUIREMENTS) $(CROSSENV_CONFIG_DEFAULT)) | head -1 | sed -E 's/.*[<>=]=//') +CROSSENV_DEFAULT_SETUPTOOLS_VERSION = $(shell grep -h -E "^setuptools[<>=]=" $(wildcard $(CROSSENV_BUILD_REQUIREMENTS) $(CROSSENV_CONFIG_DEFAULT)) | head -1 | sed -E 's/.*[<>=]=//') +CROSSENV_DEFAULT_WHEEL_VERSION = $(shell grep -h -E "^wheel[<>=]=" $(wildcard $(CROSSENV_BUILD_REQUIREMENTS) $(CROSSENV_CONFIG_DEFAULT)) | head -1 | sed -E 's/.*[<>=]=//') endif ifneq ($(CROSSENV_DEFAULT_PIP_VERSION),) @@ -114,13 +115,8 @@ post_crossenv_target: $(CROSSENV_TARGET) ### crossenv-%: -ifneq ($(filter error-%, $(CROSSENV_BUILD_WHEEL)),) - @$(MSG) $(MAKE) $(CROSSENV_BUILD_WHEEL) - @$(MAKE) $(CROSSENV_BUILD_WHEEL) --no-print-directory -else - @$(MSG) $(MAKE) ARCH=$(firstword $(subst -, ,$*)) TCVERSION=$(lastword $(subst -, ,$*)) WHEEL=$(CROSSENV_BUILD_WHEEL) crossenv - -@MAKEFLAGS= $(MAKE) ARCH=$(firstword $(subst -, ,$*)) TCVERSION=$(lastword $(subst -, ,$*)) WHEEL=$(CROSSENV_BUILD_WHEEL) crossenv --no-print-directory -endif + @$(MSG) $(MAKE) ARCH=$(firstword $(subst -, ,$*)) TCVERSION=$(lastword $(subst -, ,$*)) WHEEL_NAME=$(WHEEL_NAME) WHEEL_VERSION=$(WHEEL_VERSION) crossenv + -@MAKEFLAGS= $(MAKE) ARCH=$(firstword $(subst -, ,$*)) TCVERSION=$(lastword $(subst -, ,$*)) WHEEL_NAME=$(WHEEL_NAME) WHEEL_VERSION=$(WHEEL_VERSION) crossenv --no-print-directory #### @@ -166,8 +162,8 @@ export PYTHONPATH = $(PYTHON_LIB_NATIVE):$(PYTHON_STAGING_INSTALL_PREFIX)/lib/py # >>> sys.path # build_crossenv_target: SHELL:=/bin/bash -build_crossenv_target: $(CROSSENV_PATH)/build/python-cc.mk - @$(MSG) $$(date +%Y%m%d-%H%M%S) MAKELEVEL: $(MAKELEVEL), PARALLEL_MAKE: $(PARALLEL_MAKE), ARCH: $(ARCH)-$(TCVERSION), CROSSENV: $(WHEEL) >> $(PSTAT_LOG) +build_crossenv_target: pre_crossenv_target $(CROSSENV_PATH)/build/python-cc.mk + @$(MSG) $$(date +%Y%m%d-%H%M%S) MAKELEVEL: $(MAKELEVEL), PARALLEL_MAKE: $(PARALLEL_MAKE), ARCH: $(ARCH)-$(TCVERSION), CROSSENV: $(CROSSENV_BUILD_WHEEL) >> $(PSTAT_LOG) @$(MSG) Python sources: $(wildcard $(PYTHON_WORK_DIR)/Python-[0-9]*) @$(MSG) crossenv wheel packages: $(CROSSENV_DEFAULT_PIP), $(CROSSENV_DEFAULT_SETUPTOOLS), $(CROSSENV_DEFAULT_WHEEL) @$(MSG) crossenv requirement definition: $(CROSSENV_BUILD_REQUIREMENTS) diff --git a/mk/spksrc.python-module.mk b/mk/spksrc.python-module.mk index 3f1467f5a88..85cfcdc834f 100644 --- a/mk/spksrc.python-module.mk +++ b/mk/spksrc.python-module.mk @@ -21,8 +21,8 @@ CROSSENV_MODULE_PATH = $(firstword $(wildcard $(WORK_DIR)/crossenv-$(PKG_NAME)-$ ### Prepare crossenv build_crossenv_module: - @$(MSG) WHEEL="$(PKG_NAME)-$(PKG_VERS)" $(MAKE) crossenv-$(ARCH)-$(TCVERSION) - -@MAKEFLAGS= WHEEL="$(PKG_NAME)-$(PKG_VERS)" $(MAKE) crossenv-$(ARCH)-$(TCVERSION) + @$(MSG) $(MAKE) WHEEL_NAME=\"$(PKG_VERS)\" WHEEL_VERSION=\"$(PKG_VERS)\" crossenv-$(ARCH)-$(TCVERSION) + -@MAKEFLAGS= $(MAKE) WHEEL_NAME="$(PKG_VERS)" WHEEL_VERSION="$(PKG_VERS)" crossenv-$(ARCH)-$(TCVERSION) ### Python extension module rules compile_python_module: build_crossenv_module diff --git a/mk/spksrc.python-wheel.mk b/mk/spksrc.python-wheel.mk index 27d29315a70..4bcd98e14d8 100644 --- a/mk/spksrc.python-wheel.mk +++ b/mk/spksrc.python-wheel.mk @@ -29,9 +29,6 @@ ifneq ($(wildcard $(PYTHON_STAGING_PREFIX)),) STAGING_INSTALL_PREFIX := $(PYTHON_STAGING_PREFIX) endif -## python wheel specific configurations -include ../../mk/spksrc.wheel-env.mk - ### Prepare crossenv build_crossenv_module: @$(MSG) WHEEL="$(PKG_NAME)-$(PKG_VERS)" $(MAKE) crossenv-$(ARCH)-$(TCVERSION) @@ -43,7 +40,8 @@ build_python_wheel_target: build_crossenv_module @. $(CROSSENV) ; \ $(MSG) _PYTHON_HOST_PLATFORM=$(TC_TARGET) cross-python3 -m build $(BUILD_ARGS) --wheel $(WHEELS_BUILD_ARGS) --outdir $(WHEELHOUSE) ; \ $(RUN) _PYTHON_HOST_PLATFORM=$(TC_TARGET) cross-python3 -m build $(BUILD_ARGS) --wheel $(WHEELS_BUILD_ARGS) --outdir $(WHEELHOUSE) - @$(RUN) echo "$(PKG_NAME)==$(PKG_VERS)" >> $(WHEELHOUSE)/$(WHEELS_CROSS_COMPILE) + @$(MSG) $(MAKE) REQUIREMENT=\"$(PKG_NAME)==$(PKG_VERS)\" WHEEL_NAME=\"$(PKG_NAME)\" WHEEL_VERSION=\"$(PKG_VERS)\" WHEEL_TYPE=\"cross\" wheel_install + -@MAKEFLAGS= $(MAKE) REQUIREMENT="$(PKG_NAME)==$(PKG_VERS)" WHEEL_NAME="$(PKG_NAME)" WHEEL_VERSION="$(PKG_VERS)" WHEEL_TYPE="cross" --no-print-directory wheel_install post_install_python_wheel_target: $(WHEEL_TARGET) install_python_wheel @@ -53,3 +51,9 @@ all: install # Use crossenv include ../../mk/spksrc.crossenv.mk + +## python wheel specific configurations +include ../../mk/spksrc.wheel-env.mk + +## install wheel specific routines +include ../../mk/spksrc.wheel-install.mk diff --git a/mk/spksrc.spk.mk b/mk/spksrc.spk.mk index 7cf7592a421..661589ab787 100644 --- a/mk/spksrc.spk.mk +++ b/mk/spksrc.spk.mk @@ -538,7 +538,8 @@ spkclean: work-*/WIZARD_UIFILES wheelclean: spkclean - rm -fr work-*/.wheel_done \ + rm -fr work*/.wheel_done \ + work*/.wheel_*_done \ work-*/wheelhouse \ work-*/install/var/packages/**/target/share/wheelhouse @make --no-print-directory dependency-flat | sort -u | grep cross/ | while read depend ; do \ @@ -552,10 +553,15 @@ wheelclean: spkclean fi ; \ done -wheelcleancache: wheelclean +wheelclean-%: spkclean + rm -f work-*/.wheel_done \ + work-*/wheelhouse/$*-*.whl + find work-* -type f -regex '.*\.wheel_\(download\|compile\|install\)-$*_done' -exec rm -f {} \; + +wheelcleancache: rm -fr work-*/pip -wheelcleanall: wheelcleancache +wheelcleanall: wheelcleancache wheelclean rm -fr ../../distrib/pip crossenvclean: wheelclean diff --git a/mk/spksrc.wheel-compile.mk b/mk/spksrc.wheel-compile.mk new file mode 100644 index 00000000000..7153a0b2b81 --- /dev/null +++ b/mk/spksrc.wheel-compile.mk @@ -0,0 +1,148 @@ +### Wheel rules +# Compile wheels for modules listed in WHEELS. +# +# Targets are executed in the following order: +# wheel_compile_msg_target +# pre_wheel_compile_target (override with PRE_WHEEL_COMPILE_TARGET) +# wheel_compile_target (override with WHEEL_COMPILE_TARGET) +# post_wheel_compile_target (override with POST_WHEEL_COMPILE_TARGET) +# Variables: +# REQUIREMENT Requirement formatted wheel information +# WHEEL_NAME Name of wheel to process +# WHEEL_VERSION Version of wheel to process (can be empty) +# WHEEL_TYPE Type of wheel to process (abi3, crossenv, pure) + +ifeq ($(WHEEL_VERSION),) +WHEEL_COMPILE_COOKIE = $(WORK_DIR)/.$(COOKIE_PREFIX)wheel_compile-$(WHEEL_NAME)_done +else +WHEEL_COMPILE_COOKIE = $(WORK_DIR)/.$(COOKIE_PREFIX)wheel_compile-$(WHEEL_NAME)-$(WHEEL_VERSION)_done +endif + +## + +ifeq ($(strip $(PRE_WHEEL_COMPILE_TARGET)),) +PRE_WHEEL_COMPILE_TARGET = pre_wheel_compile_target +else +$(PRE_WHEEL_COMPILE_TARGET): wheel_compile_msg_target +endif +ifeq ($(strip $(WHEEL_COMPILE_TARGET)),) +WHEEL_COMPILE_TARGET = wheel_compile_target +else +$(WHEEL_COMPILE_TARGET): $(BUILD_WHEEL_COMPILE_TARGET) +endif +ifeq ($(strip $(POST_WHEEL_COMPILE_TARGET)),) +POST_WHEEL_COMPILE_TARGET = post_wheel_compile_target +else +$(POST_WHEEL_COMPILE_TARGET): $(WHEEL_COMPILE_TARGET) +endif + +wheel_compile_msg_target: + @$(MSG) "Processing wheels of $(NAME)" + +pre_wheel_compile_target: wheel_compile_msg_target + +wheel_compile_target: SHELL:=/bin/bash +wheel_compile_target: +ifeq ($(wildcard $(WHEELHOUSE)),) + @$(MSG) Creating wheelhouse directory: $(WHEELHOUSE) + @mkdir -p $(WHEELHOUSE) +endif + @$(MSG) Compiling wheel [$(WHEEL_NAME)], version [$(WHEEL_VERSION)], type [$(WHEEL_TYPE)] +ifneq ($(WHEEL_TYPE),pure) + @$(MSG) $(MAKE) WHEEL_NAME=\"$(WHEEL_NAME)\" WHEEL_VERSION=\"$(WHEEL_VERSION)\" crossenv-$(ARCH)-$(TCVERSION) ; \ + MAKEFLAGS= $(MAKE) WHEEL_NAME="$(WHEEL_NAME)" WHEEL_VERSION="$(WHEEL_VERSION)" crossenv-$(ARCH)-$(TCVERSION) --no-print-directory + @[ "$(WHEEL_TYPE)" = "$(WHEELS_LIMITED_API)" ] && abi3="--build-option=--py-limited-api=$(PYTHON_LIMITED_API)" || abi3="" ; \ + global_options=$$(echo $(WHEELS_BUILD_ARGS) | sed -e 's/ \[/\n\[/g' | grep -i $(WHEEL_NAME) | cut -f2 -d] | xargs) ; \ + localCFLAGS=($$(echo $(WHEELS_CFLAGS) | sed -e 's/ \[/\n\[/g' | grep -i $(WHEEL_NAME) | cut -f2 -d] | xargs)) ; \ + localLDFLAGS=($$(echo $(WHEELS_LDFLAGS) | sed -e 's/ \[/\n\[/g' | grep -i $(WHEEL_NAME) | cut -f2 -d] | xargs)) ; \ + localCPPFLAGS=($$(echo $(WHEELS_CPPFLAGS) | sed -e 's/ \[/\n\[/g' | grep -i $(WHEEL_NAME) | cut -f2 -d] | xargs)) ; \ + localCXXFLAGS=($$(echo $(WHEELS_CXXFLAGS) | sed -e 's/ \[/\n\[/g' | grep -i $(WHEEL_NAME) | cut -f2 -d] | xargs)) ; \ + $(MSG) pip build [$(WHEEL_NAME)], version: [$(WHEEL_VERSION)] \ + $$([ "$$(echo $${localCFLAGS[@]})" ] && echo "CFLAGS=\"$${localCFLAGS[@]}\" ") \ + $$([ "$$(echo $${localCPPFLAGS[@]})" ] && echo "CPPFLAGS=\"$${localCPPFLAGS[@]}\" ") \ + $$([ "$$(echo $${localCXXFLAGS[@]})" ] && echo "CXXFLAGS=\"$${localCXXFLAGS[@]}\" ") \ + $$([ "$$(echo $${localLDFLAGS[@]})" ] && echo "LDFLAGS=\"$${localLDFLAGS[@]}\" ") \ + $$([ "$$(echo $${abi3})" ] && echo "$${abi3} ")" \ + $${global_options}" ; \ + REQUIREMENT=$(REQUIREMENT) \ + WHEEL_NAME=$(WHEEL_NAME) \ + WHEEL_VERSION=$(WHEEL_VERSION) \ + ADDITIONAL_CFLAGS="-I$(STAGING_INSTALL_PREFIX)/$(PYTHON_INC_DIR) $${localCFLAGS[@]}" \ + ADDITIONAL_CPPFLAGS="-I$(STAGING_INSTALL_PREFIX)/$(PYTHON_INC_DIR) $${localCPPFLAGS[@]}" \ + ADDITIONAL_CXXFLAGS="-I$(STAGING_INSTALL_PREFIX)/$(PYTHON_INC_DIR) $${localCXXFLAGS[@]}" \ + ADDITIONAL_LDFLAGS="$${localLDFLAGS[@]}" \ + ABI3="$${abi3}" \ + PIP_GLOBAL_OPTION="$${global_options}" \ + $(MAKE) --no-print-directory \ + cross-compile-wheel-$(WHEEL_NAME) || exit 1 +else ifneq ($(filter 1 ON TRUE,$(WHEELS_PURE_PYTHON_PACKAGING_ENABLE)),) + @if [ -s "$(WHEELHOUSE)/$(WHEELS_PURE_PYTHON)" ]; then \ + export LD= LDSHARED= CPP= NM= CC= AS= RANLIB= CXX= AR= STRIP= OBJDUMP= OBJCOPY= READELF= CFLAGS= CPPFLAGS= CXXFLAGS= LDFLAGS= && \ + $(RUN) \ + PATH="$(abspath $(WORK_DIR)/../../../native/$(PYTHON_PKG_NAME)/work-native/install/usr/local/bin):$(PATH)" \ + LD_LIBRARY_PATH="$(abspath $(WORK_DIR)/../../../native/$(PYTHON_PKG_NAME)/work-native/install/usr/local/lib):$(LD_LIBRARY_PATH)" \ + $(MSG) $(PIP_NATIVE) $(PIP_WHEEL_ARGS) $(REQUIREMENT) ; \ + $(PIP_NATIVE) $(PIP_WHEEL_ARGS) $(REQUIREMENT) ; \ + fi +endif + +## +## crossenv PATH environment requires a combination of: +## 1) unique PATH variable from $(ENV) -> using merge + dedup macros +## Note: Multiple declarations of ENV += PATH=bla creates confusion in its interpretation. +## Solution implemented fetches all PATH from ENV and combine them in reversed order. +## 2) access to maturin from native/python/.../bin -> ${PYTHON_NATIVE_PATH}/bin +## 3) access to crossenv/bin/cross* tools, mainly cross-pip -> ${CROSSENV_PATH}/bin +## +cross-compile-wheel-%: SHELL:=/bin/bash +cross-compile-wheel-%: + @for crossenv in $(WORK_DIR)/crossenv-$(WHEEL_NAME)-$(WHEEL_VERSION) $(WORK_DIR)/crossenv-$(WHEEL_NAME) $(WORK_DIR)/crossenv ; do \ + [ -d $${crossenv} ] && . $${crossenv}/build/python-cc.mk && break ; \ + done ; \ + if [ -d "$${CROSSENV_PATH}" ] ; then \ + PATH=$(call dedup, $(call merge, $(ENV), PATH, :), :):$${PYTHON_NATIVE_PATH}:$${CROSSENV_PATH}/bin:$${PATH} ; \ + $(MSG) "crossenv: [$${CROSSENV_PATH}]" ; \ + $(MSG) "pip: [$$(which cross-pip)]" ; \ + $(MSG) "maturin: [$$(which maturin)]" ; \ + else \ + echo "ERROR: crossenv not found!" ; \ + exit 2 ; \ + fi ; \ + if [ "$(PIP_GLOBAL_OPTION)" ]; then \ + pip_global_option=$$(echo $(PIP_GLOBAL_OPTION) | sed 's/=\([^ ]*\)/="\1"/g; s/[^ ]*/--global-option=&/g') ; \ + pip_global_option=$${pip_global_option}" --no-use-pep517" ; \ + fi ; \ + $(RUN) $(MSG) \ + _PYTHON_HOST_PLATFORM=\"$(TC_TARGET)\" \ + PATH=$${PATH} \ + CMAKE_TOOLCHAIN_FILE=$${CMAKE_TOOLCHAIN_FILE} \ + MESON_CROSS_FILE=$${MESON_CROSS_FILE} \ + cross-pip \ + $(PIP_WHEEL_ARGS_CROSSENV) \ + $${pip_global_option} \ + --no-build-isolation \ + $(ABI3) \ + $(REQUIREMENT) ; \ + $(RUN) \ + _PYTHON_HOST_PLATFORM="$(TC_TARGET)" \ + PATH=$${PATH} \ + CMAKE_TOOLCHAIN_FILE=$${CMAKE_TOOLCHAIN_FILE} \ + MESON_CROSS_FILE=$${MESON_CROSS_FILE} \ + cross-pip \ + $(PIP_WHEEL_ARGS_CROSSENV) \ + $${pip_global_option} \ + --no-build-isolation \ + $(ABI3) \ + $(REQUIREMENT) + +post_wheel_compile_target: $(WHEEL_COMPILE_TARGET) + +ifeq ($(wildcard $(WHEEL_COMPILE_COOKIE)),) +wheel_compile: $(WHEEL_COMPILE_COOKIE) + +$(WHEEL_COMPILE_COOKIE): $(POST_WHEEL_COMPILE_TARGET) + $(create_target_dir) + @touch -f $@ +else +wheel_compile: ; +endif diff --git a/mk/spksrc.wheel-download.mk b/mk/spksrc.wheel-download.mk new file mode 100644 index 00000000000..479fa24a024 --- /dev/null +++ b/mk/spksrc.wheel-download.mk @@ -0,0 +1,86 @@ +### Wheel rules +# Download wheels for modules listed in WHEELS. +# +# Targets are executed in the following order: +# wheel_download_msg_target +# pre_wheel_download_target (override with PRE_WHEEL_DOWNLOAD_TARGET) +# wheel_download_target (override with WHEEL_DOWNLOAD_TARGET) +# post_wheel_download_target (override with POST_WHEEL_DOWNLOAD_TARGET) +# Variables: +# REQUIREMENT Requirement formatted wheel information +# WHEEL_NAME Name of wheel to process +# WHEEL_VERSION Version of wheel to process (can be empty) +# WHEEL_TYPE Type of wheel to process (abi3, crossenv, pure) + +ifeq ($(WHEEL_VERSION),) +WHEEL_DOWNLOAD_COOKIE = $(WORK_DIR)/.$(COOKIE_PREFIX)wheel_download-$(WHEEL_NAME)_done +else +WHEEL_DOWNLOAD_COOKIE = $(WORK_DIR)/.$(COOKIE_PREFIX)wheel_download-$(WHEEL_NAME)-$(WHEEL_VERSION)_done +endif + +## + +ifeq ($(strip $(PRE_WHEEL_DOWNLOAD_TARGET)),) +PRE_WHEEL_DOWNLOAD_TARGET = pre_wheel_download_target +else +$(PRE_WHEEL_DOWNLOAD_TARGET): wheel_download_msg_target +endif +ifeq ($(strip $(WHEEL_DOWNLOAD_TARGET)),) +WHEEL_DOWNLOAD_TARGET = wheel_download_target +else +$(WHEEL_DOWNLOAD_TARGET): $(BUILD_WHEEL_DOWNLOAD_TARGET) +endif +ifeq ($(strip $(POST_WHEEL_DOWNLOAD_TARGET)),) +POST_WHEEL_DOWNLOAD_TARGET = post_wheel_download_target +else +$(POST_WHEEL_DOWNLOAD_TARGET): $(WHEEL_DOWNLOAD_TARGET) +endif + +wheel_download_msg_target: + @$(MSG) "Processing wheels of $(NAME)" + +pre_wheel_download_target: wheel_download_msg_target + +wheel_download_target: SHELL:=/bin/bash +wheel_download_target: +ifeq ($(wildcard $(PIP_DISTRIB_DIR)),) + @$(MSG) Creating pip download directory: $(PIP_DISTRIB_DIR) + @mkdir -p $(PIP_DISTRIB_DIR) +endif +ifeq ($(wildcard $(PIP_CACHE_DIR)),) + @$(MSG) Creating pip caching directory: $(PIP_CACHE_DIR) + @mkdir -p $(PIP_CACHE_DIR) +endif + @$(MSG) Downloading wheel [$(WHEEL_NAME)], version [$(WHEEL_VERSION)] ; \ + if [ "$$(grep -Eo 'http://|https://' <<< $(REQUIREMENT))" ] ; then \ + echo "WARNING: Skipping download URL - Downloaded at build time" ; \ + elif [ "$(WHEEL_TYPE)" = "pure" ] && [ ! "$(WHEELS_PURE_PYTHON_PACKAGING_ENABLE)" = "1" ]; then \ + echo "WARNING: Skipping download - pure python packaging disabled" ; \ + else \ + query="curl -s https://pypi.org/pypi/$(WHEEL_NAME)/json" ; \ + query+=" | jq -r '.releases[][]" ; \ + query+=" | select(.packagetype==\"sdist\")" ; \ + query+=" | select((.filename|test(\"-$(WHEEL_VERSION).tar.gz\")) or (.filename|test(\"-$(WHEEL_VERSION).zip\"))) | .url'" ; \ + outFile=$$(basename $$(eval $${query} 2>/dev/null) 2/dev/null 2>&1; then \ - $(MSG) Copying $(WHEELS_DEFAULT) to wheelhouse ; \ - cp requirements*.txt $(STAGING_INSTALL_WHEELHOUSE) ; \ - cat requirements*.txt >> $(STAGING_INSTALL_WHEELHOUSE)/$(WHEELS_DEFAULT) ; \ - sed -i -e '/^#/! s/^.*egg=//g' $(STAGING_INSTALL_WHEELHOUSE)/requirements*.txt ; \ - sort -u -o $(STAGING_INSTALL_WHEELHOUSE)/$(WHEELS_DEFAULT) $(STAGING_INSTALL_WHEELHOUSE)/$(WHEELS_DEFAULT) ; \ - else \ - $(MSG) [SKIP] Copying $(WHEELS_DEFAULT) to wheelhouse ; \ - fi ; \ - if stat -t *.whl >/dev/null 2>&1; then \ - for w in *.whl; do \ - if echo $${w} | grep -iq "-none-any\.whl" ; then \ - _new_name=$$(echo $$w | cut -d"-" -f -3)-none-any.whl ; \ - else \ - _new_name=$$(echo $$w | sed -E "s/(.*-).*(linux_).*(\.whl)/\1\2$(PYTHON_ARCH)\3/") ; \ - fi ; \ - $(MSG) Copying to wheelhouse: $$_new_name ; \ - cp -f $$w $(STAGING_INSTALL_WHEELHOUSE)/$$_new_name ; \ - done ; \ - fi ; \ - fi - diff --git a/mk/spksrc.wheel-install.mk b/mk/spksrc.wheel-install.mk new file mode 100644 index 00000000000..e72acd2dd28 --- /dev/null +++ b/mk/spksrc.wheel-install.mk @@ -0,0 +1,103 @@ +### Wheel rules +# Install wheels for modules listed in WHEELS. +# +# Targets are executed in the following order: +# wheel_install_msg_target +# pre_wheel_install_target (override with PRE_WHEEL_INSTALL_TARGET) +# wheel_install_target (override with WHEEL_INSTALL_TARGET) +# post_wheel_install_target (override with POST_WHEEL_INSTALL_TARGET) +# Variables: +# REQUIREMENT Requirement formatted wheel information +# WHEEL_NAME Name of wheel to process +# WHEEL_VERSION Version of wheel to process (can be empty) +# WHEEL_TYPE Type of wheel to process (abi3, crossenv, pure) + +ifeq ($(WHEEL_VERSION),) +WHEEL_INSTALL_COOKIE = $(WORK_DIR)/.$(COOKIE_PREFIX)wheel_install-$(WHEEL_NAME)_done +else +WHEEL_INSTALL_COOKIE = $(WORK_DIR)/.$(COOKIE_PREFIX)wheel_install-$(WHEEL_NAME)-$(WHEEL_VERSION)_done +endif + +## + +ifeq ($(strip $(PRE_WHEEL_INSTALL_TARGET)),) +PRE_WHEEL_INSTALL_TARGET = pre_wheel_install_target +else +$(PRE_WHEEL_INSTALL_TARGET): wheel_install_msg_target +endif +ifeq ($(strip $(WHEEL_INSTALL_TARGET)),) +WHEEL_INSTALL_TARGET = wheel_install_target +else +$(WHEEL_INSTALL_TARGET): $(BUILD_WHEEL_INSTALL_TARGET) +endif +ifeq ($(strip $(POST_WHEEL_INSTALL_TARGET)),) +POST_WHEEL_INSTALL_TARGET = post_wheel_install_target +else +$(POST_WHEEL_INSTALL_TARGET): $(WHEEL_INSTALL_TARGET) +endif + +wheel_install_msg_target: + @$(MSG) "Processing wheels of $(NAME)" + +pre_wheel_install_target: wheel_install_msg_target + +wheel_install_target: SHELL:=/bin/bash +wheel_install_target: + @$(MSG) Installing wheel [$(WHEEL_NAME)], version [$(WHEEL_VERSION)], type [$(WHEEL_TYPE)] ; \ + case $(WHEEL_TYPE) in \ + abi3) $(MSG) Adding $(WHEEL_NAME)==$(WHEEL_VERSION) to wheelhouse/$(WHEELS_LIMITED_API) ; \ + echo $(WHEEL_NAME)==$(WHEEL_VERSION) | sed -e '/^[[:blank:]]*$$\|^#/d' >> $(WHEELHOUSE)/$(WHEELS_LIMITED_API) ; \ + ;; \ + cross) $(MSG) Adding $(WHEEL_NAME)==$(WHEEL_VERSION) to wheelhouse/$(WHEELS_CROSS_COMPILE) ; \ + echo $(WHEEL_NAME)==$(WHEEL_VERSION) | sed -e '/^[[:blank:]]*$$\|^#/d' >> $(WHEELHOUSE)/$(WHEELS_CROSS_COMPILE) ; \ + ;; \ + crossenv) $(MSG) Adding $(WHEEL_NAME)==$(WHEEL_VERSION) to wheelhouse/$(WHEELS_CROSSENV_COMPILE) ; \ + echo $(WHEEL_NAME)==$(WHEEL_VERSION) | sed -e '/^[[:blank:]]*$$\|^#/d' >> $(WHEELHOUSE)/$(WHEELS_CROSSENV_COMPILE) ; \ + ;; \ + pure) $(MSG) Adding $(WHEEL_NAME)==$(WHEEL_VERSION) to wheelhouse/$(WHEELS_PURE_PYTHON) ; \ + echo $(WHEEL_NAME)==$(WHEEL_VERSION) | sed -e '/^[[:blank:]]*$$\|^#/d' >> $(WHEELHOUSE)/$(WHEELS_PURE_PYTHON) ; \ + ;; \ + *) $(MSG) No type found for wheel [$(REQUIREMENT)] ; \ + ;; \ + esac + @for file in $$(ls -1 $(WHEELHOUSE)/requirements-*.txt) ; do \ + sort -u -o $${file}{,} ; \ + done + +install_python_wheel: + @if [ -d "$(WHEELHOUSE)" ] ; then \ + mkdir -p $(STAGING_INSTALL_WHEELHOUSE) ; \ + cd $(WHEELHOUSE) ; \ + if stat -t requirements*.txt >/dev/null 2>&1; then \ + $(MSG) Copying $(WHEELS_DEFAULT) to wheelhouse ; \ + cp requirements*.txt $(STAGING_INSTALL_WHEELHOUSE) ; \ + cat requirements*.txt >> $(STAGING_INSTALL_WHEELHOUSE)/$(WHEELS_DEFAULT) ; \ + sort -u -o $(STAGING_INSTALL_WHEELHOUSE)/$(WHEELS_DEFAULT) $(STAGING_INSTALL_WHEELHOUSE)/$(WHEELS_DEFAULT) ; \ + else \ + $(MSG) [SKIP] Copying $(WHEELS_DEFAULT) to wheelhouse ; \ + fi ; \ + if stat -t *.whl >/dev/null 2>&1; then \ + for w in *.whl; do \ + if echo $${w} | grep -iq "-none-any\.whl" ; then \ + _new_name=$$(echo $$w | cut -d"-" -f -3)-none-any.whl ; \ + else \ + _new_name=$$(echo $$w | sed -E "s/(.*-).*(linux_).*(\.whl)/\1\2$(PYTHON_ARCH)\3/") ; \ + fi ; \ + $(MSG) Copying to wheelhouse: $$_new_name ; \ + cp -f $$w $(STAGING_INSTALL_WHEELHOUSE)/$$_new_name ; \ + done ; \ + fi ; \ + fi + + +post_wheel_install_target: $(WHEEL_INSTALL_TARGET) + +ifeq ($(wildcard $(WHEEL_INSTALL_COOKIE)),) +wheel_install: $(WHEEL_INSTALL_COOKIE) + +$(WHEEL_INSTALL_COOKIE): $(POST_WHEEL_INSTALL_TARGET) + $(create_target_dir) + @touch -f $@ +else +wheel_install: ; +endif diff --git a/mk/spksrc.wheel.mk b/mk/spksrc.wheel.mk index c816c0c6999..3090d260146 100644 --- a/mk/spksrc.wheel.mk +++ b/mk/spksrc.wheel.mk @@ -1,9 +1,9 @@ ### Wheel rules -# Create wheels for modules listed in WHEELS. -# If CROSS_COMPILE_WHEELS is set via python-cc.mk, -# wheels are cross-compiled. If not, pure-python -# wheels are created. - +# Process wheels for modules listed in WHEELS. +# 1. wheel_download +# 2. wheel_compile +# 3. wheel_install +# # Targets are executed in the following order: # wheel_msg_target # pre_wheel_target (override with PRE_WHEEL_TARGET) @@ -12,6 +12,13 @@ # Variables: # WHEELS List of wheels to go through +# When wheel is called from: +# spksrc.spk.mk: MAKECMDGOALS is empty (needs to be set to wheel) +# make wheel-- : MAKECMDGOALS is wheel +# make download-wheels : MAKECMDGOALS is download-wheels +WHEEL_GOAL := $(if $(MAKECMDGOALS),$(MAKECMDGOALS),wheel) + +# Completion status file WHEEL_COOKIE = $(WORK_DIR)/.$(COOKIE_PREFIX)wheel_done ## python wheel specific configurations @@ -26,8 +33,21 @@ include ../../mk/spksrc.cross-cmake-env.mk ## meson specific configurations include ../../mk/spksrc.cross-meson-env.mk +include ../../mk/spksrc.wheel-download.mk + +wheel_compile: wheel_download +include ../../mk/spksrc.wheel-compile.mk + +wheel_install: wheel_compile +include ../../mk/spksrc.wheel-install.mk + ## +ifneq ($(strip $(REQUIREMENT)),) +download-wheels: wheel_download +wheel: wheel_install +else + ifeq ($(strip $(PRE_WHEEL_TARGET)),) PRE_WHEEL_TARGET = pre_wheel_target else @@ -45,217 +65,89 @@ $(POST_WHEEL_TARGET): $(WHEEL_TARGET) endif wheel_msg_target: - @$(MSG) "Processing wheels of $(NAME)" + @$(MSG) "Processing wheel for $(NAME)" -wheeldownload: SHELL:=/bin/bash -wheeldownload: -ifeq ($(wildcard $(PIP_DISTRIB_DIR)),) - @$(MSG) Creating pip download directory: $(PIP_DISTRIB_DIR) - @mkdir -p $(PIP_DISTRIB_DIR) -else - @$(MSG) Using existing pip download directory: $(PIP_DISTRIB_DIR) -endif -ifeq ($(wildcard $(PIP_CACHE_DIR)),) - @$(MSG) Creating pip caching directory: $(PIP_CACHE_DIR) - @mkdir -p $(PIP_CACHE_DIR) -else - @$(MSG) Using existing pip cache directory: $(PIP_CACHE_DIR) -endif - @if [ -n "$(WHEELS)" ] ; then \ - for wheel in $(WHEELS_2_DOWNLOAD) ; do \ - $(MSG) "Downloading wheels from $$wheel ..." ; \ - # BROKEN: https://github.com/pypa/pip/issues/1884 ; \ - # xargs -n 1 $(PIP_SYSTEM) $(PIP_DOWNLOAD_ARGS) 2>/dev/null < $$wheel || true ; \ - while IFS= read -r requirement ; do \ - if [ "$$(grep -s egg <<< $${requirement})" ] ; then \ - name=$$(echo $${requirement#*egg=} | cut -f1 -d=) ; \ - url=$${requirement} ; \ - else \ - name=$${requirement%%[<>=]=*} ; \ - url="" ; \ - fi ; \ - version=$$(echo $${requirement#*[<>=]=} | cut -f1 -d' ') ; \ - # If no version was provided then find the latest version ; \ - if [ "$${version}" == "$${name}" ]; then \ - query="curl -s https://pypi.org/pypi/$${name}/json" ; \ - query+=" | jq -r '.releases[][]" ; \ - query+=" | select(.packagetype==\"sdist\")" ; \ - query+=" | .filename'" ; \ - query+=" | sort -V" ; \ - query+=" | tail -1" ; \ - query+=" | sed -e 's/.tar.gz//g' -e 's/.zip//g'" ; \ - query+=" | awk -F'-' '{print \$$2}'" ; \ - version=$$(eval $${query} 2>/dev/null) ; \ - fi ; \ - $(MSG) pip download [$${name}], version [$${version}]$$([ "$${url}" ] && echo ", URL: [$${url}] ") ; \ - if [ "$$(grep -s egg <<< $${requirement})" ] ; then \ - echo "WARNING: Skipping download URL - Downloaded at build time" ; \ - # Will be re-downloaded anyway at build time ; \ - # $(PIP) $(PIP_DOWNLOAD_ARGS) $${requirement} 2>/dev/null ; \ - else \ - query="curl -s https://pypi.org/pypi/$${name}/json" ; \ - query+=" | jq -r '.releases[][]" ; \ - query+=" | select(.packagetype==\"sdist\")" ; \ - query+=" | select((.filename|test(\"-$${version}.tar.gz\")) or (.filename|test(\"-$${version}.zip\"))) | .url'" ; \ - localFile=$$(basename $$(eval $${query} 2>/dev/null) 2 Adding $$wheel as pure-python (wheelhouse/$(WHEELS_PURE_PYTHON))" ; \ - sed -e '/^[[:blank:]]*$$\|^#/d' $$wheel >> $(WHEELHOUSE)/$(WHEELS_PURE_PYTHON) ; \ - elif [ $$(basename $$wheel) = $(WHEELS_CROSSENV_COMPILE) ]; then \ - echo "===> Adding $$wheel as cross-compiled (wheelhouse/$(WHEELS_CROSSENV_COMPILE))" ; \ - sed -e '/^[[:blank:]]*$$\|^#/d' $$wheel >> $(WHEELHOUSE)/$(WHEELS_CROSSENV_COMPILE) ; \ - elif [ $$(basename $$wheel) = $(WHEELS_LIMITED_API) ]; then \ - echo "===> Adding $$wheel as ABI-limited (wheelhouse/$(WHEELS_LIMITED_API))" ; \ - sed -e '/^[[:blank:]]*$$\|^#/d' $$wheel >> $(WHEELHOUSE)/$(WHEELS_LIMITED_API) ; \ - else \ - echo "===> Adding $$wheel to default (wheelhouse/$(WHEELS_DEFAULT_REQUIREMENT))" ; \ - sed -e '/^[[:blank:]]*$$\|^#/d' $$wheel >> $(WHEELHOUSE)/$(WHEELS_DEFAULT_REQUIREMENT) ; \ - fi ;\ - done - @for file in $$(ls -1 $(WHEELHOUSE)/requirements-*.txt) ; do \ - sort -u -o $${file}{,} ; \ - done + $(error No wheel to process) endif -# Build cross compiled wheels first, to fail fast. build_wheel_target: SHELL:=/bin/bash -build_wheel_target: $(PRE_WHEEL_TARGET) -ifneq ($(wildcard $(abspath $(addprefix $(WORK_DIR)/../,$(patsubst %-pure.txt,,$(WHEELS))))),) +build_wheel_target: pre_wheel_target +ifneq ($(wildcard $(abspath $(addprefix $(WORK_DIR)/../,$(WHEELS)))),) @while IFS= read -r requirement ; do \ - wheel=$${requirement#*:} ; \ + $(MSG) Processing requirement [$${requirement}] ; \ + wheel=$${requirement#*requirements-*.txt:} ; \ file=$$(basename $${requirement%%:*}) ; \ - [ "$$(grep -s egg <<< $${wheel})" ] && name=$$(echo $${wheel#*egg=} | cut -f1 -d=) || name=$${wheel%%[<>=]=*} ; \ - version=$$(echo $${requirement#*[<>=]=} | cut -f1 -d' ') ; \ - $(MSG) "WHEEL=\"$${name}-$${version}\" $(MAKE) crossenv-$(ARCH)-$(TCVERSION)" ; \ - MAKEFLAGS= WHEEL="$${name}-$${version}" $(MAKE) crossenv-$(ARCH)-$(TCVERSION) --no-print-directory ; \ - [ "$${file}" = "$(WHEELS_LIMITED_API)" ] && abi3="--build-option=--py-limited-api=$(PYTHON_LIMITED_API)" || abi3="" ; \ - global_options=$$(echo $(WHEELS_BUILD_ARGS) | sed -e 's/ \[/\n\[/g' | grep -i $${name} | cut -f2 -d] | xargs) ; \ - localCFLAGS=($$(echo $(WHEELS_CFLAGS) | sed -e 's/ \[/\n\[/g' | grep -i $${name} | cut -f2 -d] | xargs)) ; \ - localLDFLAGS=($$(echo $(WHEELS_LDFLAGS) | sed -e 's/ \[/\n\[/g' | grep -i $${name} | cut -f2 -d] | xargs)) ; \ - localCPPFLAGS=($$(echo $(WHEELS_CPPFLAGS) | sed -e 's/ \[/\n\[/g' | grep -i $${name} | cut -f2 -d] | xargs)) ; \ - localCXXFLAGS=($$(echo $(WHEELS_CXXFLAGS) | sed -e 's/ \[/\n\[/g' | grep -i $${name} | cut -f2 -d] | xargs)) ; \ - $(MSG) pip build [$${name}], version: [$${version}] \ - $$([ "$$(echo $${localCFLAGS[@]})" ] && echo "CFLAGS=\"$${localCFLAGS[@]}\" ") \ - $$([ "$$(echo $${localCPPFLAGS[@]})" ] && echo "CPPFLAGS=\"$${localCPPFLAGS[@]}\" ") \ - $$([ "$$(echo $${localCXXFLAGS[@]})" ] && echo "CXXFLAGS=\"$${localCXXFLAGS[@]}\" ") \ - $$([ "$$(echo $${localLDFLAGS[@]})" ] && echo "LDFLAGS=\"$${localLDFLAGS[@]}\" ") \ - $$([ "$$(echo $${abi3})" ] && echo "$${abi3} ")" \ - $${global_options}" ; \ - REQUIREMENT=$$(echo $${wheel%% *}) \ - WHEEL_NAME=$${name} \ - WHEEL_VERSION=$${version} \ - ADDITIONAL_CFLAGS="-I$(STAGING_INSTALL_PREFIX)/$(PYTHON_INC_DIR) $${localCFLAGS[@]}" \ - ADDITIONAL_CPPFLAGS="-I$(STAGING_INSTALL_PREFIX)/$(PYTHON_INC_DIR) $${localCPPFLAGS[@]}" \ - ADDITIONAL_CXXFLAGS="-I$(STAGING_INSTALL_PREFIX)/$(PYTHON_INC_DIR) $${localCXXFLAGS[@]}" \ - ADDITIONAL_LDFLAGS="$${localLDFLAGS[@]}" \ - ABI3="$${abi3}" \ - PIP_GLOBAL_OPTION="$${global_options}" \ - $(MAKE) --no-print-directory \ - cross-compile-wheel-$${name} || exit 1 ; \ - done < <(grep -svH -e "^\#" -e "^\$$" $(wildcard $(abspath $(addprefix $(WORK_DIR)/../,$(patsubst %-pure.txt,,$(WHEELS)))))) -else - @$(MSG) "[SKIP] Cross-compiling wheels" + case $${file} in \ + requirements-pure.txt) type=pure ;; \ + requirements-crossenv*.txt) type=crossenv ;; \ + requirements-abi3.txt) type=abi3 ;; \ + *) type=$(WHEEL_DEFAULT_PREFIX) ;; \ + esac ; \ + version=$$(echo $${wheel} | grep -oP '(?<=([<>=]=))[^ ]*' || echo "") ; \ + if [ "$$(grep -s egg <<< $${wheel})" ]; then \ + name=$$(echo $${wheel#*egg=} | cut -f1 -d=) ; \ + wheel=$$(echo $${wheel%%#egg=*}) ; \ + else \ + name=$$(echo $${wheel%%[<>=]=*} | sed -E "s/^(abi3|crossenv|pure)://") ; \ + fi ; \ + if [ ! "$${version}" ]; then \ + $(MSG) Fetching latest version available ; \ + query="curl -s https://pypi.org/pypi/$${name}/json" ; \ + query+=" | jq -r '.releases[][]" ; \ + query+=" | select(.packagetype==\"sdist\")" ; \ + query+=" | .filename'" ; \ + query+=" | sort -V" ; \ + query+=" | tail -1" ; \ + query+=" | sed -e 's/.tar.gz//g' -e 's/.zip//g'" ; \ + query+=" | awk -F'-' '{print \$$2}'" ; \ + version=$$(eval $${query} 2>/dev/null) ; \ + fi ; \ + $(MSG) $(MAKE) ARCH=$(ARCH) TCVERSION=$(TCVERSION) REQUIREMENT=\"$${wheel}\" WHEEL_NAME=\"$${name}\" WHEEL_VERSION=\"$${version}\" WHEEL_TYPE=\"$${type}\" $(WHEEL_GOAL) ; \ + MAKEFLAGS= $(MAKE) ARCH="$(ARCH)" TCVERSION="$(TCVERSION)" REQUIREMENT="$${wheel}" WHEEL_NAME="$${name}" WHEEL_VERSION="$${version}" WHEEL_TYPE="$${type}" $(WHEEL_GOAL) --no-print-directory ; \ + done < <(grep -svH -e "^\#" -e "^\$$" $(wildcard $(abspath $(addprefix $(WORK_DIR)/../,$(WHEELS)))) | sed 's/\s* #.*//') endif -ifneq ($(filter 1 ON TRUE,$(WHEELS_PURE_PYTHON_PACKAGING_ENABLE)),) - @if [ -s "$(WHEELHOUSE)/$(WHEELS_PURE_PYTHON)" ]; then \ - $(MSG) "Building pure-python" ; \ - export LD= LDSHARED= CPP= NM= CC= AS= RANLIB= CXX= AR= STRIP= OBJDUMP= OBJCOPY= READELF= CFLAGS= CPPFLAGS= CXXFLAGS= LDFLAGS= && \ - $(RUN) \ - PATH="$(abspath $(WORK_DIR)/../../../native/$(PYTHON_PKG_NAME)/work-native/install/usr/local/bin):$(PATH)" \ - LD_LIBRARY_PATH="$(abspath $(WORK_DIR)/../../../native/$(PYTHON_PKG_NAME)/work-native/install/usr/local/lib):$(LD_LIBRARY_PATH)" \ - $(PIP) $(PIP_WHEEL_ARGS) --requirement $(WHEELHOUSE)/$(WHEELS_PURE_PYTHON) ; \ - fi -else - @$(MSG) "[SKIP] Building pure-python" +ifneq ($(filter-out $(addprefix src/,$(notdir $(wildcard $(abspath $(addprefix $(WORK_DIR)/../,$(WHEELS)))))),$(WHEELS)),) + @for requirement in $(filter-out $(addprefix src/,$(notdir $(wildcard $(abspath $(addprefix $(WORK_DIR)/../,$(WHEELS)))))),$(WHEELS)) ; do \ + $(MSG) Processing requirement [$${requirement}] ; \ + wheel=$$(echo $${requirement} | sed -E "s/^(abi3|crossenv|pure)://") ; \ + case $${requirement} in \ + abi3:*) type=abi3 ;; \ + crossenv:*) type=crossenv ;; \ + pure:*) type=pure ;; \ + *) type=$(WHEEL_DEFAULT_PREFIX) ;; \ + esac ; \ + version=$$(echo $${wheel} | grep -oP '(?<=([<>=]=))[^ ]*' || echo "") ; \ + if [ "$$(grep -s egg <<< $${requirement})" ]; then \ + name=$$(echo $${wheel#*egg=} | cut -f1 -d=) ; \ + wheel=$$(echo $${wheel%%#egg=*}) ; \ + else \ + name=$$(echo $${wheel%%[<>=]=*} | sed -E "s/^(abi3|crossenv|pure)://") ; \ + fi ; \ + if [ ! "$${version}" ]; then \ + $(MSG) Fetching latest version available ; \ + query="curl -s https://pypi.org/pypi/$${name}/json" ; \ + query+=" | jq -r '.releases[][]" ; \ + query+=" | select(.packagetype==\"sdist\")" ; \ + query+=" | .filename'" ; \ + query+=" | sort -V" ; \ + query+=" | tail -1" ; \ + query+=" | sed -e 's/.tar.gz//g' -e 's/.zip//g'" ; \ + query+=" | awk -F'-' '{print \$$2}'" ; \ + version=$$(eval $${query} 2>/dev/null) ; \ + fi ; \ + $(MSG) $(MAKE) ARCH=$(ARCH) TCVERSION=$(TCVERSION) REQUIREMENT=\"$${wheel}\" WHEEL_NAME=\"$${name}\" WHEEL_VERSION=\"$${version}\" WHEEL_TYPE=\"$${type}\" $(WHEEL_GOAL) ; \ + MAKEFLAGS= $(MAKE) ARCH="$(ARCH)" TCVERSION="$(TCVERSION)" REQUIREMENT="$${wheel}" WHEEL_NAME="$${name}" WHEEL_VERSION="$${version}" WHEEL_TYPE="$${type}" $(WHEEL_GOAL) --no-print-directory ; \ + done endif -## -## crossenv PATH environment requires a combination of: -## 1) unique PATH variable from $(ENV) -> using merge + dedup macros -## Note: Multiple declarations of ENV += PATH=bla creates confusion in its interpretation. -## Solution implemented fetches all PATH from ENV and combine them in reversed order. -## 2) access to maturin from native/python/.../bin -> ${PYTHON_NATIVE_PATH}/bin -## 3) access to crossenv/bin/cross* tools, mainly cross-pip -> ${CROSSENV_PATH}/bin -## -cross-compile-wheel-%: SHELL:=/bin/bash -cross-compile-wheel-%: - @for crossenv in $(WORK_DIR)/crossenv-$(WHEEL_NAME)-$(WHEEL_VERSION) $(WORK_DIR)/crossenv-$(WHEEL_NAME) $(WORK_DIR)/crossenv ; do \ - [ -d $${crossenv} ] && . $${crossenv}/build/python-cc.mk && break ; \ - done ; \ - if [ -d "$${CROSSENV_PATH}" ] ; then \ - PATH=$(call dedup, $(call merge, $(ENV), PATH, :), :):$${PYTHON_NATIVE_PATH}:$${CROSSENV_PATH}/bin:$${PATH} ; \ - $(MSG) "crossenv: [$${CROSSENV_PATH}]" ; \ - $(MSG) "pip: [$$(which cross-pip)]" ; \ - $(MSG) "maturin: [$$(which maturin)]" ; \ - else \ - echo "ERROR: crossenv not found!" ; \ - exit 2 ; \ - fi ; \ - if [ "$(PIP_GLOBAL_OPTION)" ]; then \ - pip_global_option=$$(echo $(PIP_GLOBAL_OPTION) | sed 's/=\([^ ]*\)/="\1"/g; s/[^ ]*/--global-option=&/g') ; \ - pip_global_option=$${pip_global_option}" --no-use-pep517" ; \ - fi ; \ - $(RUN) $(MSG) \ - _PYTHON_HOST_PLATFORM=\"$(TC_TARGET)\" \ - PATH=$${PATH} \ - CMAKE_TOOLCHAIN_FILE=$${CMAKE_TOOLCHAIN_FILE} \ - MESON_CROSS_FILE=$${MESON_CROSS_FILE} \ - cross-pip \ - $(PIP_WHEEL_ARGS_CROSSENV) \ - $${pip_global_option} \ - --no-build-isolation \ - $(ABI3) \ - $(REQUIREMENT) ; \ - $(RUN) \ - _PYTHON_HOST_PLATFORM="$(TC_TARGET)" \ - PATH=$${PATH} \ - CMAKE_TOOLCHAIN_FILE=$${CMAKE_TOOLCHAIN_FILE} \ - MESON_CROSS_FILE=$${MESON_CROSS_FILE} \ - cross-pip \ - $(PIP_WHEEL_ARGS_CROSSENV) \ - $${pip_global_option} \ - --no-build-isolation \ - $(ABI3) \ - $(REQUIREMENT) - +download-wheels: $(WHEEL_TARGET) post_wheel_target: $(WHEEL_TARGET) install_python_wheel @@ -268,3 +160,6 @@ $(WHEEL_COOKIE): $(POST_WHEEL_TARGET) else wheel: ; endif + +# endif REQUIREMENT non-empty +endif