From 92894686fcacb98334424ff4a69f01a64912e0a4 Mon Sep 17 00:00:00 2001 From: Vincent Fortier Date: Mon, 13 Jan 2025 01:46:15 +0000 Subject: [PATCH 01/24] wheel.mk: Use status cookie for general wheel + wheel_download --- mk/spksrc.wheel-download.mk | 129 ++++++++++++++++ mk/spksrc.wheel.mk | 286 ++++++++++-------------------------- 2 files changed, 210 insertions(+), 205 deletions(-) create mode 100644 mk/spksrc.wheel-download.mk diff --git a/mk/spksrc.wheel-download.mk b/mk/spksrc.wheel-download.mk new file mode 100644 index 00000000000..55909daf003 --- /dev/null +++ b/mk/spksrc.wheel-download.mk @@ -0,0 +1,129 @@ +### Wheel rules +# Create wheels for modules listed in WHEELS. +# If CROSS_COMPILE_WHEELS is set via python-cc.mk, +# wheels are cross-compiled. If not, pure-python +# wheels are created. + +# Targets are executed in the following order: +# wheel_download_msg_target +# pre_wheel_download_target (override with PRE_WHEEL_DOWNLOAD_TARGET) +# wheel_download_target (override with WHEEL_DOWNLOAD_TARGET) +# post_wheel_download_target (override with POST_WHEEL_DOWNLOAD_TARGET) +# Variables: +# REQUIREMENT Requirement formatted wheel information +# WHEEL_NAME Name of wheel to process +# WHEEL_VERSION Version of wheel to process (can be empty) +# WHEEL_TYPE Type of wheel to process (abi3, crossenv, pure) + +ifeq ($(WHEEL_VERSION),) +WHEEL_DOWNLOAD_COOKIE = $(WORK_DIR)/.$(COOKIE_PREFIX)wheel_download-$(WHEEL_NAME)_done +else +WHEEL_DOWNLOAD_COOKIE = $(WORK_DIR)/.$(COOKIE_PREFIX)wheel_download-$(WHEEL_NAME)-$(WHEEL_VERSION)_done +endif + +## + +ifeq ($(strip $(PRE_WHEEL_DOWNLOAD_TARGET)),) +PRE_WHEEL_DOWNLOAD_TARGET = pre_wheel_download_target +else +$(PRE_WHEEL_DOWNLOAD_TARGET): wheel_download_msg_target +endif +ifeq ($(strip $(WHEEL_DOWNLOAD_TARGET)),) +WHEEL_DOWNLOAD_TARGET = wheel_download_target +else +$(WHEEL_DOWNLOAD_TARGET): $(BUILD_WHEEL_DOWNLOAD_TARGET) +endif +ifeq ($(strip $(POST_WHEEL_DOWNLOAD_TARGET)),) +POST_WHEEL_DOWNLOAD_TARGET = post_wheel_download_target +else +$(POST_WHEEL_DOWNLOAD_TARGET): $(WHEEL_DOWNLOAD_TARGET) +endif + +wheel_download_msg_target: + @$(MSG) "Processing wheels of $(NAME)" + +pre_wheel_download_target: wheel_download_msg_target + +wheel_download_target: SHELL:=/bin/bash +wheel_download_target: +ifeq ($(wildcard $(PIP_DISTRIB_DIR)),) + @$(MSG) Creating pip download directory: $(PIP_DISTRIB_DIR) + @mkdir -p $(PIP_DISTRIB_DIR) +endif +ifeq ($(wildcard $(PIP_CACHE_DIR)),) + @$(MSG) Creating pip caching directory: $(PIP_CACHE_DIR) + @mkdir -p $(PIP_CACHE_DIR) +endif +ifeq ($(wildcard $(WHEELHOUSE)),) + @$(MSG) Creating wheelhouse directory: $(WHEELHOUSE) + @mkdir -p $(WHEELHOUSE) +endif + @if [ ! "$(WHEEL_VERSION)" ]; then \ + $(MSG) Fetching latest version available ; \ + query="curl -s https://pypi.org/pypi/$(WHEEL_NAME)/json" ; \ + query+=" | jq -r '.releases[][]" ; \ + query+=" | select(.packagetype==\"sdist\")" ; \ + query+=" | .filename'" ; \ + query+=" | sort -V" ; \ + query+=" | tail -1" ; \ + query+=" | sed -e 's/.tar.gz//g' -e 's/.zip//g'" ; \ + query+=" | awk -F'-' '{print \$$2}'" ; \ + version=$$(eval $${query} 2>/dev/null) ; \ + else \ + version=$(WHEEL_VERSION) ; \ + fi ; \ +# $(MSG) requirement: [$(REQUIREMENT)] ; \ +# $(MSG) requirement-grep-egg: [$$(grep -s egg <<< $(REQUIREMENT))] ; \ +# $(MSG) name: [$(WHEEL_NAME)] ; \ +# $(MSG) type: [$(WHEEL_TYPE)] ; \ +# $(MSG) version-shell: [$${version}] ; \ +# $(MSG) version-make: [$(WHEEL_VERSION)] ; \ +# $(MSG) type: [$(WHEEL_TYPE)] ; \ + $(MSG) Downloading wheel [$(WHEEL_NAME)], version [$${version}] ; \ + if [ "$$(grep -s egg <<< $(REQUIREMENT))" ] ; then \ + echo "WARNING: Skipping download URL - Downloaded at build time" ; \ + else \ + query="curl -s https://pypi.org/pypi/$(WHEEL_NAME)/json" ; \ + query+=" | jq -r '.releases[][]" ; \ + query+=" | select(.packagetype==\"sdist\")" ; \ + query+=" | select((.filename|test(\"-$${version}.tar.gz\")) or (.filename|test(\"-$${version}.zip\"))) | .url'" ; \ + outFile=$$(basename $$(eval $${query} 2>/dev/null) 2> $(WHEELHOUSE)/$(WHEELS_LIMITED_API) ; \ + ;; \ + crossenv) $(MSG) Adding $(WHEEL_NAME)==$${version} to wheelhouse/$(WHEELS_CROSSENV_COMPILE) ; \ + echo $(WHEEL_NAME)==$${version} | sed -e '/^[[:blank:]]*$$\|^#/d' >> $(WHEELHOUSE)/$(WHEELS_CROSSENV_COMPILE) ; \ + ;; \ + pure) $(MSG) Adding $(WHEEL_NAME)==$${version} to wheelhouse/$(WHEELS_PURE_PYTHON) ; \ + echo $(WHEEL_NAME)==$${version} | sed -e '/^[[:blank:]]*$$\|^#/d' >> $(WHEELHOUSE)/$(WHEELS_PURE_PYTHON) ; \ + ;; \ + *) $(MSG) No type found for wheel [$(REQUIREMENT)] ; \ + ;; \ + esac + @for file in $$(ls -1 $(WHEELHOUSE)/requirements-*.txt) ; do \ + sort -u -o $${file}{,} ; \ + done + +post_wheel_download_target: $(WHEEL_DOWNLOAD_TARGET) + + +ifeq ($(wildcard $(WHEEL_DOWNLOAD_COOKIE)),) +wheel_download: $(WHEEL_DOWNLOAD_COOKIE) + +$(WHEEL_DOWNLOAD_COOKIE): $(POST_WHEEL_DOWNLOAD_TARGET) + $(create_target_dir) + @touch -f $@ +else +wheel_download: ; +endif diff --git a/mk/spksrc.wheel.mk b/mk/spksrc.wheel.mk index c816c0c6999..87cc33e333d 100644 --- a/mk/spksrc.wheel.mk +++ b/mk/spksrc.wheel.mk @@ -1,8 +1,8 @@ ### Wheel rules -# Create wheels for modules listed in WHEELS. +# Create wheel for modules listed in WHEELS. # If CROSS_COMPILE_WHEELS is set via python-cc.mk, -# wheels are cross-compiled. If not, pure-python -# wheels are created. +# wheel are cross-compiled. If not, pure-python +# wheel are created. # Targets are executed in the following order: # wheel_msg_target @@ -12,6 +12,7 @@ # Variables: # WHEELS List of wheels to go through +# Completion status file WHEEL_COOKIE = $(WORK_DIR)/.$(COOKIE_PREFIX)wheel_done ## python wheel specific configurations @@ -26,8 +27,25 @@ include ../../mk/spksrc.cross-cmake-env.mk ## meson specific configurations include ../../mk/spksrc.cross-meson-env.mk +include ../../mk/spksrc.wheel-download.mk + +#wheel_configure: wheel_download +#include ../../mk/spksrc.wheel-configure.mk +# +#wheel_compile: wheel_configure +#include ../../mk/spksrc.wheel-compile.mk +# +#wheel_install: wheel_compile +#include ../../mk/spksrc.wheel-install.mk +# +#all: wheel_install + ## +ifneq ($(strip $(REQUIREMENT)),) +wheel: wheel_download +else + ifeq ($(strip $(PRE_WHEEL_TARGET)),) PRE_WHEEL_TARGET = pre_wheel_target else @@ -45,219 +63,74 @@ $(POST_WHEEL_TARGET): $(WHEEL_TARGET) endif wheel_msg_target: - @$(MSG) "Processing wheels of $(NAME)" + @$(MSG) "Processing wheel for $(NAME)" -wheeldownload: SHELL:=/bin/bash -wheeldownload: -ifeq ($(wildcard $(PIP_DISTRIB_DIR)),) - @$(MSG) Creating pip download directory: $(PIP_DISTRIB_DIR) - @mkdir -p $(PIP_DISTRIB_DIR) -else - @$(MSG) Using existing pip download directory: $(PIP_DISTRIB_DIR) -endif -ifeq ($(wildcard $(PIP_CACHE_DIR)),) - @$(MSG) Creating pip caching directory: $(PIP_CACHE_DIR) - @mkdir -p $(PIP_CACHE_DIR) -else - @$(MSG) Using existing pip cache directory: $(PIP_CACHE_DIR) -endif - @if [ -n "$(WHEELS)" ] ; then \ - for wheel in $(WHEELS_2_DOWNLOAD) ; do \ - $(MSG) "Downloading wheels from $$wheel ..." ; \ - # BROKEN: https://github.com/pypa/pip/issues/1884 ; \ - # xargs -n 1 $(PIP_SYSTEM) $(PIP_DOWNLOAD_ARGS) 2>/dev/null < $$wheel || true ; \ - while IFS= read -r requirement ; do \ - if [ "$$(grep -s egg <<< $${requirement})" ] ; then \ - name=$$(echo $${requirement#*egg=} | cut -f1 -d=) ; \ - url=$${requirement} ; \ - else \ - name=$${requirement%%[<>=]=*} ; \ - url="" ; \ - fi ; \ - version=$$(echo $${requirement#*[<>=]=} | cut -f1 -d' ') ; \ - # If no version was provided then find the latest version ; \ - if [ "$${version}" == "$${name}" ]; then \ - query="curl -s https://pypi.org/pypi/$${name}/json" ; \ - query+=" | jq -r '.releases[][]" ; \ - query+=" | select(.packagetype==\"sdist\")" ; \ - query+=" | .filename'" ; \ - query+=" | sort -V" ; \ - query+=" | tail -1" ; \ - query+=" | sed -e 's/.tar.gz//g' -e 's/.zip//g'" ; \ - query+=" | awk -F'-' '{print \$$2}'" ; \ - version=$$(eval $${query} 2>/dev/null) ; \ - fi ; \ - $(MSG) pip download [$${name}], version [$${version}]$$([ "$${url}" ] && echo ", URL: [$${url}] ") ; \ - if [ "$$(grep -s egg <<< $${requirement})" ] ; then \ - echo "WARNING: Skipping download URL - Downloaded at build time" ; \ - # Will be re-downloaded anyway at build time ; \ - # $(PIP) $(PIP_DOWNLOAD_ARGS) $${requirement} 2>/dev/null ; \ - else \ - query="curl -s https://pypi.org/pypi/$${name}/json" ; \ - query+=" | jq -r '.releases[][]" ; \ - query+=" | select(.packagetype==\"sdist\")" ; \ - query+=" | select((.filename|test(\"-$${version}.tar.gz\")) or (.filename|test(\"-$${version}.zip\"))) | .url'" ; \ - localFile=$$(basename $$(eval $${query} 2>/dev/null) 2 Adding $$wheel as pure-python (wheelhouse/$(WHEELS_PURE_PYTHON))" ; \ - sed -e '/^[[:blank:]]*$$\|^#/d' $$wheel >> $(WHEELHOUSE)/$(WHEELS_PURE_PYTHON) ; \ - elif [ $$(basename $$wheel) = $(WHEELS_CROSSENV_COMPILE) ]; then \ - echo "===> Adding $$wheel as cross-compiled (wheelhouse/$(WHEELS_CROSSENV_COMPILE))" ; \ - sed -e '/^[[:blank:]]*$$\|^#/d' $$wheel >> $(WHEELHOUSE)/$(WHEELS_CROSSENV_COMPILE) ; \ - elif [ $$(basename $$wheel) = $(WHEELS_LIMITED_API) ]; then \ - echo "===> Adding $$wheel as ABI-limited (wheelhouse/$(WHEELS_LIMITED_API))" ; \ - sed -e '/^[[:blank:]]*$$\|^#/d' $$wheel >> $(WHEELHOUSE)/$(WHEELS_LIMITED_API) ; \ - else \ - echo "===> Adding $$wheel to default (wheelhouse/$(WHEELS_DEFAULT_REQUIREMENT))" ; \ - sed -e '/^[[:blank:]]*$$\|^#/d' $$wheel >> $(WHEELHOUSE)/$(WHEELS_DEFAULT_REQUIREMENT) ; \ - fi ;\ - done - @for file in $$(ls -1 $(WHEELHOUSE)/requirements-*.txt) ; do \ - sort -u -o $${file}{,} ; \ - done + $(error No wheel to process) endif -# Build cross compiled wheels first, to fail fast. build_wheel_target: SHELL:=/bin/bash -build_wheel_target: $(PRE_WHEEL_TARGET) -ifneq ($(wildcard $(abspath $(addprefix $(WORK_DIR)/../,$(patsubst %-pure.txt,,$(WHEELS))))),) +build_wheel_target: pre_wheel_target +ifneq ($(wildcard $(abspath $(addprefix $(WORK_DIR)/../,$(WHEELS)))),) @while IFS= read -r requirement ; do \ - wheel=$${requirement#*:} ; \ + $(MSG) Processing requirement [$${requirement}] ; \ + wheel=$${requirement#*requirements-*.txt:} ; \ file=$$(basename $${requirement%%:*}) ; \ - [ "$$(grep -s egg <<< $${wheel})" ] && name=$$(echo $${wheel#*egg=} | cut -f1 -d=) || name=$${wheel%%[<>=]=*} ; \ - version=$$(echo $${requirement#*[<>=]=} | cut -f1 -d' ') ; \ - $(MSG) "WHEEL=\"$${name}-$${version}\" $(MAKE) crossenv-$(ARCH)-$(TCVERSION)" ; \ - MAKEFLAGS= WHEEL="$${name}-$${version}" $(MAKE) crossenv-$(ARCH)-$(TCVERSION) --no-print-directory ; \ - [ "$${file}" = "$(WHEELS_LIMITED_API)" ] && abi3="--build-option=--py-limited-api=$(PYTHON_LIMITED_API)" || abi3="" ; \ - global_options=$$(echo $(WHEELS_BUILD_ARGS) | sed -e 's/ \[/\n\[/g' | grep -i $${name} | cut -f2 -d] | xargs) ; \ - localCFLAGS=($$(echo $(WHEELS_CFLAGS) | sed -e 's/ \[/\n\[/g' | grep -i $${name} | cut -f2 -d] | xargs)) ; \ - localLDFLAGS=($$(echo $(WHEELS_LDFLAGS) | sed -e 's/ \[/\n\[/g' | grep -i $${name} | cut -f2 -d] | xargs)) ; \ - localCPPFLAGS=($$(echo $(WHEELS_CPPFLAGS) | sed -e 's/ \[/\n\[/g' | grep -i $${name} | cut -f2 -d] | xargs)) ; \ - localCXXFLAGS=($$(echo $(WHEELS_CXXFLAGS) | sed -e 's/ \[/\n\[/g' | grep -i $${name} | cut -f2 -d] | xargs)) ; \ - $(MSG) pip build [$${name}], version: [$${version}] \ - $$([ "$$(echo $${localCFLAGS[@]})" ] && echo "CFLAGS=\"$${localCFLAGS[@]}\" ") \ - $$([ "$$(echo $${localCPPFLAGS[@]})" ] && echo "CPPFLAGS=\"$${localCPPFLAGS[@]}\" ") \ - $$([ "$$(echo $${localCXXFLAGS[@]})" ] && echo "CXXFLAGS=\"$${localCXXFLAGS[@]}\" ") \ - $$([ "$$(echo $${localLDFLAGS[@]})" ] && echo "LDFLAGS=\"$${localLDFLAGS[@]}\" ") \ - $$([ "$$(echo $${abi3})" ] && echo "$${abi3} ")" \ - $${global_options}" ; \ - REQUIREMENT=$$(echo $${wheel%% *}) \ - WHEEL_NAME=$${name} \ - WHEEL_VERSION=$${version} \ - ADDITIONAL_CFLAGS="-I$(STAGING_INSTALL_PREFIX)/$(PYTHON_INC_DIR) $${localCFLAGS[@]}" \ - ADDITIONAL_CPPFLAGS="-I$(STAGING_INSTALL_PREFIX)/$(PYTHON_INC_DIR) $${localCPPFLAGS[@]}" \ - ADDITIONAL_CXXFLAGS="-I$(STAGING_INSTALL_PREFIX)/$(PYTHON_INC_DIR) $${localCXXFLAGS[@]}" \ - ADDITIONAL_LDFLAGS="$${localLDFLAGS[@]}" \ - ABI3="$${abi3}" \ - PIP_GLOBAL_OPTION="$${global_options}" \ - $(MAKE) --no-print-directory \ - cross-compile-wheel-$${name} || exit 1 ; \ - done < <(grep -svH -e "^\#" -e "^\$$" $(wildcard $(abspath $(addprefix $(WORK_DIR)/../,$(patsubst %-pure.txt,,$(WHEELS)))))) -else - @$(MSG) "[SKIP] Cross-compiling wheels" + case $${file} in \ + requirements-pure.txt) type=pure ;; \ + requirements-crossenv*.txt) type=crossenv ;; \ + requirements-abi3.txt) type=abi3 ;; \ + *) type=$(WHEEL_DEFAULT_PREFIX) ;; \ + esac ; \ + if [ "$$(grep -s egg <<< $${wheel})" ]; then \ + name=$$(echo $${wheel#*egg=} | cut -f1 -d=) ; \ + else \ + name=$$(echo $${wheel%%[<>=]=*} | sed -E "s/^(abi3|crossenv|pure)://") ; \ + fi ; \ + version=$$(echo $${wheel} | grep -oP '(?<=([<>=]=))[^ ]*' || echo "") ; \ +# $(MSG) requirement: [$${requirement}] ; \ +# $(MSG) file: [$${file}] ; \ +# $(MSG) wheel: [$${wheel}] ; \ +# $(MSG) name: [$${name}] ; \ +# $(MSG) version: [$${version}] ; \ +# $(MSG) type: [$${type}] ; \ + $(MSG) $(MAKE) ARCH=$(ARCH) TCVERSION=$(TCVERSION) REQUIREMENT=\"$${wheel}\" WHEEL_NAME=\"$${name}\" WHEEL_VERSION=\"$${version}\" WHEEL_TYPE=\"$${type}\" wheel ; \ + $(MAKE) ARCH="$(ARCH)" TCVERSION="$(TCVERSION)" REQUIREMENT="$${wheel}" WHEEL_NAME="$${name}" WHEEL_VERSION="$${version}" WHEEL_TYPE="$${type}" wheel --no-print-directory ; \ + done < <(sed '/^\#/d; /^\s*$$/d; s/\s* #.*//' $(wildcard $(abspath $(addprefix $(WORK_DIR)/../,$(WHEELS))))) endif -ifneq ($(filter 1 ON TRUE,$(WHEELS_PURE_PYTHON_PACKAGING_ENABLE)),) - @if [ -s "$(WHEELHOUSE)/$(WHEELS_PURE_PYTHON)" ]; then \ - $(MSG) "Building pure-python" ; \ - export LD= LDSHARED= CPP= NM= CC= AS= RANLIB= CXX= AR= STRIP= OBJDUMP= OBJCOPY= READELF= CFLAGS= CPPFLAGS= CXXFLAGS= LDFLAGS= && \ - $(RUN) \ - PATH="$(abspath $(WORK_DIR)/../../../native/$(PYTHON_PKG_NAME)/work-native/install/usr/local/bin):$(PATH)" \ - LD_LIBRARY_PATH="$(abspath $(WORK_DIR)/../../../native/$(PYTHON_PKG_NAME)/work-native/install/usr/local/lib):$(LD_LIBRARY_PATH)" \ - $(PIP) $(PIP_WHEEL_ARGS) --requirement $(WHEELHOUSE)/$(WHEELS_PURE_PYTHON) ; \ - fi -else - @$(MSG) "[SKIP] Building pure-python" +ifneq ($(filter-out $(addprefix src/,$(notdir $(wildcard $(abspath $(addprefix $(WORK_DIR)/../,$(WHEELS)))))),$(WHEELS)),) + @for requirement in $(filter-out $(addprefix src/,$(notdir $(wildcard $(abspath $(addprefix $(WORK_DIR)/../,$(WHEELS)))))),$(WHEELS)) ; do \ + $(MSG) Processing requirement [$${requirement}] ; \ + wheel=$$(echo $${requirement} | sed -E "s/^(abi3|crossenv|pure)://") ; \ + case $${requirement} in \ + abi3:*) type=abi3 ;; \ + crossenv:*) type=crossenv ;; \ + pure:*) type=pure ;; \ + *) type=$(WHEEL_DEFAULT_PREFIX) ;; \ + esac ; \ + if [ "$$(grep -s egg <<< $${requirement})" ]; then \ + name=$$(echo $${wheel#*egg=} | cut -f1 -d=) ; \ + else \ + name=$$(echo $${wheel%%[<>=]=*} | sed -E "s/^(abi3|crossenv|pure)://") ; \ + fi ; \ + version=$$(echo $${wheel} | grep -oP '(?<=([<>=]=))[^ ]*' || echo "") ; \ +# $(MSG) requirement: [$${requirement}] ; \ +# $(MSG) wheel: [$${wheel}] ; \ +# $(MSG) name: [$${name}] ; \ +# $(MSG) version: [$${version}] ; \ +# $(MSG) type: [$${type}] ; \ + $(MSG) $(MAKE) ARCH=$(ARCH) TCVERSION=$(TCVERSION) REQUIREMENT=\"$${wheel}\" WHEEL_NAME=\"$${name}\" WHEEL_VERSION=\"$${version}\" WHEEL_TYPE=\"$${type}\" wheel ; \ + $(MAKE) ARCH="$(ARCH)" TCVERSION="$(TCVERSION)" REQUIREMENT="$${wheel}" WHEEL_NAME="$${name}" WHEEL_VERSION="$${version}" WHEEL_TYPE="$${type}" wheel --no-print-directory ; \ + done endif -## -## crossenv PATH environment requires a combination of: -## 1) unique PATH variable from $(ENV) -> using merge + dedup macros -## Note: Multiple declarations of ENV += PATH=bla creates confusion in its interpretation. -## Solution implemented fetches all PATH from ENV and combine them in reversed order. -## 2) access to maturin from native/python/.../bin -> ${PYTHON_NATIVE_PATH}/bin -## 3) access to crossenv/bin/cross* tools, mainly cross-pip -> ${CROSSENV_PATH}/bin -## -cross-compile-wheel-%: SHELL:=/bin/bash -cross-compile-wheel-%: - @for crossenv in $(WORK_DIR)/crossenv-$(WHEEL_NAME)-$(WHEEL_VERSION) $(WORK_DIR)/crossenv-$(WHEEL_NAME) $(WORK_DIR)/crossenv ; do \ - [ -d $${crossenv} ] && . $${crossenv}/build/python-cc.mk && break ; \ - done ; \ - if [ -d "$${CROSSENV_PATH}" ] ; then \ - PATH=$(call dedup, $(call merge, $(ENV), PATH, :), :):$${PYTHON_NATIVE_PATH}:$${CROSSENV_PATH}/bin:$${PATH} ; \ - $(MSG) "crossenv: [$${CROSSENV_PATH}]" ; \ - $(MSG) "pip: [$$(which cross-pip)]" ; \ - $(MSG) "maturin: [$$(which maturin)]" ; \ - else \ - echo "ERROR: crossenv not found!" ; \ - exit 2 ; \ - fi ; \ - if [ "$(PIP_GLOBAL_OPTION)" ]; then \ - pip_global_option=$$(echo $(PIP_GLOBAL_OPTION) | sed 's/=\([^ ]*\)/="\1"/g; s/[^ ]*/--global-option=&/g') ; \ - pip_global_option=$${pip_global_option}" --no-use-pep517" ; \ - fi ; \ - $(RUN) $(MSG) \ - _PYTHON_HOST_PLATFORM=\"$(TC_TARGET)\" \ - PATH=$${PATH} \ - CMAKE_TOOLCHAIN_FILE=$${CMAKE_TOOLCHAIN_FILE} \ - MESON_CROSS_FILE=$${MESON_CROSS_FILE} \ - cross-pip \ - $(PIP_WHEEL_ARGS_CROSSENV) \ - $${pip_global_option} \ - --no-build-isolation \ - $(ABI3) \ - $(REQUIREMENT) ; \ - $(RUN) \ - _PYTHON_HOST_PLATFORM="$(TC_TARGET)" \ - PATH=$${PATH} \ - CMAKE_TOOLCHAIN_FILE=$${CMAKE_TOOLCHAIN_FILE} \ - MESON_CROSS_FILE=$${MESON_CROSS_FILE} \ - cross-pip \ - $(PIP_WHEEL_ARGS_CROSSENV) \ - $${pip_global_option} \ - --no-build-isolation \ - $(ABI3) \ - $(REQUIREMENT) - - -post_wheel_target: $(WHEEL_TARGET) install_python_wheel +post_wheel_target: $(WHEEL_TARGET) ifeq ($(wildcard $(WHEEL_COOKIE)),) wheel: $(WHEEL_COOKIE) @@ -268,3 +141,6 @@ $(WHEEL_COOKIE): $(POST_WHEEL_TARGET) else wheel: ; endif + +# endif REQUIREMENT non-empty +endif From 2f662ae3028ade44bede1454b76b6875bec83d8a Mon Sep 17 00:00:00 2001 From: Vincent Fortier Date: Mon, 13 Jan 2025 22:40:36 +0000 Subject: [PATCH 02/24] wheel.mk: Move wheel jq query to master wheel mk file --- mk/spksrc.wheel-download.mk | 33 +++++++++------------------------ mk/spksrc.wheel.mk | 30 +++++++++++++++++++++++++++--- 2 files changed, 36 insertions(+), 27 deletions(-) diff --git a/mk/spksrc.wheel-download.mk b/mk/spksrc.wheel-download.mk index 55909daf003..276fdc0141f 100644 --- a/mk/spksrc.wheel-download.mk +++ b/mk/spksrc.wheel-download.mk @@ -58,35 +58,20 @@ ifeq ($(wildcard $(WHEELHOUSE)),) @$(MSG) Creating wheelhouse directory: $(WHEELHOUSE) @mkdir -p $(WHEELHOUSE) endif - @if [ ! "$(WHEEL_VERSION)" ]; then \ - $(MSG) Fetching latest version available ; \ - query="curl -s https://pypi.org/pypi/$(WHEEL_NAME)/json" ; \ - query+=" | jq -r '.releases[][]" ; \ - query+=" | select(.packagetype==\"sdist\")" ; \ - query+=" | .filename'" ; \ - query+=" | sort -V" ; \ - query+=" | tail -1" ; \ - query+=" | sed -e 's/.tar.gz//g' -e 's/.zip//g'" ; \ - query+=" | awk -F'-' '{print \$$2}'" ; \ - version=$$(eval $${query} 2>/dev/null) ; \ - else \ - version=$(WHEEL_VERSION) ; \ - fi ; \ + @$(MSG) Downloading wheel [$(WHEEL_NAME)], version [$(WHEEL_VERSION)] ; \ # $(MSG) requirement: [$(REQUIREMENT)] ; \ # $(MSG) requirement-grep-egg: [$$(grep -s egg <<< $(REQUIREMENT))] ; \ # $(MSG) name: [$(WHEEL_NAME)] ; \ # $(MSG) type: [$(WHEEL_TYPE)] ; \ -# $(MSG) version-shell: [$${version}] ; \ -# $(MSG) version-make: [$(WHEEL_VERSION)] ; \ +# $(MSG) version: [$(WHEEL_VERSION)] ; \ # $(MSG) type: [$(WHEEL_TYPE)] ; \ - $(MSG) Downloading wheel [$(WHEEL_NAME)], version [$${version}] ; \ if [ "$$(grep -s egg <<< $(REQUIREMENT))" ] ; then \ echo "WARNING: Skipping download URL - Downloaded at build time" ; \ else \ query="curl -s https://pypi.org/pypi/$(WHEEL_NAME)/json" ; \ query+=" | jq -r '.releases[][]" ; \ query+=" | select(.packagetype==\"sdist\")" ; \ - query+=" | select((.filename|test(\"-$${version}.tar.gz\")) or (.filename|test(\"-$${version}.zip\"))) | .url'" ; \ + query+=" | select((.filename|test(\"-$(WHEEL_VERSION).tar.gz\")) or (.filename|test(\"-$(WHEEL_VERSION).zip\"))) | .url'" ; \ outFile=$$(basename $$(eval $${query} 2>/dev/null) 2> $(WHEELHOUSE)/$(WHEELS_LIMITED_API) ; \ + abi3) $(MSG) Adding $(WHEEL_NAME)==$$(WHEEL_VERSION) to wheelhouse/$(WHEELS_LIMITED_API) ; \ + echo $(WHEEL_NAME)==$(WHEEL_VERSION) | sed -e '/^[[:blank:]]*$$\|^#/d' >> $(WHEELHOUSE)/$(WHEELS_LIMITED_API) ; \ ;; \ - crossenv) $(MSG) Adding $(WHEEL_NAME)==$${version} to wheelhouse/$(WHEELS_CROSSENV_COMPILE) ; \ - echo $(WHEEL_NAME)==$${version} | sed -e '/^[[:blank:]]*$$\|^#/d' >> $(WHEELHOUSE)/$(WHEELS_CROSSENV_COMPILE) ; \ + crossenv) $(MSG) Adding $(WHEEL_NAME)==$(WHEEL_VERSION) to wheelhouse/$(WHEELS_CROSSENV_COMPILE) ; \ + echo $(WHEEL_NAME)==$(WHEEL_VERSION) | sed -e '/^[[:blank:]]*$$\|^#/d' >> $(WHEELHOUSE)/$(WHEELS_CROSSENV_COMPILE) ; \ ;; \ - pure) $(MSG) Adding $(WHEEL_NAME)==$${version} to wheelhouse/$(WHEELS_PURE_PYTHON) ; \ - echo $(WHEEL_NAME)==$${version} | sed -e '/^[[:blank:]]*$$\|^#/d' >> $(WHEELHOUSE)/$(WHEELS_PURE_PYTHON) ; \ + pure) $(MSG) Adding $(WHEEL_NAME)==$(WHEEL_VERSION) to wheelhouse/$(WHEELS_PURE_PYTHON) ; \ + echo $(WHEEL_NAME)==$(WHEEL_VERSION) | sed -e '/^[[:blank:]]*$$\|^#/d' >> $(WHEELHOUSE)/$(WHEELS_PURE_PYTHON) ; \ ;; \ *) $(MSG) No type found for wheel [$(REQUIREMENT)] ; \ ;; \ diff --git a/mk/spksrc.wheel.mk b/mk/spksrc.wheel.mk index 87cc33e333d..9531589d314 100644 --- a/mk/spksrc.wheel.mk +++ b/mk/spksrc.wheel.mk @@ -1,8 +1,8 @@ ### Wheel rules -# Create wheel for modules listed in WHEELS. +# Create wheels for modules listed in WHEELS. # If CROSS_COMPILE_WHEELS is set via python-cc.mk, -# wheel are cross-compiled. If not, pure-python -# wheel are created. +# wheels are cross-compiled. If not, pure-python +# wheels are created. # Targets are executed in the following order: # wheel_msg_target @@ -94,6 +94,18 @@ ifneq ($(wildcard $(abspath $(addprefix $(WORK_DIR)/../,$(WHEELS)))),) name=$$(echo $${wheel%%[<>=]=*} | sed -E "s/^(abi3|crossenv|pure)://") ; \ fi ; \ version=$$(echo $${wheel} | grep -oP '(?<=([<>=]=))[^ ]*' || echo "") ; \ + if [ ! "$${version}" ]; then \ + $(MSG) Fetching latest version available ; \ + query="curl -s https://pypi.org/pypi/$${name}/json" ; \ + query+=" | jq -r '.releases[][]" ; \ + query+=" | select(.packagetype==\"sdist\")" ; \ + query+=" | .filename'" ; \ + query+=" | sort -V" ; \ + query+=" | tail -1" ; \ + query+=" | sed -e 's/.tar.gz//g' -e 's/.zip//g'" ; \ + query+=" | awk -F'-' '{print \$$2}'" ; \ + version=$$(eval $${query} 2>/dev/null) ; \ + fi ; \ # $(MSG) requirement: [$${requirement}] ; \ # $(MSG) file: [$${file}] ; \ # $(MSG) wheel: [$${wheel}] ; \ @@ -120,6 +132,18 @@ ifneq ($(filter-out $(addprefix src/,$(notdir $(wildcard $(abspath $(addprefix $ name=$$(echo $${wheel%%[<>=]=*} | sed -E "s/^(abi3|crossenv|pure)://") ; \ fi ; \ version=$$(echo $${wheel} | grep -oP '(?<=([<>=]=))[^ ]*' || echo "") ; \ + if [ ! "$${version}" ]; then \ + $(MSG) Fetching latest version available ; \ + query="curl -s https://pypi.org/pypi/$${name}/json" ; \ + query+=" | jq -r '.releases[][]" ; \ + query+=" | select(.packagetype==\"sdist\")" ; \ + query+=" | .filename'" ; \ + query+=" | sort -V" ; \ + query+=" | tail -1" ; \ + query+=" | sed -e 's/.tar.gz//g' -e 's/.zip//g'" ; \ + query+=" | awk -F'-' '{print \$$2}'" ; \ + version=$$(eval $${query} 2>/dev/null) ; \ + fi ; \ # $(MSG) requirement: [$${requirement}] ; \ # $(MSG) wheel: [$${wheel}] ; \ # $(MSG) name: [$${name}] ; \ From 38c35fd0797ea7a43d60164ed2edb1b0259c429f Mon Sep 17 00:00:00 2001 From: Vincent Fortier Date: Mon, 13 Jan 2025 23:05:26 +0000 Subject: [PATCH 03/24] spk.mk: Add new cokkoe files to wheelclean --- mk/spksrc.spk.mk | 1 + 1 file changed, 1 insertion(+) diff --git a/mk/spksrc.spk.mk b/mk/spksrc.spk.mk index 7cf7592a421..b86059175bd 100644 --- a/mk/spksrc.spk.mk +++ b/mk/spksrc.spk.mk @@ -539,6 +539,7 @@ spkclean: wheelclean: spkclean rm -fr work-*/.wheel_done \ + work-*/.wheel_*_done \ work-*/wheelhouse \ work-*/install/var/packages/**/target/share/wheelhouse @make --no-print-directory dependency-flat | sort -u | grep cross/ | while read depend ; do \ From 892b8c08224fe47d2e0d8e6c985f6b4d48aa5ca6 Mon Sep 17 00:00:00 2001 From: Vincent Fortier Date: Mon, 13 Jan 2025 23:06:09 +0000 Subject: [PATCH 04/24] wheel.mk: Declare crossenv default wheel type --- mk/spksrc.wheel-env.mk | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mk/spksrc.wheel-env.mk b/mk/spksrc.wheel-env.mk index fbf9d01f199..6d5fe0d1596 100644 --- a/mk/spksrc.wheel-env.mk +++ b/mk/spksrc.wheel-env.mk @@ -34,9 +34,6 @@ endif ifeq ($(strip $(WHEELS_PURE_PYTHON)),) WHEELS_PURE_PYTHON = requirements-pure.txt endif -ifeq ($(strip $(WHEELS_CROSS_COMPILE)),) -WHEELS_CROSS_COMPILE = requirements-cross.txt -endif ifeq ($(strip $(WHEELS_CROSSENV_COMPILE)),) WHEELS_CROSSENV_COMPILE = requirements-crossenv.txt endif @@ -46,7 +43,7 @@ ifeq ($(strip $(WHEEL_DEFAULT_PREFIX)),) ifeq ($(strip $(ARCH)),) WHEEL_DEFAULT_PREFIX = pure else -WHEEL_DEFAULT_PREFIX = cross +WHEEL_DEFAULT_PREFIX = crossenv endif endif From ceed5cf716ee180c87cd10937ff80fe632afc671 Mon Sep 17 00:00:00 2001 From: Vincent Fortier Date: Mon, 13 Jan 2025 23:06:55 +0000 Subject: [PATCH 05/24] wheel.mk: Fix grep+sed file looping to avoid loosing filename --- mk/spksrc.wheel.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mk/spksrc.wheel.mk b/mk/spksrc.wheel.mk index 9531589d314..a26ead1cd38 100644 --- a/mk/spksrc.wheel.mk +++ b/mk/spksrc.wheel.mk @@ -114,7 +114,7 @@ ifneq ($(wildcard $(abspath $(addprefix $(WORK_DIR)/../,$(WHEELS)))),) # $(MSG) type: [$${type}] ; \ $(MSG) $(MAKE) ARCH=$(ARCH) TCVERSION=$(TCVERSION) REQUIREMENT=\"$${wheel}\" WHEEL_NAME=\"$${name}\" WHEEL_VERSION=\"$${version}\" WHEEL_TYPE=\"$${type}\" wheel ; \ $(MAKE) ARCH="$(ARCH)" TCVERSION="$(TCVERSION)" REQUIREMENT="$${wheel}" WHEEL_NAME="$${name}" WHEEL_VERSION="$${version}" WHEEL_TYPE="$${type}" wheel --no-print-directory ; \ - done < <(sed '/^\#/d; /^\s*$$/d; s/\s* #.*//' $(wildcard $(abspath $(addprefix $(WORK_DIR)/../,$(WHEELS))))) + done < <(grep -svH -e "^\#" -e "^\$$" $(wildcard $(abspath $(addprefix $(WORK_DIR)/../,$(WHEELS)))) | sed 's/\s* #.*//') endif ifneq ($(filter-out $(addprefix src/,$(notdir $(wildcard $(abspath $(addprefix $(WORK_DIR)/../,$(WHEELS)))))),$(WHEELS)),) @for requirement in $(filter-out $(addprefix src/,$(notdir $(wildcard $(abspath $(addprefix $(WORK_DIR)/../,$(WHEELS)))))),$(WHEELS)) ; do \ From 57b8cbbc6d34ebc12d6618e905908283dbcf1a40 Mon Sep 17 00:00:00 2001 From: Vincent Fortier Date: Mon, 13 Jan 2025 23:07:29 +0000 Subject: [PATCH 06/24] wheel.mk: Avoid downloading if pure wheel packaging is disabled --- mk/spksrc.wheel-download.mk | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mk/spksrc.wheel-download.mk b/mk/spksrc.wheel-download.mk index 276fdc0141f..ec72d314756 100644 --- a/mk/spksrc.wheel-download.mk +++ b/mk/spksrc.wheel-download.mk @@ -67,6 +67,8 @@ endif # $(MSG) type: [$(WHEEL_TYPE)] ; \ if [ "$$(grep -s egg <<< $(REQUIREMENT))" ] ; then \ echo "WARNING: Skipping download URL - Downloaded at build time" ; \ + elif [ "$(WHEEL_TYPE)" = "pure" ] && [ "$(WHEELS_PURE_PYTHON_PACKAGING_ENABLE)" = "1" ]; then \ + echo "WARNING: Skipping download - pure python packaging disabled" ; \ else \ query="curl -s https://pypi.org/pypi/$(WHEEL_NAME)/json" ; \ query+=" | jq -r '.releases[][]" ; \ From 5a7def6c6d84fbe9ef0f2f052b7cc76a0a63b58d Mon Sep 17 00:00:00 2001 From: Vincent Fortier Date: Tue, 14 Jan 2025 23:57:23 +0000 Subject: [PATCH 07/24] common.mk: Add PIP_NATIVE to use our own pip native always --- mk/spksrc.common.mk | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mk/spksrc.common.mk b/mk/spksrc.common.mk index 27b27744445..abbb8d8cd85 100644 --- a/mk/spksrc.common.mk +++ b/mk/spksrc.common.mk @@ -32,6 +32,9 @@ PIP ?= pip # System default pip outside from build environment PIP_SYSTEM = $(shell which pip) +# System default pip outside from build environment +PIP_NATIVE = $(WORK_DIR)/../../../native/$(or $(PYTHON_PACKAGE),$(SPK_NAME))/work-native/install/usr/local/bin/pip + # Why ask for the same thing twice? Always cache downloads PIP_CACHE_OPT ?= --find-links $(PIP_DISTRIB_DIR) --cache-dir $(PIP_CACHE_DIR) PIP_WHEEL_ARGS = wheel --disable-pip-version-check --no-binary :all: $(PIP_CACHE_OPT) --no-deps --wheel-dir $(WHEELHOUSE) From 0a84ca1b16a2a90e8eddb7877faa2f9d7de155e9 Mon Sep 17 00:00:00 2001 From: Vincent Fortier Date: Tue, 14 Jan 2025 23:59:13 +0000 Subject: [PATCH 08/24] crossenv.mk: Missing call to pre_crossenv_target --- mk/spksrc.crossenv.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mk/spksrc.crossenv.mk b/mk/spksrc.crossenv.mk index 900cde068ec..5285b7810d7 100644 --- a/mk/spksrc.crossenv.mk +++ b/mk/spksrc.crossenv.mk @@ -166,7 +166,7 @@ export PYTHONPATH = $(PYTHON_LIB_NATIVE):$(PYTHON_STAGING_INSTALL_PREFIX)/lib/py # >>> sys.path # build_crossenv_target: SHELL:=/bin/bash -build_crossenv_target: $(CROSSENV_PATH)/build/python-cc.mk +build_crossenv_target: pre_crossenv_target $(CROSSENV_PATH)/build/python-cc.mk @$(MSG) $$(date +%Y%m%d-%H%M%S) MAKELEVEL: $(MAKELEVEL), PARALLEL_MAKE: $(PARALLEL_MAKE), ARCH: $(ARCH)-$(TCVERSION), CROSSENV: $(WHEEL) >> $(PSTAT_LOG) @$(MSG) Python sources: $(wildcard $(PYTHON_WORK_DIR)/Python-[0-9]*) @$(MSG) crossenv wheel packages: $(CROSSENV_DEFAULT_PIP), $(CROSSENV_DEFAULT_SETUPTOOLS), $(CROSSENV_DEFAULT_WHEEL) From 59482d74172d7fb95ea23129bf33733bbc3e2e3c Mon Sep 17 00:00:00 2001 From: Vincent Fortier Date: Tue, 14 Jan 2025 23:59:43 +0000 Subject: [PATCH 09/24] spk.mk: Add ability to clean one specific wheel --- mk/spksrc.spk.mk | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/mk/spksrc.spk.mk b/mk/spksrc.spk.mk index b86059175bd..2f69c4430fc 100644 --- a/mk/spksrc.spk.mk +++ b/mk/spksrc.spk.mk @@ -553,10 +553,15 @@ wheelclean: spkclean fi ; \ done -wheelcleancache: wheelclean +wheelclean-%: spkclean + rm -f work-*/.wheel_done \ + work-*/wheelhouse/$*-*.whl + find work-* -type f -regex '.*\.wheel_\(download\|compile\|install\)-$*_done' -exec rm -f {} \; + +wheelcleancache: rm -fr work-*/pip -wheelcleanall: wheelcleancache +wheelcleanall: wheelcleancache wheelclean rm -fr ../../distrib/pip crossenvclean: wheelclean From 4c50a767106895bff5eaf74ea72c863894ff77f0 Mon Sep 17 00:00:00 2001 From: Vincent Fortier Date: Wed, 15 Jan 2025 00:00:34 +0000 Subject: [PATCH 10/24] wheel.mk: Enable compile and install steps for wheel building --- mk/spksrc.wheel-compile.mk | 148 ++++++++++++++++++++++++++++++++++++ mk/spksrc.wheel-download.mk | 32 +------- mk/spksrc.wheel-install.mk | 78 +++++++++++++++++++ mk/spksrc.wheel.mk | 27 +++---- 4 files changed, 241 insertions(+), 44 deletions(-) create mode 100644 mk/spksrc.wheel-compile.mk create mode 100644 mk/spksrc.wheel-install.mk diff --git a/mk/spksrc.wheel-compile.mk b/mk/spksrc.wheel-compile.mk new file mode 100644 index 00000000000..b5a8ed7df2d --- /dev/null +++ b/mk/spksrc.wheel-compile.mk @@ -0,0 +1,148 @@ +### Wheel rules +# Compile wheels for modules listed in WHEELS. +# +# Targets are executed in the following order: +# wheel_compile_msg_target +# pre_wheel_compile_target (override with PRE_WHEEL_COMPILE_TARGET) +# wheel_compile_target (override with WHEEL_COMPILE_TARGET) +# post_wheel_compile_target (override with POST_WHEEL_COMPILE_TARGET) +# Variables: +# REQUIREMENT Requirement formatted wheel information +# WHEEL_NAME Name of wheel to process +# WHEEL_VERSION Version of wheel to process (can be empty) +# WHEEL_TYPE Type of wheel to process (abi3, crossenv, pure) + +ifeq ($(WHEEL_VERSION),) +WHEEL_COMPILE_COOKIE = $(WORK_DIR)/.$(COOKIE_PREFIX)wheel_compile-$(WHEEL_NAME)_done +else +WHEEL_COMPILE_COOKIE = $(WORK_DIR)/.$(COOKIE_PREFIX)wheel_compile-$(WHEEL_NAME)-$(WHEEL_VERSION)_done +endif + +## + +ifeq ($(strip $(PRE_WHEEL_COMPILE_TARGET)),) +PRE_WHEEL_COMPILE_TARGET = pre_wheel_compile_target +else +$(PRE_WHEEL_COMPILE_TARGET): wheel_compile_msg_target +endif +ifeq ($(strip $(WHEEL_COMPILE_TARGET)),) +WHEEL_COMPILE_TARGET = wheel_compile_target +else +$(WHEEL_COMPILE_TARGET): $(BUILD_WHEEL_COMPILE_TARGET) +endif +ifeq ($(strip $(POST_WHEEL_COMPILE_TARGET)),) +POST_WHEEL_COMPILE_TARGET = post_wheel_compile_target +else +$(POST_WHEEL_COMPILE_TARGET): $(WHEEL_COMPILE_TARGET) +endif + +wheel_compile_msg_target: + @$(MSG) "Processing wheels of $(NAME)" + +pre_wheel_compile_target: wheel_compile_msg_target + +wheel_compile_target: SHELL:=/bin/bash +wheel_compile_target: +ifeq ($(wildcard $(WHEELHOUSE)),) + @$(MSG) Creating wheelhouse directory: $(WHEELHOUSE) + @mkdir -p $(WHEELHOUSE) +endif + @$(MSG) Compiling wheel [$(WHEEL_NAME)], version [$(WHEEL_VERSION)], type [$(WHEEL_TYPE)] +ifneq ($(WHEEL_TYPE),pure) + @$(MSG) $(MAKE) WHEEL=$(WHEEL_NAME)==$(WHEEL_VERSION) crossenv-$(ARCH)-$(TCVERSION) ; \ + MAKEFLAGS= $(MAKE) WHEEL=$(WHEEL_NAME)==$(WHEEL_VERSION) crossenv-$(ARCH)-$(TCVERSION) --no-print-directory ; \ + [ "$(WHEEL_TYPE)" = "$(WHEELS_LIMITED_API)" ] && abi3="--build-option=--py-limited-api=$(PYTHON_LIMITED_API)" || abi3="" ; \ + global_options=$$(echo $(WHEELS_BUILD_ARGS) | sed -e 's/ \[/\n\[/g' | grep -i $(WHEEL_NAME) | cut -f2 -d] | xargs) ; \ + localCFLAGS=($$(echo $(WHEELS_CFLAGS) | sed -e 's/ \[/\n\[/g' | grep -i $(WHEEL_NAME) | cut -f2 -d] | xargs)) ; \ + localLDFLAGS=($$(echo $(WHEELS_LDFLAGS) | sed -e 's/ \[/\n\[/g' | grep -i $(WHEEL_NAME) | cut -f2 -d] | xargs)) ; \ + localCPPFLAGS=($$(echo $(WHEELS_CPPFLAGS) | sed -e 's/ \[/\n\[/g' | grep -i $(WHEEL_NAME) | cut -f2 -d] | xargs)) ; \ + localCXXFLAGS=($$(echo $(WHEELS_CXXFLAGS) | sed -e 's/ \[/\n\[/g' | grep -i $(WHEEL_NAME) | cut -f2 -d] | xargs)) ; \ + $(MSG) pip build [$(WHEEL_NAME)], version: [$(WHEEL_VERSION)] \ + $$([ "$$(echo $${localCFLAGS[@]})" ] && echo "CFLAGS=\"$${localCFLAGS[@]}\" ") \ + $$([ "$$(echo $${localCPPFLAGS[@]})" ] && echo "CPPFLAGS=\"$${localCPPFLAGS[@]}\" ") \ + $$([ "$$(echo $${localCXXFLAGS[@]})" ] && echo "CXXFLAGS=\"$${localCXXFLAGS[@]}\" ") \ + $$([ "$$(echo $${localLDFLAGS[@]})" ] && echo "LDFLAGS=\"$${localLDFLAGS[@]}\" ") \ + $$([ "$$(echo $${abi3})" ] && echo "$${abi3} ")" \ + $${global_options}" ; \ + REQUIREMENT=$(REQUIREMENT) \ + WHEEL_NAME=$(WHEEL_NAME) \ + WHEEL_VERSION=$(WHEEL_VERSION) \ + ADDITIONAL_CFLAGS="-I$(STAGING_INSTALL_PREFIX)/$(PYTHON_INC_DIR) $${localCFLAGS[@]}" \ + ADDITIONAL_CPPFLAGS="-I$(STAGING_INSTALL_PREFIX)/$(PYTHON_INC_DIR) $${localCPPFLAGS[@]}" \ + ADDITIONAL_CXXFLAGS="-I$(STAGING_INSTALL_PREFIX)/$(PYTHON_INC_DIR) $${localCXXFLAGS[@]}" \ + ADDITIONAL_LDFLAGS="$${localLDFLAGS[@]}" \ + ABI3="$${abi3}" \ + PIP_GLOBAL_OPTION="$${global_options}" \ + $(MAKE) --no-print-directory \ + cross-compile-wheel-$(WHEEL_NAME) || exit 1 +else ifneq ($(filter 1 ON TRUE,$(WHEELS_PURE_PYTHON_PACKAGING_ENABLE)),) + @if [ -s "$(WHEELHOUSE)/$(WHEELS_PURE_PYTHON)" ]; then \ + export LD= LDSHARED= CPP= NM= CC= AS= RANLIB= CXX= AR= STRIP= OBJDUMP= OBJCOPY= READELF= CFLAGS= CPPFLAGS= CXXFLAGS= LDFLAGS= && \ + $(RUN) \ + PATH="$(abspath $(WORK_DIR)/../../../native/$(PYTHON_PKG_NAME)/work-native/install/usr/local/bin):$(PATH)" \ + LD_LIBRARY_PATH="$(abspath $(WORK_DIR)/../../../native/$(PYTHON_PKG_NAME)/work-native/install/usr/local/lib):$(LD_LIBRARY_PATH)" \ + $(MSG) $(PIP_NATIVE) $(PIP_WHEEL_ARGS) $(REQUIREMENT) ; \ + $(PIP_NATIVE) $(PIP_WHEEL_ARGS) $(REQUIREMENT) ; \ + fi +endif + +## +## crossenv PATH environment requires a combination of: +## 1) unique PATH variable from $(ENV) -> using merge + dedup macros +## Note: Multiple declarations of ENV += PATH=bla creates confusion in its interpretation. +## Solution implemented fetches all PATH from ENV and combine them in reversed order. +## 2) access to maturin from native/python/.../bin -> ${PYTHON_NATIVE_PATH}/bin +## 3) access to crossenv/bin/cross* tools, mainly cross-pip -> ${CROSSENV_PATH}/bin +## +cross-compile-wheel-%: SHELL:=/bin/bash +cross-compile-wheel-%: + @for crossenv in $(WORK_DIR)/crossenv-$(WHEEL_NAME)-$(WHEEL_VERSION) $(WORK_DIR)/crossenv-$(WHEEL_NAME) $(WORK_DIR)/crossenv ; do \ + [ -d $${crossenv} ] && . $${crossenv}/build/python-cc.mk && break ; \ + done ; \ + if [ -d "$${CROSSENV_PATH}" ] ; then \ + PATH=$(call dedup, $(call merge, $(ENV), PATH, :), :):$${PYTHON_NATIVE_PATH}:$${CROSSENV_PATH}/bin:$${PATH} ; \ + $(MSG) "crossenv: [$${CROSSENV_PATH}]" ; \ + $(MSG) "pip: [$$(which cross-pip)]" ; \ + $(MSG) "maturin: [$$(which maturin)]" ; \ + else \ + echo "ERROR: crossenv not found!" ; \ + exit 2 ; \ + fi ; \ + if [ "$(PIP_GLOBAL_OPTION)" ]; then \ + pip_global_option=$$(echo $(PIP_GLOBAL_OPTION) | sed 's/=\([^ ]*\)/="\1"/g; s/[^ ]*/--global-option=&/g') ; \ + pip_global_option=$${pip_global_option}" --no-use-pep517" ; \ + fi ; \ + $(RUN) $(MSG) \ + _PYTHON_HOST_PLATFORM=\"$(TC_TARGET)\" \ + PATH=$${PATH} \ + CMAKE_TOOLCHAIN_FILE=$${CMAKE_TOOLCHAIN_FILE} \ + MESON_CROSS_FILE=$${MESON_CROSS_FILE} \ + cross-pip \ + $(PIP_WHEEL_ARGS_CROSSENV) \ + $${pip_global_option} \ + --no-build-isolation \ + $(ABI3) \ + $(REQUIREMENT) ; \ + $(RUN) \ + _PYTHON_HOST_PLATFORM="$(TC_TARGET)" \ + PATH=$${PATH} \ + CMAKE_TOOLCHAIN_FILE=$${CMAKE_TOOLCHAIN_FILE} \ + MESON_CROSS_FILE=$${MESON_CROSS_FILE} \ + cross-pip \ + $(PIP_WHEEL_ARGS_CROSSENV) \ + $${pip_global_option} \ + --no-build-isolation \ + $(ABI3) \ + $(REQUIREMENT) + +post_wheel_compile_target: $(WHEEL_COMPILE_TARGET) + +ifeq ($(wildcard $(WHEEL_COMPILE_COOKIE)),) +wheel_compile: $(WHEEL_COMPILE_COOKIE) + +$(WHEEL_COMPILE_COOKIE): $(POST_WHEEL_COMPILE_TARGET) + $(create_target_dir) + @touch -f $@ +else +wheel_compile: ; +endif diff --git a/mk/spksrc.wheel-download.mk b/mk/spksrc.wheel-download.mk index ec72d314756..991ae3cfb63 100644 --- a/mk/spksrc.wheel-download.mk +++ b/mk/spksrc.wheel-download.mk @@ -1,9 +1,6 @@ ### Wheel rules -# Create wheels for modules listed in WHEELS. -# If CROSS_COMPILE_WHEELS is set via python-cc.mk, -# wheels are cross-compiled. If not, pure-python -# wheels are created. - +# Download wheels for modules listed in WHEELS. +# # Targets are executed in the following order: # wheel_download_msg_target # pre_wheel_download_target (override with PRE_WHEEL_DOWNLOAD_TARGET) @@ -53,10 +50,6 @@ endif ifeq ($(wildcard $(PIP_CACHE_DIR)),) @$(MSG) Creating pip caching directory: $(PIP_CACHE_DIR) @mkdir -p $(PIP_CACHE_DIR) -endif -ifeq ($(wildcard $(WHEELHOUSE)),) - @$(MSG) Creating wheelhouse directory: $(WHEELHOUSE) - @mkdir -p $(WHEELHOUSE) endif @$(MSG) Downloading wheel [$(WHEEL_NAME)], version [$(WHEEL_VERSION)] ; \ # $(MSG) requirement: [$(REQUIREMENT)] ; \ @@ -67,7 +60,7 @@ endif # $(MSG) type: [$(WHEEL_TYPE)] ; \ if [ "$$(grep -s egg <<< $(REQUIREMENT))" ] ; then \ echo "WARNING: Skipping download URL - Downloaded at build time" ; \ - elif [ "$(WHEEL_TYPE)" = "pure" ] && [ "$(WHEELS_PURE_PYTHON_PACKAGING_ENABLE)" = "1" ]; then \ + elif [ "$(WHEEL_TYPE)" = "pure" ] && [ ! "$(WHEELS_PURE_PYTHON_PACKAGING_ENABLE)" = "1" ]; then \ echo "WARNING: Skipping download - pure python packaging disabled" ; \ else \ query="curl -s https://pypi.org/pypi/$(WHEEL_NAME)/json" ; \ @@ -84,27 +77,10 @@ endif wget --secure-protocol=TLSv1_2 -nv -O $(PIP_DISTRIB_DIR)/$${outFile}.part -nc $$(eval $${query}) ; \ mv $(PIP_DISTRIB_DIR)/$${outFile}.part $(PIP_DISTRIB_DIR)/$${outFile} ; \ fi ; \ - fi ; \ - case $(WHEEL_TYPE) in \ - abi3) $(MSG) Adding $(WHEEL_NAME)==$$(WHEEL_VERSION) to wheelhouse/$(WHEELS_LIMITED_API) ; \ - echo $(WHEEL_NAME)==$(WHEEL_VERSION) | sed -e '/^[[:blank:]]*$$\|^#/d' >> $(WHEELHOUSE)/$(WHEELS_LIMITED_API) ; \ - ;; \ - crossenv) $(MSG) Adding $(WHEEL_NAME)==$(WHEEL_VERSION) to wheelhouse/$(WHEELS_CROSSENV_COMPILE) ; \ - echo $(WHEEL_NAME)==$(WHEEL_VERSION) | sed -e '/^[[:blank:]]*$$\|^#/d' >> $(WHEELHOUSE)/$(WHEELS_CROSSENV_COMPILE) ; \ - ;; \ - pure) $(MSG) Adding $(WHEEL_NAME)==$(WHEEL_VERSION) to wheelhouse/$(WHEELS_PURE_PYTHON) ; \ - echo $(WHEEL_NAME)==$(WHEEL_VERSION) | sed -e '/^[[:blank:]]*$$\|^#/d' >> $(WHEELHOUSE)/$(WHEELS_PURE_PYTHON) ; \ - ;; \ - *) $(MSG) No type found for wheel [$(REQUIREMENT)] ; \ - ;; \ - esac - @for file in $$(ls -1 $(WHEELHOUSE)/requirements-*.txt) ; do \ - sort -u -o $${file}{,} ; \ - done + fi post_wheel_download_target: $(WHEEL_DOWNLOAD_TARGET) - ifeq ($(wildcard $(WHEEL_DOWNLOAD_COOKIE)),) wheel_download: $(WHEEL_DOWNLOAD_COOKIE) diff --git a/mk/spksrc.wheel-install.mk b/mk/spksrc.wheel-install.mk new file mode 100644 index 00000000000..20320eb3b05 --- /dev/null +++ b/mk/spksrc.wheel-install.mk @@ -0,0 +1,78 @@ +### Wheel rules +# Install wheels for modules listed in WHEELS. +# +# Targets are executed in the following order: +# wheel_install_msg_target +# pre_wheel_install_target (override with PRE_WHEEL_INSTALL_TARGET) +# wheel_install_target (override with WHEEL_INSTALL_TARGET) +# post_wheel_install_target (override with POST_WHEEL_INSTALL_TARGET) +# Variables: +# REQUIREMENT Requirement formatted wheel information +# WHEEL_NAME Name of wheel to process +# WHEEL_VERSION Version of wheel to process (can be empty) +# WHEEL_TYPE Type of wheel to process (abi3, crossenv, pure) + +ifeq ($(WHEEL_VERSION),) +WHEEL_INSTALL_COOKIE = $(WORK_DIR)/.$(COOKIE_PREFIX)wheel_install-$(WHEEL_NAME)_done +else +WHEEL_INSTALL_COOKIE = $(WORK_DIR)/.$(COOKIE_PREFIX)wheel_install-$(WHEEL_NAME)-$(WHEEL_VERSION)_done +endif + +## + +ifeq ($(strip $(PRE_WHEEL_INSTALL_TARGET)),) +PRE_WHEEL_INSTALL_TARGET = pre_wheel_install_target +else +$(PRE_WHEEL_INSTALL_TARGET): wheel_install_msg_target +endif +ifeq ($(strip $(WHEEL_INSTALL_TARGET)),) +WHEEL_INSTALL_TARGET = wheel_install_target +else +$(WHEEL_INSTALL_TARGET): $(BUILD_WHEEL_INSTALL_TARGET) +endif +ifeq ($(strip $(POST_WHEEL_INSTALL_TARGET)),) +POST_WHEEL_INSTALL_TARGET = post_wheel_install_target +else +$(POST_WHEEL_INSTALL_TARGET): $(WHEEL_INSTALL_TARGET) +endif + +wheel_install_msg_target: + @$(MSG) "Processing wheels of $(NAME)" + +pre_wheel_install_target: wheel_install_msg_target + +wheel_install_target: SHELL:=/bin/bash +wheel_install_target: +ifeq ($(wildcard $(WHEELHOUSE)),) + @$(MSG) Creating wheelhouse directory: $(WHEELHOUSE) + @mkdir -p $(WHEELHOUSE) +endif + @$(MSG) Installing wheel [$(WHEEL_NAME)], version [$(WHEEL_VERSION)], type [$(WHEEL_TYPE)] ; \ + case $(WHEEL_TYPE) in \ + abi3) $(MSG) Adding $(WHEEL_NAME)==$$(WHEEL_VERSION) to wheelhouse/$(WHEELS_LIMITED_API) ; \ + echo $(WHEEL_NAME)==$(WHEEL_VERSION) | sed -e '/^[[:blank:]]*$$\|^#/d' >> $(WHEELHOUSE)/$(WHEELS_LIMITED_API) ; \ + ;; \ + crossenv) $(MSG) Adding $(WHEEL_NAME)==$(WHEEL_VERSION) to wheelhouse/$(WHEELS_CROSSENV_COMPILE) ; \ + echo $(WHEEL_NAME)==$(WHEEL_VERSION) | sed -e '/^[[:blank:]]*$$\|^#/d' >> $(WHEELHOUSE)/$(WHEELS_CROSSENV_COMPILE) ; \ + ;; \ + pure) $(MSG) Adding $(WHEEL_NAME)==$(WHEEL_VERSION) to wheelhouse/$(WHEELS_PURE_PYTHON) ; \ + echo $(WHEEL_NAME)==$(WHEEL_VERSION) | sed -e '/^[[:blank:]]*$$\|^#/d' >> $(WHEELHOUSE)/$(WHEELS_PURE_PYTHON) ; \ + ;; \ + *) $(MSG) No type found for wheel [$(REQUIREMENT)] ; \ + ;; \ + esac + @for file in $$(ls -1 $(WHEELHOUSE)/requirements-*.txt) ; do \ + sort -u -o $${file}{,} ; \ + done + +post_wheel_install_target: $(WHEEL_INSTALL_TARGET) + +ifeq ($(wildcard $(WHEEL_INSTALL_COOKIE)),) +wheel_install: $(WHEEL_INSTALL_COOKIE) + +$(WHEEL_INSTALL_COOKIE): $(POST_WHEEL_INSTALL_TARGET) + $(create_target_dir) + @touch -f $@ +else +wheel_install: ; +endif diff --git a/mk/spksrc.wheel.mk b/mk/spksrc.wheel.mk index a26ead1cd38..9fe922c9ca1 100644 --- a/mk/spksrc.wheel.mk +++ b/mk/spksrc.wheel.mk @@ -1,9 +1,9 @@ ### Wheel rules -# Create wheels for modules listed in WHEELS. -# If CROSS_COMPILE_WHEELS is set via python-cc.mk, -# wheels are cross-compiled. If not, pure-python -# wheels are created. - +# Process wheels for modules listed in WHEELS. +# 1. wheel_download +# 2. wheel_compile +# 3. wheel_install +# # Targets are executed in the following order: # wheel_msg_target # pre_wheel_target (override with PRE_WHEEL_TARGET) @@ -29,21 +29,16 @@ include ../../mk/spksrc.cross-meson-env.mk include ../../mk/spksrc.wheel-download.mk -#wheel_configure: wheel_download -#include ../../mk/spksrc.wheel-configure.mk -# -#wheel_compile: wheel_configure -#include ../../mk/spksrc.wheel-compile.mk -# -#wheel_install: wheel_compile -#include ../../mk/spksrc.wheel-install.mk -# -#all: wheel_install +wheel_compile: wheel_download +include ../../mk/spksrc.wheel-compile.mk + +wheel_install: wheel_compile +include ../../mk/spksrc.wheel-install.mk ## ifneq ($(strip $(REQUIREMENT)),) -wheel: wheel_download +wheel: wheel_install else ifeq ($(strip $(PRE_WHEEL_TARGET)),) From ab02b4b0dea78016a29c57d6e08ce7d83a8d7d27 Mon Sep 17 00:00:00 2001 From: Vincent Fortier Date: Wed, 15 Jan 2025 00:10:28 +0000 Subject: [PATCH 11/24] wheel.mk: Re-enable install_python_wheel towards staging directory --- mk/spksrc.wheel-env.mk | 27 --------------------------- mk/spksrc.wheel-install.mk | 27 +++++++++++++++++++++++++++ mk/spksrc.wheel.mk | 2 +- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/mk/spksrc.wheel-env.mk b/mk/spksrc.wheel-env.mk index 6d5fe0d1596..15c889c499c 100644 --- a/mk/spksrc.wheel-env.mk +++ b/mk/spksrc.wheel-env.mk @@ -86,30 +86,3 @@ endif ifeq ($(findstring $(ARCH),$(i686_ARCHS)),$(ARCH)) PYTHON_ARCH = i686 endif - -install_python_wheel: - @if [ -d "$(WHEELHOUSE)" ] ; then \ - mkdir -p $(STAGING_INSTALL_WHEELHOUSE) ; \ - cd $(WHEELHOUSE) ; \ - if stat -t requirements*.txt >/dev/null 2>&1; then \ - $(MSG) Copying $(WHEELS_DEFAULT) to wheelhouse ; \ - cp requirements*.txt $(STAGING_INSTALL_WHEELHOUSE) ; \ - cat requirements*.txt >> $(STAGING_INSTALL_WHEELHOUSE)/$(WHEELS_DEFAULT) ; \ - sed -i -e '/^#/! s/^.*egg=//g' $(STAGING_INSTALL_WHEELHOUSE)/requirements*.txt ; \ - sort -u -o $(STAGING_INSTALL_WHEELHOUSE)/$(WHEELS_DEFAULT) $(STAGING_INSTALL_WHEELHOUSE)/$(WHEELS_DEFAULT) ; \ - else \ - $(MSG) [SKIP] Copying $(WHEELS_DEFAULT) to wheelhouse ; \ - fi ; \ - if stat -t *.whl >/dev/null 2>&1; then \ - for w in *.whl; do \ - if echo $${w} | grep -iq "-none-any\.whl" ; then \ - _new_name=$$(echo $$w | cut -d"-" -f -3)-none-any.whl ; \ - else \ - _new_name=$$(echo $$w | sed -E "s/(.*-).*(linux_).*(\.whl)/\1\2$(PYTHON_ARCH)\3/") ; \ - fi ; \ - $(MSG) Copying to wheelhouse: $$_new_name ; \ - cp -f $$w $(STAGING_INSTALL_WHEELHOUSE)/$$_new_name ; \ - done ; \ - fi ; \ - fi - diff --git a/mk/spksrc.wheel-install.mk b/mk/spksrc.wheel-install.mk index 20320eb3b05..19d0bb16133 100644 --- a/mk/spksrc.wheel-install.mk +++ b/mk/spksrc.wheel-install.mk @@ -65,6 +65,33 @@ endif sort -u -o $${file}{,} ; \ done +install_python_wheel: + @if [ -d "$(WHEELHOUSE)" ] ; then \ + mkdir -p $(STAGING_INSTALL_WHEELHOUSE) ; \ + cd $(WHEELHOUSE) ; \ + if stat -t requirements*.txt >/dev/null 2>&1; then \ + $(MSG) Copying $(WHEELS_DEFAULT) to wheelhouse ; \ + cp requirements*.txt $(STAGING_INSTALL_WHEELHOUSE) ; \ + cat requirements*.txt >> $(STAGING_INSTALL_WHEELHOUSE)/$(WHEELS_DEFAULT) ; \ + sed -i -e '/^#/! s/^.*egg=//g' $(STAGING_INSTALL_WHEELHOUSE)/requirements*.txt ; \ + sort -u -o $(STAGING_INSTALL_WHEELHOUSE)/$(WHEELS_DEFAULT) $(STAGING_INSTALL_WHEELHOUSE)/$(WHEELS_DEFAULT) ; \ + else \ + $(MSG) [SKIP] Copying $(WHEELS_DEFAULT) to wheelhouse ; \ + fi ; \ + if stat -t *.whl >/dev/null 2>&1; then \ + for w in *.whl; do \ + if echo $${w} | grep -iq "-none-any\.whl" ; then \ + _new_name=$$(echo $$w | cut -d"-" -f -3)-none-any.whl ; \ + else \ + _new_name=$$(echo $$w | sed -E "s/(.*-).*(linux_).*(\.whl)/\1\2$(PYTHON_ARCH)\3/") ; \ + fi ; \ + $(MSG) Copying to wheelhouse: $$_new_name ; \ + cp -f $$w $(STAGING_INSTALL_WHEELHOUSE)/$$_new_name ; \ + done ; \ + fi ; \ + fi + + post_wheel_install_target: $(WHEEL_INSTALL_TARGET) ifeq ($(wildcard $(WHEEL_INSTALL_COOKIE)),) diff --git a/mk/spksrc.wheel.mk b/mk/spksrc.wheel.mk index 9fe922c9ca1..660b52f3cde 100644 --- a/mk/spksrc.wheel.mk +++ b/mk/spksrc.wheel.mk @@ -149,7 +149,7 @@ ifneq ($(filter-out $(addprefix src/,$(notdir $(wildcard $(abspath $(addprefix $ done endif -post_wheel_target: $(WHEEL_TARGET) +post_wheel_target: $(WHEEL_TARGET) install_python_wheel ifeq ($(wildcard $(WHEEL_COOKIE)),) wheel: $(WHEEL_COOKIE) From 3ca2223e91dd452818d45054ee1601398c31a904 Mon Sep 17 00:00:00 2001 From: Vincent Fortier Date: Wed, 15 Jan 2025 00:13:08 +0000 Subject: [PATCH 12/24] Bump package version to initiate build wheel testing --- spk/bazarr/Makefile | 2 +- spk/borgbackup/Makefile | 2 +- spk/deluge/Makefile | 2 +- spk/python311-wheels/Makefile | 2 +- spk/python312-wheels/Makefile | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/spk/bazarr/Makefile b/spk/bazarr/Makefile index f948f40a856..be4ed083544 100644 --- a/spk/bazarr/Makefile +++ b/spk/bazarr/Makefile @@ -1,6 +1,6 @@ SPK_NAME = bazarr SPK_VERS = 1.4.5 -SPK_REV = 15 +SPK_REV = 16 SPK_ICON = src/bazarr.png PYTHON_PACKAGE = python311 diff --git a/spk/borgbackup/Makefile b/spk/borgbackup/Makefile index d32281a08f4..6650db42540 100644 --- a/spk/borgbackup/Makefile +++ b/spk/borgbackup/Makefile @@ -1,6 +1,6 @@ SPK_NAME = borgbackup SPK_VERS = 1.4.0 -SPK_REV = 15 +SPK_REV = 16 SPK_ICON = src/borgbackup.png PYTHON_PACKAGE = python311 diff --git a/spk/deluge/Makefile b/spk/deluge/Makefile index 70b5699fc52..6dcf5e29d74 100644 --- a/spk/deluge/Makefile +++ b/spk/deluge/Makefile @@ -1,6 +1,6 @@ SPK_NAME = deluge SPK_VERS = 2.1.1.127 -SPK_REV = 24 +SPK_REV = 25 SPK_ICON = src/deluge.png PYTHON_PACKAGE = python311 diff --git a/spk/python311-wheels/Makefile b/spk/python311-wheels/Makefile index f7ec63a6580..05564925860 100644 --- a/spk/python311-wheels/Makefile +++ b/spk/python311-wheels/Makefile @@ -1,7 +1,7 @@ SPK_NAME = python311-wheels SPK_VERS = 1.0 SPK_VERS_MAJOR_MINOR = $(word 1,$(subst ., ,$(SPK_VERS))).$(word 2,$(subst ., ,$(SPK_VERS))) -SPK_REV = 1 +SPK_REV = 2 SPK_ICON = src/python3-pip.png # Platform "powerpc-none-linux-gnuspe" with compiler "gcc" is not supported by the diff --git a/spk/python312-wheels/Makefile b/spk/python312-wheels/Makefile index 79b5840e848..db7c047715d 100644 --- a/spk/python312-wheels/Makefile +++ b/spk/python312-wheels/Makefile @@ -1,7 +1,7 @@ SPK_NAME = python312-wheels SPK_VERS = 1.0 SPK_VERS_MAJOR_MINOR = $(word 1,$(subst ., ,$(SPK_VERS))).$(word 2,$(subst ., ,$(SPK_VERS))) -SPK_REV = 1 +SPK_REV = 2 SPK_ICON = src/python3-pip.png # Compiler must support std=c++11 From f2c6a26bebebe3e312bf5375cbaf26a7f8ea934e Mon Sep 17 00:00:00 2001 From: Vincent Fortier Date: Wed, 15 Jan 2025 12:14:37 +0000 Subject: [PATCH 13/24] python-wheel.mk: Ensure wheelhouse directory is set --- mk/spksrc.python-wheel.mk | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mk/spksrc.python-wheel.mk b/mk/spksrc.python-wheel.mk index 27d29315a70..255e4870e35 100644 --- a/mk/spksrc.python-wheel.mk +++ b/mk/spksrc.python-wheel.mk @@ -39,6 +39,10 @@ build_crossenv_module: ### Python wheel rules build_python_wheel_target: build_crossenv_module +ifeq ($(wildcard $(WHEELHOUSE)),) + @$(MSG) Creating wheelhouse directory: $(WHEELHOUSE) + @mkdir -p $(WHEELHOUSE) +endif $(foreach e,$(shell cat $(CROSSENV_MODULE_PATH)/build/python-cc.mk),$(eval $(e))) @. $(CROSSENV) ; \ $(MSG) _PYTHON_HOST_PLATFORM=$(TC_TARGET) cross-python3 -m build $(BUILD_ARGS) --wheel $(WHEELS_BUILD_ARGS) --outdir $(WHEELHOUSE) ; \ From c1f2a54459b7fc02e8a98df566363f7a5029cb78 Mon Sep 17 00:00:00 2001 From: Vincent Fortier Date: Wed, 15 Jan 2025 12:14:59 +0000 Subject: [PATCH 14/24] wheel-compile.mk: Fix call to crossenv - --- mk/spksrc.wheel-compile.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mk/spksrc.wheel-compile.mk b/mk/spksrc.wheel-compile.mk index b5a8ed7df2d..58f81eae352 100644 --- a/mk/spksrc.wheel-compile.mk +++ b/mk/spksrc.wheel-compile.mk @@ -49,8 +49,8 @@ ifeq ($(wildcard $(WHEELHOUSE)),) endif @$(MSG) Compiling wheel [$(WHEEL_NAME)], version [$(WHEEL_VERSION)], type [$(WHEEL_TYPE)] ifneq ($(WHEEL_TYPE),pure) - @$(MSG) $(MAKE) WHEEL=$(WHEEL_NAME)==$(WHEEL_VERSION) crossenv-$(ARCH)-$(TCVERSION) ; \ - MAKEFLAGS= $(MAKE) WHEEL=$(WHEEL_NAME)==$(WHEEL_VERSION) crossenv-$(ARCH)-$(TCVERSION) --no-print-directory ; \ + @$(MSG) $(MAKE) WHEEL=\"$(WHEEL_NAME)-$(WHEEL_VERSION)\" crossenv-$(ARCH)-$(TCVERSION) ; \ + MAKEFLAGS= $(MAKE) WHEEL="$(WHEEL_NAME)-$(WHEEL_VERSION)" crossenv-$(ARCH)-$(TCVERSION) --no-print-directory ; \ [ "$(WHEEL_TYPE)" = "$(WHEELS_LIMITED_API)" ] && abi3="--build-option=--py-limited-api=$(PYTHON_LIMITED_API)" || abi3="" ; \ global_options=$$(echo $(WHEELS_BUILD_ARGS) | sed -e 's/ \[/\n\[/g' | grep -i $(WHEEL_NAME) | cut -f2 -d] | xargs) ; \ localCFLAGS=($$(echo $(WHEELS_CFLAGS) | sed -e 's/ \[/\n\[/g' | grep -i $(WHEEL_NAME) | cut -f2 -d] | xargs)) ; \ From 6efb2c8a28a237bed3ade615c510ebdc7545b1b1 Mon Sep 17 00:00:00 2001 From: Vincent Fortier Date: Wed, 15 Jan 2025 23:13:15 +0000 Subject: [PATCH 15/24] python-wheel.mk: Fix post compile issue related to install --- mk/spksrc.python-wheel.mk | 16 ++++++++-------- mk/spksrc.wheel-env.mk | 3 +++ mk/spksrc.wheel-install.mk | 7 +++---- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/mk/spksrc.python-wheel.mk b/mk/spksrc.python-wheel.mk index 255e4870e35..4bcd98e14d8 100644 --- a/mk/spksrc.python-wheel.mk +++ b/mk/spksrc.python-wheel.mk @@ -29,9 +29,6 @@ ifneq ($(wildcard $(PYTHON_STAGING_PREFIX)),) STAGING_INSTALL_PREFIX := $(PYTHON_STAGING_PREFIX) endif -## python wheel specific configurations -include ../../mk/spksrc.wheel-env.mk - ### Prepare crossenv build_crossenv_module: @$(MSG) WHEEL="$(PKG_NAME)-$(PKG_VERS)" $(MAKE) crossenv-$(ARCH)-$(TCVERSION) @@ -39,15 +36,12 @@ build_crossenv_module: ### Python wheel rules build_python_wheel_target: build_crossenv_module -ifeq ($(wildcard $(WHEELHOUSE)),) - @$(MSG) Creating wheelhouse directory: $(WHEELHOUSE) - @mkdir -p $(WHEELHOUSE) -endif $(foreach e,$(shell cat $(CROSSENV_MODULE_PATH)/build/python-cc.mk),$(eval $(e))) @. $(CROSSENV) ; \ $(MSG) _PYTHON_HOST_PLATFORM=$(TC_TARGET) cross-python3 -m build $(BUILD_ARGS) --wheel $(WHEELS_BUILD_ARGS) --outdir $(WHEELHOUSE) ; \ $(RUN) _PYTHON_HOST_PLATFORM=$(TC_TARGET) cross-python3 -m build $(BUILD_ARGS) --wheel $(WHEELS_BUILD_ARGS) --outdir $(WHEELHOUSE) - @$(RUN) echo "$(PKG_NAME)==$(PKG_VERS)" >> $(WHEELHOUSE)/$(WHEELS_CROSS_COMPILE) + @$(MSG) $(MAKE) REQUIREMENT=\"$(PKG_NAME)==$(PKG_VERS)\" WHEEL_NAME=\"$(PKG_NAME)\" WHEEL_VERSION=\"$(PKG_VERS)\" WHEEL_TYPE=\"cross\" wheel_install + -@MAKEFLAGS= $(MAKE) REQUIREMENT="$(PKG_NAME)==$(PKG_VERS)" WHEEL_NAME="$(PKG_NAME)" WHEEL_VERSION="$(PKG_VERS)" WHEEL_TYPE="cross" --no-print-directory wheel_install post_install_python_wheel_target: $(WHEEL_TARGET) install_python_wheel @@ -57,3 +51,9 @@ all: install # Use crossenv include ../../mk/spksrc.crossenv.mk + +## python wheel specific configurations +include ../../mk/spksrc.wheel-env.mk + +## install wheel specific routines +include ../../mk/spksrc.wheel-install.mk diff --git a/mk/spksrc.wheel-env.mk b/mk/spksrc.wheel-env.mk index 15c889c499c..9eb51db3038 100644 --- a/mk/spksrc.wheel-env.mk +++ b/mk/spksrc.wheel-env.mk @@ -34,6 +34,9 @@ endif ifeq ($(strip $(WHEELS_PURE_PYTHON)),) WHEELS_PURE_PYTHON = requirements-pure.txt endif +ifeq ($(strip $(WHEELS_CROSS_COMPILE)),) +WHEELS_CROSS_COMPILE = requirements-cross.txt +endif ifeq ($(strip $(WHEELS_CROSSENV_COMPILE)),) WHEELS_CROSSENV_COMPILE = requirements-crossenv.txt endif diff --git a/mk/spksrc.wheel-install.mk b/mk/spksrc.wheel-install.mk index 19d0bb16133..04420fd8070 100644 --- a/mk/spksrc.wheel-install.mk +++ b/mk/spksrc.wheel-install.mk @@ -43,15 +43,14 @@ pre_wheel_install_target: wheel_install_msg_target wheel_install_target: SHELL:=/bin/bash wheel_install_target: -ifeq ($(wildcard $(WHEELHOUSE)),) - @$(MSG) Creating wheelhouse directory: $(WHEELHOUSE) - @mkdir -p $(WHEELHOUSE) -endif @$(MSG) Installing wheel [$(WHEEL_NAME)], version [$(WHEEL_VERSION)], type [$(WHEEL_TYPE)] ; \ case $(WHEEL_TYPE) in \ abi3) $(MSG) Adding $(WHEEL_NAME)==$$(WHEEL_VERSION) to wheelhouse/$(WHEELS_LIMITED_API) ; \ echo $(WHEEL_NAME)==$(WHEEL_VERSION) | sed -e '/^[[:blank:]]*$$\|^#/d' >> $(WHEELHOUSE)/$(WHEELS_LIMITED_API) ; \ ;; \ + cross) $(MSG) Adding $(WHEEL_NAME)==$(WHEEL_VERSION) to wheelhouse/$(WHEELS_CROSS_COMPILE) ; \ + echo $(WHEEL_NAME)==$(WHEEL_VERSION) | sed -e '/^[[:blank:]]*$$\|^#/d' >> $(WHEELHOUSE)/$(WHEELS_CROSS_COMPILE) ; \ + ;; \ crossenv) $(MSG) Adding $(WHEEL_NAME)==$(WHEEL_VERSION) to wheelhouse/$(WHEELS_CROSSENV_COMPILE) ; \ echo $(WHEEL_NAME)==$(WHEEL_VERSION) | sed -e '/^[[:blank:]]*$$\|^#/d' >> $(WHEELHOUSE)/$(WHEELS_CROSSENV_COMPILE) ; \ ;; \ From 7c89c52bae6cfde1661353adb8b14ddc54ad8253 Mon Sep 17 00:00:00 2001 From: Vincent Fortier Date: Fri, 17 Jan 2025 13:31:56 +0000 Subject: [PATCH 16/24] crossenv.mk: Migrate to using WHEEL_NAME and WHEEL_VERSION This allows using a unique method for both wheel and crossenv calls --- mk/spksrc.crossenv.mk | 34 +++++++++++++++------------------- mk/spksrc.python-module.mk | 4 ++-- mk/spksrc.wheel-compile.mk | 6 +++--- 3 files changed, 20 insertions(+), 24 deletions(-) diff --git a/mk/spksrc.crossenv.mk b/mk/spksrc.crossenv.mk index 5285b7810d7..fb146a32c69 100644 --- a/mk/spksrc.crossenv.mk +++ b/mk/spksrc.crossenv.mk @@ -13,7 +13,8 @@ # build_crossenv_target (override with CROSSENV_TARGET) # post_crossenv_target (override with POST_CROSSENV_TARGET) # Variables: -# WHEELS List of wheels to go through +# WHEEL_NAME Name of wheel to process +# WHEEL_VERSION Version of wheel to process (can be empty) # Defined using PYTHON_PACKAGE_WORK_DIR from spksrc.python.mk or use local work directory PYTHON_WORK_DIR = $(or $(wildcard $(PYTHON_PACKAGE_WORK_DIR)),$(wildcard $(WORK_DIR))) @@ -55,13 +56,13 @@ endif ### -# Check for wheel==x.y, then fallback to wheel, then default -ifneq ($(wildcard $(CROSSENV_CONFIG_PATH)/requirements-$(WHEEL).txt),) -CROSSENV_BUILD_WHEEL = $(WHEEL) -CROSSENV_BUILD_REQUIREMENTS = $(CROSSENV_CONFIG_PATH)/requirements-$(CROSSENV_BUILD_WHEEL).txt -else ifneq ($(wildcard $(CROSSENV_CONFIG_PATH)/requirements-$(firstword $(subst -, ,$(WHEEL))).txt),) -CROSSENV_BUILD_WHEEL = $(firstword $(subst -, ,$(WHEEL))) +# Check for -, then fallback to , then default +ifneq ($(wildcard $(CROSSENV_CONFIG_PATH)/requirements-$(WHEEL_NAME)-$(WHEEL_VERSION).txt),) +CROSSENV_BUILD_WHEEL = $(WHEEL_NAME)-$(WHEEL_VERSION) CROSSENV_BUILD_REQUIREMENTS = $(CROSSENV_CONFIG_PATH)/requirements-$(CROSSENV_BUILD_WHEEL).txt +else ifneq ($(wildcard $(CROSSENV_CONFIG_PATH)/requirements-$(WHEEL_NAME).txt),) +CROSSENV_BUILD_WHEEL = $(WHEEL_NAME) +CROSSENV_BUILD_REQUIREMENTS = $(CROSSENV_CONFIG_PATH)/requirements-$(WHEEL_NAME).txt else CROSSENV_BUILD_WHEEL = default CROSSENV_BUILD_REQUIREMENTS = $(CROSSENV_CONFIG_DEFAULT) @@ -73,10 +74,10 @@ CROSSENV_COOKIE = $(WORK_DIR)/.crossenv-$(CROSSENV_BUILD_WHEEL)_done ### # default wheel packages to install in crossenv -ifneq ($(wildcard $(CROSSENV_CONFIG_PATH)/requirements-$(WHEEL).txt $(CROSSENV_CONFIG_DEFAULT)),) -CROSSENV_DEFAULT_PIP_VERSION = $(shell grep -h -E "^pip[<>=]=" $(wildcard $(CROSSENV_CONFIG_PATH)/requirements-$(WHEEL).txt $(CROSSENV_CONFIG_DEFAULT)) | head -1 | sed -E 's/.*[<>=]=//') -CROSSENV_DEFAULT_SETUPTOOLS_VERSION = $(shell grep -h -E "^setuptools[<>=]=" $(wildcard $(CROSSENV_CONFIG_PATH)/requirements-$(WHEEL).txt $(CROSSENV_CONFIG_DEFAULT)) | head -1 | sed -E 's/.*[<>=]=//') -CROSSENV_DEFAULT_WHEEL_VERSION = $(shell grep -h -E "^wheel[<>=]=" $(wildcard $(CROSSENV_CONFIG_PATH)/requirements-$(WHEEL).txt $(CROSSENV_CONFIG_DEFAULT)) | head -1 | sed -E 's/.*[<>=]=//') +ifneq ($(wildcard $(CROSSENV_BUILD_REQUIREMENTS) $(CROSSENV_CONFIG_DEFAULT)),) +CROSSENV_DEFAULT_PIP_VERSION = $(shell grep -h -E "^pip[<>=]=" $(wildcard $(CROSSENV_BUILD_REQUIREMENTS) $(CROSSENV_CONFIG_DEFAULT)) | head -1 | sed -E 's/.*[<>=]=//') +CROSSENV_DEFAULT_SETUPTOOLS_VERSION = $(shell grep -h -E "^setuptools[<>=]=" $(wildcard $(CROSSENV_BUILD_REQUIREMENTS) $(CROSSENV_CONFIG_DEFAULT)) | head -1 | sed -E 's/.*[<>=]=//') +CROSSENV_DEFAULT_WHEEL_VERSION = $(shell grep -h -E "^wheel[<>=]=" $(wildcard $(CROSSENV_BUILD_REQUIREMENTS) $(CROSSENV_CONFIG_DEFAULT)) | head -1 | sed -E 's/.*[<>=]=//') endif ifneq ($(CROSSENV_DEFAULT_PIP_VERSION),) @@ -114,13 +115,8 @@ post_crossenv_target: $(CROSSENV_TARGET) ### crossenv-%: -ifneq ($(filter error-%, $(CROSSENV_BUILD_WHEEL)),) - @$(MSG) $(MAKE) $(CROSSENV_BUILD_WHEEL) - @$(MAKE) $(CROSSENV_BUILD_WHEEL) --no-print-directory -else - @$(MSG) $(MAKE) ARCH=$(firstword $(subst -, ,$*)) TCVERSION=$(lastword $(subst -, ,$*)) WHEEL=$(CROSSENV_BUILD_WHEEL) crossenv - -@MAKEFLAGS= $(MAKE) ARCH=$(firstword $(subst -, ,$*)) TCVERSION=$(lastword $(subst -, ,$*)) WHEEL=$(CROSSENV_BUILD_WHEEL) crossenv --no-print-directory -endif + @$(MSG) $(MAKE) ARCH=$(firstword $(subst -, ,$*)) TCVERSION=$(lastword $(subst -, ,$*)) WHEEL_NAME=$(WHEEL_NAME) WHEEL_VERSION=$(WHEEL_VERSION) crossenv + -@MAKEFLAGS= $(MAKE) ARCH=$(firstword $(subst -, ,$*)) TCVERSION=$(lastword $(subst -, ,$*)) WHEEL_NAME=$(WHEEL_NAME) WHEEL_VERSION=$(WHEEL_VERSION) crossenv --no-print-directory #### @@ -167,7 +163,7 @@ export PYTHONPATH = $(PYTHON_LIB_NATIVE):$(PYTHON_STAGING_INSTALL_PREFIX)/lib/py # build_crossenv_target: SHELL:=/bin/bash build_crossenv_target: pre_crossenv_target $(CROSSENV_PATH)/build/python-cc.mk - @$(MSG) $$(date +%Y%m%d-%H%M%S) MAKELEVEL: $(MAKELEVEL), PARALLEL_MAKE: $(PARALLEL_MAKE), ARCH: $(ARCH)-$(TCVERSION), CROSSENV: $(WHEEL) >> $(PSTAT_LOG) + @$(MSG) $$(date +%Y%m%d-%H%M%S) MAKELEVEL: $(MAKELEVEL), PARALLEL_MAKE: $(PARALLEL_MAKE), ARCH: $(ARCH)-$(TCVERSION), CROSSENV: $(CROSSENV_BUILD_WHEEL) >> $(PSTAT_LOG) @$(MSG) Python sources: $(wildcard $(PYTHON_WORK_DIR)/Python-[0-9]*) @$(MSG) crossenv wheel packages: $(CROSSENV_DEFAULT_PIP), $(CROSSENV_DEFAULT_SETUPTOOLS), $(CROSSENV_DEFAULT_WHEEL) @$(MSG) crossenv requirement definition: $(CROSSENV_BUILD_REQUIREMENTS) diff --git a/mk/spksrc.python-module.mk b/mk/spksrc.python-module.mk index 3f1467f5a88..85cfcdc834f 100644 --- a/mk/spksrc.python-module.mk +++ b/mk/spksrc.python-module.mk @@ -21,8 +21,8 @@ CROSSENV_MODULE_PATH = $(firstword $(wildcard $(WORK_DIR)/crossenv-$(PKG_NAME)-$ ### Prepare crossenv build_crossenv_module: - @$(MSG) WHEEL="$(PKG_NAME)-$(PKG_VERS)" $(MAKE) crossenv-$(ARCH)-$(TCVERSION) - -@MAKEFLAGS= WHEEL="$(PKG_NAME)-$(PKG_VERS)" $(MAKE) crossenv-$(ARCH)-$(TCVERSION) + @$(MSG) $(MAKE) WHEEL_NAME=\"$(PKG_VERS)\" WHEEL_VERSION=\"$(PKG_VERS)\" crossenv-$(ARCH)-$(TCVERSION) + -@MAKEFLAGS= $(MAKE) WHEEL_NAME="$(PKG_VERS)" WHEEL_VERSION="$(PKG_VERS)" crossenv-$(ARCH)-$(TCVERSION) ### Python extension module rules compile_python_module: build_crossenv_module diff --git a/mk/spksrc.wheel-compile.mk b/mk/spksrc.wheel-compile.mk index 58f81eae352..7153a0b2b81 100644 --- a/mk/spksrc.wheel-compile.mk +++ b/mk/spksrc.wheel-compile.mk @@ -49,9 +49,9 @@ ifeq ($(wildcard $(WHEELHOUSE)),) endif @$(MSG) Compiling wheel [$(WHEEL_NAME)], version [$(WHEEL_VERSION)], type [$(WHEEL_TYPE)] ifneq ($(WHEEL_TYPE),pure) - @$(MSG) $(MAKE) WHEEL=\"$(WHEEL_NAME)-$(WHEEL_VERSION)\" crossenv-$(ARCH)-$(TCVERSION) ; \ - MAKEFLAGS= $(MAKE) WHEEL="$(WHEEL_NAME)-$(WHEEL_VERSION)" crossenv-$(ARCH)-$(TCVERSION) --no-print-directory ; \ - [ "$(WHEEL_TYPE)" = "$(WHEELS_LIMITED_API)" ] && abi3="--build-option=--py-limited-api=$(PYTHON_LIMITED_API)" || abi3="" ; \ + @$(MSG) $(MAKE) WHEEL_NAME=\"$(WHEEL_NAME)\" WHEEL_VERSION=\"$(WHEEL_VERSION)\" crossenv-$(ARCH)-$(TCVERSION) ; \ + MAKEFLAGS= $(MAKE) WHEEL_NAME="$(WHEEL_NAME)" WHEEL_VERSION="$(WHEEL_VERSION)" crossenv-$(ARCH)-$(TCVERSION) --no-print-directory + @[ "$(WHEEL_TYPE)" = "$(WHEELS_LIMITED_API)" ] && abi3="--build-option=--py-limited-api=$(PYTHON_LIMITED_API)" || abi3="" ; \ global_options=$$(echo $(WHEELS_BUILD_ARGS) | sed -e 's/ \[/\n\[/g' | grep -i $(WHEEL_NAME) | cut -f2 -d] | xargs) ; \ localCFLAGS=($$(echo $(WHEELS_CFLAGS) | sed -e 's/ \[/\n\[/g' | grep -i $(WHEEL_NAME) | cut -f2 -d] | xargs)) ; \ localLDFLAGS=($$(echo $(WHEELS_LDFLAGS) | sed -e 's/ \[/\n\[/g' | grep -i $(WHEEL_NAME) | cut -f2 -d] | xargs)) ; \ From 6243fd98c201763e24e14c10f168d93eac8b8895 Mon Sep 17 00:00:00 2001 From: Vincent Fortier Date: Fri, 17 Jan 2025 16:27:00 +0000 Subject: [PATCH 17/24] wheelclean: Also clean-up work/.wheel* due to download-wheels --- mk/spksrc.spk.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mk/spksrc.spk.mk b/mk/spksrc.spk.mk index 2f69c4430fc..661589ab787 100644 --- a/mk/spksrc.spk.mk +++ b/mk/spksrc.spk.mk @@ -538,8 +538,8 @@ spkclean: work-*/WIZARD_UIFILES wheelclean: spkclean - rm -fr work-*/.wheel_done \ - work-*/.wheel_*_done \ + rm -fr work*/.wheel_done \ + work*/.wheel_*_done \ work-*/wheelhouse \ work-*/install/var/packages/**/target/share/wheelhouse @make --no-print-directory dependency-flat | sort -u | grep cross/ | while read depend ; do \ From 54d8d69aff5d75008fab00cd4efe10b433035661 Mon Sep 17 00:00:00 2001 From: Vincent Fortier Date: Fri, 17 Jan 2025 16:28:26 +0000 Subject: [PATCH 18/24] wheel.mk: Add new download-wheels sub-routine --- mk/spksrc.wheel-env.mk | 5 ++++- mk/spksrc.wheel.mk | 13 ++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/mk/spksrc.wheel-env.mk b/mk/spksrc.wheel-env.mk index 9eb51db3038..addf5a2dc41 100644 --- a/mk/spksrc.wheel-env.mk +++ b/mk/spksrc.wheel-env.mk @@ -43,7 +43,10 @@ endif ifeq ($(strip $(WHEEL_DEFAULT_PREFIX)),) # If no ARCH then pure by default -ifeq ($(strip $(ARCH)),) +# unless called using download-wheels +ifeq ($(MAKECMDGOALS),download-wheels) +WHEEL_DEFAULT_PREFIX = crossenv +else ifeq ($(strip $(ARCH)),) WHEEL_DEFAULT_PREFIX = pure else WHEEL_DEFAULT_PREFIX = crossenv diff --git a/mk/spksrc.wheel.mk b/mk/spksrc.wheel.mk index 660b52f3cde..eab52c4c6c1 100644 --- a/mk/spksrc.wheel.mk +++ b/mk/spksrc.wheel.mk @@ -38,6 +38,7 @@ include ../../mk/spksrc.wheel-install.mk ## ifneq ($(strip $(REQUIREMENT)),) +download-wheels: wheel_download wheel: wheel_install else @@ -107,8 +108,8 @@ ifneq ($(wildcard $(abspath $(addprefix $(WORK_DIR)/../,$(WHEELS)))),) # $(MSG) name: [$${name}] ; \ # $(MSG) version: [$${version}] ; \ # $(MSG) type: [$${type}] ; \ - $(MSG) $(MAKE) ARCH=$(ARCH) TCVERSION=$(TCVERSION) REQUIREMENT=\"$${wheel}\" WHEEL_NAME=\"$${name}\" WHEEL_VERSION=\"$${version}\" WHEEL_TYPE=\"$${type}\" wheel ; \ - $(MAKE) ARCH="$(ARCH)" TCVERSION="$(TCVERSION)" REQUIREMENT="$${wheel}" WHEEL_NAME="$${name}" WHEEL_VERSION="$${version}" WHEEL_TYPE="$${type}" wheel --no-print-directory ; \ + $(MSG) $(MAKE) ARCH=$(ARCH) TCVERSION=$(TCVERSION) REQUIREMENT=\"$${wheel}\" WHEEL_NAME=\"$${name}\" WHEEL_VERSION=\"$${version}\" WHEEL_TYPE=\"$${type}\" $(MAKECMDGOALS) ; \ + $(MAKE) ARCH="$(ARCH)" TCVERSION="$(TCVERSION)" REQUIREMENT="$${wheel}" WHEEL_NAME="$${name}" WHEEL_VERSION="$${version}" WHEEL_TYPE="$${type}" $(MAKECMDGOALS) --no-print-directory ; \ done < <(grep -svH -e "^\#" -e "^\$$" $(wildcard $(abspath $(addprefix $(WORK_DIR)/../,$(WHEELS)))) | sed 's/\s* #.*//') endif ifneq ($(filter-out $(addprefix src/,$(notdir $(wildcard $(abspath $(addprefix $(WORK_DIR)/../,$(WHEELS)))))),$(WHEELS)),) @@ -143,12 +144,14 @@ ifneq ($(filter-out $(addprefix src/,$(notdir $(wildcard $(abspath $(addprefix $ # $(MSG) wheel: [$${wheel}] ; \ # $(MSG) name: [$${name}] ; \ # $(MSG) version: [$${version}] ; \ -# $(MSG) type: [$${type}] ; \ - $(MSG) $(MAKE) ARCH=$(ARCH) TCVERSION=$(TCVERSION) REQUIREMENT=\"$${wheel}\" WHEEL_NAME=\"$${name}\" WHEEL_VERSION=\"$${version}\" WHEEL_TYPE=\"$${type}\" wheel ; \ - $(MAKE) ARCH="$(ARCH)" TCVERSION="$(TCVERSION)" REQUIREMENT="$${wheel}" WHEEL_NAME="$${name}" WHEEL_VERSION="$${version}" WHEEL_TYPE="$${type}" wheel --no-print-directory ; \ +# $(MSG) type: [$${type}]; \ + $(MSG) $(MAKE) ARCH=$(ARCH) TCVERSION=$(TCVERSION) REQUIREMENT=\"$${wheel}\" WHEEL_NAME=\"$${name}\" WHEEL_VERSION=\"$${version}\" WHEEL_TYPE=\"$${type}\" $(MAKECMDGOALS) ; \ + $(MAKE) ARCH="$(ARCH)" TCVERSION="$(TCVERSION)" REQUIREMENT="$${wheel}" WHEEL_NAME="$${name}" WHEEL_VERSION="$${version}" WHEEL_TYPE="$${type}" $(MAKECMDGOALS) --no-print-directory ; \ done endif +download-wheels: $(WHEEL_TARGET) + post_wheel_target: $(WHEEL_TARGET) install_python_wheel ifeq ($(wildcard $(WHEEL_COOKIE)),) From 8d52a823141f12844ba673b692d379dd4cede437 Mon Sep 17 00:00:00 2001 From: Vincent Fortier Date: Fri, 17 Jan 2025 16:30:18 +0000 Subject: [PATCH 19/24] wheel.mk: Remove commented debug traces --- mk/spksrc.wheel-download.mk | 6 ------ mk/spksrc.wheel.mk | 11 ----------- 2 files changed, 17 deletions(-) diff --git a/mk/spksrc.wheel-download.mk b/mk/spksrc.wheel-download.mk index 991ae3cfb63..34632833ab1 100644 --- a/mk/spksrc.wheel-download.mk +++ b/mk/spksrc.wheel-download.mk @@ -52,12 +52,6 @@ ifeq ($(wildcard $(PIP_CACHE_DIR)),) @mkdir -p $(PIP_CACHE_DIR) endif @$(MSG) Downloading wheel [$(WHEEL_NAME)], version [$(WHEEL_VERSION)] ; \ -# $(MSG) requirement: [$(REQUIREMENT)] ; \ -# $(MSG) requirement-grep-egg: [$$(grep -s egg <<< $(REQUIREMENT))] ; \ -# $(MSG) name: [$(WHEEL_NAME)] ; \ -# $(MSG) type: [$(WHEEL_TYPE)] ; \ -# $(MSG) version: [$(WHEEL_VERSION)] ; \ -# $(MSG) type: [$(WHEEL_TYPE)] ; \ if [ "$$(grep -s egg <<< $(REQUIREMENT))" ] ; then \ echo "WARNING: Skipping download URL - Downloaded at build time" ; \ elif [ "$(WHEEL_TYPE)" = "pure" ] && [ ! "$(WHEELS_PURE_PYTHON_PACKAGING_ENABLE)" = "1" ]; then \ diff --git a/mk/spksrc.wheel.mk b/mk/spksrc.wheel.mk index eab52c4c6c1..8c4a03e287f 100644 --- a/mk/spksrc.wheel.mk +++ b/mk/spksrc.wheel.mk @@ -102,12 +102,6 @@ ifneq ($(wildcard $(abspath $(addprefix $(WORK_DIR)/../,$(WHEELS)))),) query+=" | awk -F'-' '{print \$$2}'" ; \ version=$$(eval $${query} 2>/dev/null) ; \ fi ; \ -# $(MSG) requirement: [$${requirement}] ; \ -# $(MSG) file: [$${file}] ; \ -# $(MSG) wheel: [$${wheel}] ; \ -# $(MSG) name: [$${name}] ; \ -# $(MSG) version: [$${version}] ; \ -# $(MSG) type: [$${type}] ; \ $(MSG) $(MAKE) ARCH=$(ARCH) TCVERSION=$(TCVERSION) REQUIREMENT=\"$${wheel}\" WHEEL_NAME=\"$${name}\" WHEEL_VERSION=\"$${version}\" WHEEL_TYPE=\"$${type}\" $(MAKECMDGOALS) ; \ $(MAKE) ARCH="$(ARCH)" TCVERSION="$(TCVERSION)" REQUIREMENT="$${wheel}" WHEEL_NAME="$${name}" WHEEL_VERSION="$${version}" WHEEL_TYPE="$${type}" $(MAKECMDGOALS) --no-print-directory ; \ done < <(grep -svH -e "^\#" -e "^\$$" $(wildcard $(abspath $(addprefix $(WORK_DIR)/../,$(WHEELS)))) | sed 's/\s* #.*//') @@ -140,11 +134,6 @@ ifneq ($(filter-out $(addprefix src/,$(notdir $(wildcard $(abspath $(addprefix $ query+=" | awk -F'-' '{print \$$2}'" ; \ version=$$(eval $${query} 2>/dev/null) ; \ fi ; \ -# $(MSG) requirement: [$${requirement}] ; \ -# $(MSG) wheel: [$${wheel}] ; \ -# $(MSG) name: [$${name}] ; \ -# $(MSG) version: [$${version}] ; \ -# $(MSG) type: [$${type}]; \ $(MSG) $(MAKE) ARCH=$(ARCH) TCVERSION=$(TCVERSION) REQUIREMENT=\"$${wheel}\" WHEEL_NAME=\"$${name}\" WHEEL_VERSION=\"$${version}\" WHEEL_TYPE=\"$${type}\" $(MAKECMDGOALS) ; \ $(MAKE) ARCH="$(ARCH)" TCVERSION="$(TCVERSION)" REQUIREMENT="$${wheel}" WHEEL_NAME="$${name}" WHEEL_VERSION="$${version}" WHEEL_TYPE="$${type}" $(MAKECMDGOALS) --no-print-directory ; \ done From ce0afe894ca323fbe69883832833f3ee337ce21b Mon Sep 17 00:00:00 2001 From: Vincent Fortier Date: Fri, 17 Jan 2025 18:19:04 +0000 Subject: [PATCH 20/24] wheel.mk: Pre-process #egg= tags for compatibility with newer pip --- mk/spksrc.wheel-download.mk | 4 ++-- mk/spksrc.wheel-install.mk | 3 +-- mk/spksrc.wheel.mk | 6 ++++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/mk/spksrc.wheel-download.mk b/mk/spksrc.wheel-download.mk index 34632833ab1..479fa24a024 100644 --- a/mk/spksrc.wheel-download.mk +++ b/mk/spksrc.wheel-download.mk @@ -52,7 +52,7 @@ ifeq ($(wildcard $(PIP_CACHE_DIR)),) @mkdir -p $(PIP_CACHE_DIR) endif @$(MSG) Downloading wheel [$(WHEEL_NAME)], version [$(WHEEL_VERSION)] ; \ - if [ "$$(grep -s egg <<< $(REQUIREMENT))" ] ; then \ + if [ "$$(grep -Eo 'http://|https://' <<< $(REQUIREMENT))" ] ; then \ echo "WARNING: Skipping download URL - Downloaded at build time" ; \ elif [ "$(WHEEL_TYPE)" = "pure" ] && [ ! "$(WHEELS_PURE_PYTHON_PACKAGING_ENABLE)" = "1" ]; then \ echo "WARNING: Skipping download - pure python packaging disabled" ; \ @@ -63,7 +63,7 @@ endif query+=" | select((.filename|test(\"-$(WHEEL_VERSION).tar.gz\")) or (.filename|test(\"-$(WHEEL_VERSION).zip\"))) | .url'" ; \ outFile=$$(basename $$(eval $${query} 2>/dev/null) 2> $(WHEELHOUSE)/$(WHEELS_LIMITED_API) ; \ ;; \ cross) $(MSG) Adding $(WHEEL_NAME)==$(WHEEL_VERSION) to wheelhouse/$(WHEELS_CROSS_COMPILE) ; \ @@ -72,7 +72,6 @@ install_python_wheel: $(MSG) Copying $(WHEELS_DEFAULT) to wheelhouse ; \ cp requirements*.txt $(STAGING_INSTALL_WHEELHOUSE) ; \ cat requirements*.txt >> $(STAGING_INSTALL_WHEELHOUSE)/$(WHEELS_DEFAULT) ; \ - sed -i -e '/^#/! s/^.*egg=//g' $(STAGING_INSTALL_WHEELHOUSE)/requirements*.txt ; \ sort -u -o $(STAGING_INSTALL_WHEELHOUSE)/$(WHEELS_DEFAULT) $(STAGING_INSTALL_WHEELHOUSE)/$(WHEELS_DEFAULT) ; \ else \ $(MSG) [SKIP] Copying $(WHEELS_DEFAULT) to wheelhouse ; \ diff --git a/mk/spksrc.wheel.mk b/mk/spksrc.wheel.mk index 8c4a03e287f..71c60131827 100644 --- a/mk/spksrc.wheel.mk +++ b/mk/spksrc.wheel.mk @@ -84,12 +84,13 @@ ifneq ($(wildcard $(abspath $(addprefix $(WORK_DIR)/../,$(WHEELS)))),) requirements-abi3.txt) type=abi3 ;; \ *) type=$(WHEEL_DEFAULT_PREFIX) ;; \ esac ; \ + version=$$(echo $${wheel} | grep -oP '(?<=([<>=]=))[^ ]*' || echo "") ; \ if [ "$$(grep -s egg <<< $${wheel})" ]; then \ name=$$(echo $${wheel#*egg=} | cut -f1 -d=) ; \ + wheel=$$(echo $${wheel%%#egg=*}) ; \ else \ name=$$(echo $${wheel%%[<>=]=*} | sed -E "s/^(abi3|crossenv|pure)://") ; \ fi ; \ - version=$$(echo $${wheel} | grep -oP '(?<=([<>=]=))[^ ]*' || echo "") ; \ if [ ! "$${version}" ]; then \ $(MSG) Fetching latest version available ; \ query="curl -s https://pypi.org/pypi/$${name}/json" ; \ @@ -116,12 +117,13 @@ ifneq ($(filter-out $(addprefix src/,$(notdir $(wildcard $(abspath $(addprefix $ pure:*) type=pure ;; \ *) type=$(WHEEL_DEFAULT_PREFIX) ;; \ esac ; \ + version=$$(echo $${wheel} | grep -oP '(?<=([<>=]=))[^ ]*' || echo "") ; \ if [ "$$(grep -s egg <<< $${requirement})" ]; then \ name=$$(echo $${wheel#*egg=} | cut -f1 -d=) ; \ + wheel=$$(echo $${wheel%%#egg=*}) ; \ else \ name=$$(echo $${wheel%%[<>=]=*} | sed -E "s/^(abi3|crossenv|pure)://") ; \ fi ; \ - version=$$(echo $${wheel} | grep -oP '(?<=([<>=]=))[^ ]*' || echo "") ; \ if [ ! "$${version}" ]; then \ $(MSG) Fetching latest version available ; \ query="curl -s https://pypi.org/pypi/$${name}/json" ; \ From c5f16e78c57863dbfcab1b53feb6329b0ed09755 Mon Sep 17 00:00:00 2001 From: Vincent Fortier Date: Sat, 18 Jan 2025 13:31:39 +0000 Subject: [PATCH 21/24] wheel.mk: Fix MAKECMDGOALS when called from spk.mk --- mk/spksrc.wheel.mk | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/mk/spksrc.wheel.mk b/mk/spksrc.wheel.mk index 71c60131827..3090d260146 100644 --- a/mk/spksrc.wheel.mk +++ b/mk/spksrc.wheel.mk @@ -12,6 +12,12 @@ # Variables: # WHEELS List of wheels to go through +# When wheel is called from: +# spksrc.spk.mk: MAKECMDGOALS is empty (needs to be set to wheel) +# make wheel-- : MAKECMDGOALS is wheel +# make download-wheels : MAKECMDGOALS is download-wheels +WHEEL_GOAL := $(if $(MAKECMDGOALS),$(MAKECMDGOALS),wheel) + # Completion status file WHEEL_COOKIE = $(WORK_DIR)/.$(COOKIE_PREFIX)wheel_done @@ -103,8 +109,8 @@ ifneq ($(wildcard $(abspath $(addprefix $(WORK_DIR)/../,$(WHEELS)))),) query+=" | awk -F'-' '{print \$$2}'" ; \ version=$$(eval $${query} 2>/dev/null) ; \ fi ; \ - $(MSG) $(MAKE) ARCH=$(ARCH) TCVERSION=$(TCVERSION) REQUIREMENT=\"$${wheel}\" WHEEL_NAME=\"$${name}\" WHEEL_VERSION=\"$${version}\" WHEEL_TYPE=\"$${type}\" $(MAKECMDGOALS) ; \ - $(MAKE) ARCH="$(ARCH)" TCVERSION="$(TCVERSION)" REQUIREMENT="$${wheel}" WHEEL_NAME="$${name}" WHEEL_VERSION="$${version}" WHEEL_TYPE="$${type}" $(MAKECMDGOALS) --no-print-directory ; \ + $(MSG) $(MAKE) ARCH=$(ARCH) TCVERSION=$(TCVERSION) REQUIREMENT=\"$${wheel}\" WHEEL_NAME=\"$${name}\" WHEEL_VERSION=\"$${version}\" WHEEL_TYPE=\"$${type}\" $(WHEEL_GOAL) ; \ + MAKEFLAGS= $(MAKE) ARCH="$(ARCH)" TCVERSION="$(TCVERSION)" REQUIREMENT="$${wheel}" WHEEL_NAME="$${name}" WHEEL_VERSION="$${version}" WHEEL_TYPE="$${type}" $(WHEEL_GOAL) --no-print-directory ; \ done < <(grep -svH -e "^\#" -e "^\$$" $(wildcard $(abspath $(addprefix $(WORK_DIR)/../,$(WHEELS)))) | sed 's/\s* #.*//') endif ifneq ($(filter-out $(addprefix src/,$(notdir $(wildcard $(abspath $(addprefix $(WORK_DIR)/../,$(WHEELS)))))),$(WHEELS)),) @@ -136,8 +142,8 @@ ifneq ($(filter-out $(addprefix src/,$(notdir $(wildcard $(abspath $(addprefix $ query+=" | awk -F'-' '{print \$$2}'" ; \ version=$$(eval $${query} 2>/dev/null) ; \ fi ; \ - $(MSG) $(MAKE) ARCH=$(ARCH) TCVERSION=$(TCVERSION) REQUIREMENT=\"$${wheel}\" WHEEL_NAME=\"$${name}\" WHEEL_VERSION=\"$${version}\" WHEEL_TYPE=\"$${type}\" $(MAKECMDGOALS) ; \ - $(MAKE) ARCH="$(ARCH)" TCVERSION="$(TCVERSION)" REQUIREMENT="$${wheel}" WHEEL_NAME="$${name}" WHEEL_VERSION="$${version}" WHEEL_TYPE="$${type}" $(MAKECMDGOALS) --no-print-directory ; \ + $(MSG) $(MAKE) ARCH=$(ARCH) TCVERSION=$(TCVERSION) REQUIREMENT=\"$${wheel}\" WHEEL_NAME=\"$${name}\" WHEEL_VERSION=\"$${version}\" WHEEL_TYPE=\"$${type}\" $(WHEEL_GOAL) ; \ + MAKEFLAGS= $(MAKE) ARCH="$(ARCH)" TCVERSION="$(TCVERSION)" REQUIREMENT="$${wheel}" WHEEL_NAME="$${name}" WHEEL_VERSION="$${version}" WHEEL_TYPE="$${type}" $(WHEEL_GOAL) --no-print-directory ; \ done endif From 54150587583f8fc01435ae6c6dc0075ad475e6ca Mon Sep 17 00:00:00 2001 From: Vincent Fortier Date: Sat, 18 Jan 2025 13:33:33 +0000 Subject: [PATCH 22/24] github-action: Enable downloading wheels from new method --- .github/actions/download.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/download.sh b/.github/actions/download.sh index ea16c2784c9..b9eb706ff26 100755 --- a/.github/actions/download.sh +++ b/.github/actions/download.sh @@ -33,7 +33,7 @@ if [ -z "${build_packages}" ]; then else for package in ${build_packages}; do echo "===> Download wheels: ${package}" - make -C spk/${package} wheeldownload + make -C spk/${package} download-wheels done fi From 965d64f95e9018502103d453d78c768e4889135d Mon Sep 17 00:00:00 2001 From: Vincent Fortier Date: Sat, 18 Jan 2025 17:08:38 +0000 Subject: [PATCH 23/24] Revert package bump to test wheels --- spk/bazarr/Makefile | 2 +- spk/borgbackup/Makefile | 2 +- spk/deluge/Makefile | 2 +- spk/python311-wheels/Makefile | 2 +- spk/python312-wheels/Makefile | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/spk/bazarr/Makefile b/spk/bazarr/Makefile index be4ed083544..f948f40a856 100644 --- a/spk/bazarr/Makefile +++ b/spk/bazarr/Makefile @@ -1,6 +1,6 @@ SPK_NAME = bazarr SPK_VERS = 1.4.5 -SPK_REV = 16 +SPK_REV = 15 SPK_ICON = src/bazarr.png PYTHON_PACKAGE = python311 diff --git a/spk/borgbackup/Makefile b/spk/borgbackup/Makefile index 6650db42540..d32281a08f4 100644 --- a/spk/borgbackup/Makefile +++ b/spk/borgbackup/Makefile @@ -1,6 +1,6 @@ SPK_NAME = borgbackup SPK_VERS = 1.4.0 -SPK_REV = 16 +SPK_REV = 15 SPK_ICON = src/borgbackup.png PYTHON_PACKAGE = python311 diff --git a/spk/deluge/Makefile b/spk/deluge/Makefile index 6dcf5e29d74..692e0b0b322 100644 --- a/spk/deluge/Makefile +++ b/spk/deluge/Makefile @@ -1,6 +1,6 @@ SPK_NAME = deluge SPK_VERS = 2.1.1.127 -SPK_REV = 25 +SPK_REV = 26 SPK_ICON = src/deluge.png PYTHON_PACKAGE = python311 diff --git a/spk/python311-wheels/Makefile b/spk/python311-wheels/Makefile index 05564925860..f7ec63a6580 100644 --- a/spk/python311-wheels/Makefile +++ b/spk/python311-wheels/Makefile @@ -1,7 +1,7 @@ SPK_NAME = python311-wheels SPK_VERS = 1.0 SPK_VERS_MAJOR_MINOR = $(word 1,$(subst ., ,$(SPK_VERS))).$(word 2,$(subst ., ,$(SPK_VERS))) -SPK_REV = 2 +SPK_REV = 1 SPK_ICON = src/python3-pip.png # Platform "powerpc-none-linux-gnuspe" with compiler "gcc" is not supported by the diff --git a/spk/python312-wheels/Makefile b/spk/python312-wheels/Makefile index db7c047715d..79b5840e848 100644 --- a/spk/python312-wheels/Makefile +++ b/spk/python312-wheels/Makefile @@ -1,7 +1,7 @@ SPK_NAME = python312-wheels SPK_VERS = 1.0 SPK_VERS_MAJOR_MINOR = $(word 1,$(subst ., ,$(SPK_VERS))).$(word 2,$(subst ., ,$(SPK_VERS))) -SPK_REV = 2 +SPK_REV = 1 SPK_ICON = src/python3-pip.png # Compiler must support std=c++11 From c744059ac1a16571f5ef273bafa6184fdb1fde34 Mon Sep 17 00:00:00 2001 From: Vincent Fortier Date: Sat, 18 Jan 2025 17:09:40 +0000 Subject: [PATCH 24/24] deluge: Revert package bump --- spk/deluge/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spk/deluge/Makefile b/spk/deluge/Makefile index 692e0b0b322..70b5699fc52 100644 --- a/spk/deluge/Makefile +++ b/spk/deluge/Makefile @@ -1,6 +1,6 @@ SPK_NAME = deluge SPK_VERS = 2.1.1.127 -SPK_REV = 26 +SPK_REV = 24 SPK_ICON = src/deluge.png PYTHON_PACKAGE = python311