Merge pull request #9204 from ThomasWaldmann/rel143 #5300

Workflow file for this run

	# badge: https://github.com/borgbackup/borg/workflows/CI/badge.svg?branch=1.4-maint

	name: CI

	on:
	push:
	branches: [ 1.4-maint ]
	tags:
	- '1.*'
	paths:
	- '**.py'
	- '**.pyx'
	- '**.c'
	- '**.h'
	- '**.yml'
	- '**.toml'
	- '**.cfg'
	- '**.ini'
	- 'requirements.d/*'
	- '!docs/**'
	pull_request:
	branches: [ 1.4-maint ]
	paths:
	- '**.py'
	- '**.pyx'
	- '**.c'
	- '**.h'
	- '**.yml'
	- '**.toml'
	- '**.cfg'
	- '**.ini'
	- 'requirements.d/*'
	- '!docs/**'

	jobs:
	lint:

	runs-on: ubuntu-22.04
	timeout-minutes: 5

	steps:
	- uses: actions/checkout@v4
	- uses: chartboost/ruff-action@v1


	asan_ubsan:

	runs-on: ubuntu-24.04
	timeout-minutes: 25
	needs: [lint]

	steps:
	- uses: actions/checkout@v4
	with:
	# Just fetching one commit is not enough for setuptools-scm, so we fetch all.
	fetch-depth: 0
	fetch-tags: true

	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: '3.12'

	- name: Install system packages
	run: \|
	sudo apt-get update
	sudo apt-get install -y pkg-config build-essential
	sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev

	- name: Install Python dependencies
	run: \|
	python -m pip install --upgrade pip
	pip install -r requirements.d/development.txt

	- name: Build Borg with ASan/UBSan
	# Build the C/Cython extensions with AddressSanitizer and UndefinedBehaviorSanitizer enabled.
	# How this works:
	# - The -fsanitize=address,undefined flags inject runtime checks into our native code. If a bug is hit
	# (e.g., buffer overflow, use-after-free, out-of-bounds, or undefined behavior), the sanitizer prints
	# a detailed error report to stderr, including a stack trace, and forces the process to exit with
	# non-zero status. In CI, this will fail the step/job so you will notice.
	# - ASAN_OPTIONS/UBSAN_OPTIONS configure the sanitizers' runtime behavior (see below for meanings).
	env:
	CFLAGS: "-O1 -g -fno-omit-frame-pointer -fsanitize=address,undefined"
	CXXFLAGS: "-O1 -g -fno-omit-frame-pointer -fsanitize=address,undefined"
	LDFLAGS: "-fsanitize=address,undefined"
	# ASAN_OPTIONS controls AddressSanitizer runtime tweaks:
	# - detect_leaks=0: Disable LeakSanitizer to avoid false positives with CPython/pymalloc in short-lived tests.
	# - strict_string_checks=1: Make invalid string operations (e.g., over-reads) more likely to be detected.
	# - check_initialization_order=1: Catch uses that depend on static initialization order (C++).
	# - detect_stack_use_after_return=1: Detect stack-use-after-return via stack poisoning (may increase overhead).
	ASAN_OPTIONS: "detect_leaks=0:strict_string_checks=1:check_initialization_order=1:detect_stack_use_after_return=1"
	# UBSAN_OPTIONS controls UndefinedBehaviorSanitizer runtime:
	# - print_stacktrace=1: Include a stack trace for UB reports to ease debugging.
	# Note: UBSan is recoverable by default (process may continue after reporting). If you want CI to
	# abort immediately and fail on the first UB, add `halt_on_error=1` (e.g., UBSAN_OPTIONS="print_stacktrace=1:halt_on_error=1").
	UBSAN_OPTIONS: "print_stacktrace=1"
	# PYTHONDEVMODE enables additional Python runtime checks and warnings.
	PYTHONDEVMODE: "1"
	run: pip install -e .

	- name: Run tests under sanitizers
	env:
	ASAN_OPTIONS: "detect_leaks=0:strict_string_checks=1:check_initialization_order=1:detect_stack_use_after_return=1"
	UBSAN_OPTIONS: "print_stacktrace=1"
	PYTHONDEVMODE: "1"
	# Ensure the ASan runtime is loaded first to avoid "ASan runtime does not come first" warnings.
	# We discover libasan/libubsan paths via gcc and preload them for the Python test process.
	# the remote tests are slow and likely won't find anything useful
	run: \|
	set -euo pipefail
	export LD_PRELOAD="$(gcc -print-file-name=libasan.so):$(gcc -print-file-name=libubsan.so)"
	echo "Using LD_PRELOAD=$LD_PRELOAD"
	pytest -v --benchmark-skip -k "not remote"

	native_tests:

	needs: [lint]
	permissions:
	contents: read
	id-token: write
	attestations: write
	strategy:
	fail-fast: false
	# noinspection YAMLSchemaValidation
	matrix: >-
	${{ fromJSON(
	github.event_name == 'pull_request' && '{
	"include": [
	{"os": "ubuntu-22.04", "python-version": "3.10", "toxenv": "py310-fuse2"},
	{"os": "ubuntu-24.04", "python-version": "3.14", "toxenv": "py314-fuse3"}
	]
	}' \|\| '{
	"include": [
	{"os": "ubuntu-22.04", "python-version": "3.10", "toxenv": "py310-fuse2"},
	{"os": "ubuntu-22.04", "python-version": "3.11", "toxenv": "py311-fuse2", "binary": "borg-linux-glibc235-x86_64-gh"},
	{"os": "ubuntu-22.04-arm", "python-version": "3.11", "toxenv": "py311-fuse2", "binary": "borg-linux-glibc235-arm64-gh"},
	{"os": "ubuntu-24.04", "python-version": "3.12", "toxenv": "py312-fuse3"},
	{"os": "ubuntu-24.04", "python-version": "3.13", "toxenv": "py313-fuse3"},
	{"os": "ubuntu-24.04", "python-version": "3.14", "toxenv": "py314-fuse3"},
	{"os": "macos-13", "python-version": "3.11", "toxenv": "py311-none", "binary": "borg-macos-13-x86_64-gh"},
	{"os": "macos-14", "python-version": "3.11", "toxenv": "py311-none", "binary": "borg-macos-14-arm64-gh"}
	]
	}'
	) }}

	env:
	TOXENV: ${{ matrix.toxenv }}

	runs-on: ${{ matrix.os }}
	timeout-minutes: 180

	steps:
	- uses: actions/checkout@v4
	with:
	# Just fetching one commit is not enough for setuptools-scm, so we fetch all
	fetch-depth: 0
	fetch-tags: true

	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}

	- name: Cache pip
	uses: actions/cache@v4
	with:
	path: ~/.cache/pip
	key: ${{ runner.os }}-pip-${{ hashFiles('requirements.d/development.txt') }}
	restore-keys: \|
	${{ runner.os }}-pip-
	${{ runner.os }}-

	- name: Install Linux packages
	if: ${{ runner.os == 'Linux' }}
	run: \|
	sudo apt-get update
	sudo apt-get install -y libssl-dev libacl1-dev liblz4-dev libzstd-dev pkg-config build-essential
	sudo apt-get install -y libxxhash-dev
	if [[ "$TOXENV" == "fuse2" ]]; then
	sudo apt-get install -y libfuse-dev fuse # Required for Python llfuse module
	elif [[ "$TOXENV" == "fuse3" ]]; then
	sudo apt-get install -y libfuse3-dev fuse3 # Required for Python pyfuse3 module
	fi

	- name: Install macOS packages
	if: ${{ runner.os == 'macOS' }}
	run: brew bundle install

	- name: Install Python requirements
	run: \|
	python -m pip install --upgrade pip setuptools wheel
	pip install -r requirements.d/development.txt

	- name: Install BorgBackup
	run: \|
	if [[ "$TOXENV" == "fuse2" ]]; then
	pip install -ve ".[llfuse]"
	elif [[ "$TOXENV" == "fuse3" ]]; then
	pip install -ve ".[pyfuse3]"
	else
	pip install -ve .
	fi

	- name: Build Borg fat binaries (${{ matrix.binary }})
	if: ${{ matrix.binary && startsWith(github.ref, 'refs/tags/') }}
	run: \|
	pip install 'pyinstaller==6.14.2'
	mkdir -p dist/binary
	# Ensure locally built binaries in ./dist/binary are found during tox tests
	echo "$GITHUB_WORKSPACE/dist/binary" >> "$GITHUB_PATH"
	pyinstaller --clean --distpath=dist/binary scripts/borg.exe.spec

	- name: Smoke-test the built binary (${{ matrix.binary }})
	if: ${{ matrix.binary && startsWith(github.ref, 'refs/tags/') }}
	run: \|
	pushd dist/binary
	echo "single-file binary"
	chmod +x borg.exe
	./borg.exe -V
	echo "single-directory binary"
	chmod +x borg-dir/borg.exe
	./borg-dir/borg.exe -V
	tar czf borg.tgz borg-dir
	popd
	echo "borg.exe binary in PATH"
	borg.exe -V

	- name: Prepare binaries (${{ matrix.binary }})
	if: ${{ matrix.binary && startsWith(github.ref, 'refs/tags/') }}
	run: \|
	mkdir -p artifacts
	if [ -f dist/binary/borg.exe ]; then
	cp dist/binary/borg.exe artifacts/${{ matrix.binary }}
	fi
	if [ -f dist/binary/borg.tgz ]; then
	cp dist/binary/borg.tgz artifacts/${{ matrix.binary }}.tgz
	fi
	echo "binary files"
	ls -l artifacts/

	- name: Attest binaries provenance (${{ matrix.binary }})
	if: ${{ matrix.binary && startsWith(github.ref, 'refs/tags/') }}
	uses: actions/attest-build-provenance@v3
	with:
	subject-path: 'artifacts/*'

	- name: Upload binaries (${{ matrix.binary }})
	if: ${{ matrix.binary && startsWith(github.ref, 'refs/tags/') }}
	uses: actions/upload-artifact@v4
	with:
	name: ${{ matrix.binary }}
	path: artifacts/*
	if-no-files-found: error

	- name: Run pytest via tox
	run: \|
	# Do not use fakeroot; run as root. Avoids the dreaded sporadic EISDIR failures; see #2482.
	#sudo -E bash -c "tox -e py"
	tox --skip-missing-interpreters

	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v4
	env:
	OS: ${{ runner.os }}
	python: ${{ matrix.python-version }}
	with:
	token: ${{ secrets.CODECOV_TOKEN }}
	env_vars: OS, python

	vm_tests:

	# Cross-OS tests running inside VMs, aligned with master branch structure.
	# Uses cross-platform-actions/action to run on FreeBSD, NetBSD, OpenBSD, Haiku.
	permissions:
	contents: read
	id-token: write
	attestations: write
	runs-on: ubuntu-24.04
	timeout-minutes: 90
	needs: [lint]
	continue-on-error: true
	strategy:
	fail-fast: false
	matrix:
	include:
	- os: freebsd
	version: '14.3'
	display_name: FreeBSD
	# Controls binary build and provenance attestation on tags
	do_binaries: true
	artifact_prefix: borg-freebsd-14-x86_64-gh
	- os: netbsd
	version: '10.1'
	display_name: NetBSD
	do_binaries: false
	- os: openbsd
	version: '7.7'
	display_name: OpenBSD
	do_binaries: false
	- os: haiku
	version: 'r1beta5'
	display_name: Haiku
	do_binaries: false

	steps:
	- name: Check out repository
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	fetch-tags: true

	- name: Test on ${{ matrix.display_name }}
	id: cross_os
	uses: cross-platform-actions/[email protected]
	env:
	DO_BINARIES: ${{ matrix.do_binaries }}
	with:
	operating_system: ${{ matrix.os }}
	version: ${{ matrix.version }}
	shell: bash
	run: \|
	set -euxo pipefail
	case "${{ matrix.os }}" in
	freebsd)
	# Ensure a proper hostname/FQDN is set (VMs may not have one by default)
	sudo -E /bin/sh -c 'grep -q "freebsd\.local" /etc/hosts \|\| echo "127.0.0.1 freebsd.local freebsd" >> /etc/hosts'
	sudo -E hostname freebsd.local
	hostname
	export IGNORE_OSVERSION=yes
	sudo -E pkg update -f
	sudo -E pkg install -y xxhash liblz4 zstd pkgconf
	sudo -E pkg install -y fusefs-libs
	sudo -E kldload fusefs
	sudo -E sysctl vfs.usermount=1
	sudo -E chmod 666 /dev/fuse
	sudo -E pkg install -y rust
	sudo -E pkg install -y git
	sudo -E pkg install -y python310 py310-sqlite3
	sudo -E pkg install -y python311 py311-sqlite3 py311-pip py311-virtualenv
	sudo ln -sf /usr/local/bin/python3.11 /usr/local/bin/python3
	sudo ln -sf /usr/local/bin/python3.11 /usr/local/bin/python
	sudo ln -sf /usr/local/bin/pip3.11 /usr/local/bin/pip3
	sudo ln -sf /usr/local/bin/pip3.11 /usr/local/bin/pip
	python -m venv .venv
	. .venv/bin/activate
	python -V
	pip -V
	python -m pip install --upgrade pip wheel
	pip install -r requirements.d/development.txt
	pip install -e ".[llfuse]"
	if [[ "${{ matrix.do_binaries }}" == "true" && "${{ startsWith(github.ref, 'refs/tags/') }}" == "true" ]]; then
	python -m pip install 'pyinstaller==6.14.2'
	mkdir -p dist/binary
	pyinstaller --clean --distpath=dist/binary scripts/borg.exe.spec
	pushd dist/binary
	echo "single-file binary"
	chmod +x borg.exe
	./borg.exe -V
	echo "single-directory binary"
	chmod +x borg-dir/borg.exe
	./borg-dir/borg.exe -V
	tar czf borg.tgz borg-dir
	popd
	mkdir -p artifacts
	if [ -f dist/binary/borg.exe ]; then
	cp -v dist/binary/borg.exe artifacts/${{ matrix.artifact_prefix }}
	fi
	if [ -f dist/binary/borg.tgz ]; then
	cp -v dist/binary/borg.tgz artifacts/${{ matrix.artifact_prefix }}.tgz
	fi
	echo "binary files"
	ls -l artifacts/
	fi
	export PATH="$(pwd)/dist/binary:$PATH"
	tox -e py311-fuse2
	;;
	netbsd)
	# Ensure a proper hostname/FQDN is set (VMs may not have one by default)
	sudo -E /bin/sh -c 'grep -q "netbsd\.local" /etc/hosts \|\| echo "127.0.0.1 netbsd.local netbsd" >> /etc/hosts'
	sudo -E hostname netbsd.local
	hostname
	arch="$(uname -m)"
	sudo -E mkdir -p /usr/pkg/etc/pkgin
	echo "http://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD/${arch}/10.1/All" \| sudo tee /usr/pkg/etc/pkgin/repositories.conf > /dev/null
	sudo -E pkgin update
	sudo -E pkgin -y upgrade
	sudo -E pkgin -y install zstd lz4 xxhash git
	sudo -E pkgin -y install rust
	sudo -E pkgin -y install pkg-config
	sudo -E pkgin -y install py311-pip py311-virtualenv py311-tox
	sudo -E ln -sf /usr/pkg/bin/python3.11 /usr/pkg/bin/python3
	sudo -E ln -sf /usr/pkg/bin/pip3.11 /usr/pkg/bin/pip3
	sudo -E ln -sf /usr/pkg/bin/virtualenv-3.11 /usr/pkg/bin/virtualenv3
	sudo -E ln -sf /usr/pkg/bin/tox-3.11 /usr/pkg/bin/tox3
	# Ensure base system admin tools are on PATH for the non-root shell
	export PATH="/sbin:/usr/sbin:$PATH"
	echo "--- Preparing an extattr-enabled filesystem ---"
	# On many NetBSD setups /tmp is tmpfs without extended attributes.
	# Create a FFS image with extended attributes enabled and use it for TMPDIR.
	VNDDEV="vnd0"
	IMGFILE="/tmp/fs.img"
	sudo -E dd if=/dev/zero of=${IMGFILE} bs=1m count=1024
	sudo -E vndconfig -c "${VNDDEV}" "${IMGFILE}"
	sudo -E newfs -O 2ea /dev/r${VNDDEV}a
	MNT="/mnt/eafs"
	sudo -E mkdir -p ${MNT}
	sudo -E mount -t ffs -o extattr /dev/${VNDDEV}a $MNT
	export TMPDIR="${MNT}/tmp"
	sudo -E mkdir -p ${TMPDIR}
	sudo -E chmod 1777 ${TMPDIR}
	touch ${TMPDIR}/testfile
	lsextattr user ${TMPDIR}/testfile && echo "[xattr] * xattrs SUPPORTED on ${TMPDIR}! *"
	tox3 -e py311-none
	;;
	openbsd)
	sudo -E pkg_add xxhash lz4 zstd git
	sudo -E pkg_add rust
	sudo -E pkg_add openssl%3.4
	sudo -E pkg_add py3-pip py3-virtualenv py3-tox
	export BORG_OPENSSL_NAME=eopenssl34
	tox -e py312-none
	;;
	haiku)
	pkgman refresh
	pkgman install -y git pkgconfig zstd lz4 xxhash
	pkgman install -y openssl3
	pkgman install -y rust_bin
	pkgman install -y python3.10
	pkgman install -y cffi
	pkgman install -y lz4_devel zstd_devel xxhash_devel openssl3_devel libffi_devel
	# there is no pkgman package for tox, so we install it into a venv
	python3 -m ensurepip --upgrade
	python3 -m pip install --upgrade pip wheel
	python3 -m venv .venv
	. .venv/bin/activate
	export PKG_CONFIG_PATH="/system/develop/lib/pkgconfig:/system/lib/pkgconfig:${PKG_CONFIG_PATH:-}"
	export BORG_LIBLZ4_PREFIX=/system/develop
	export BORG_LIBZSTD_PREFIX=/system/develop
	export BORG_LIBXXHASH_PREFIX=/system/develop
	export BORG_OPENSSL_PREFIX=/system/develop
	pip install -r requirements.d/development.txt
	pip install -e .
	# troubles with either tox or pytest xdist, so we run pytest manually:
	pytest -v -rs --benchmark-skip -k "not remote and not socket"
	;;
	esac

	- name: Upload artifacts
	if: startsWith(github.ref, 'refs/tags/') && matrix.do_binaries
	uses: actions/upload-artifact@v4
	with:
	name: ${{ matrix.os }}-${{ matrix.version }}-dist
	path: artifacts/*
	if-no-files-found: ignore

	- name: Attest provenance
	if: startsWith(github.ref, 'refs/tags/') && matrix.do_binaries
	uses: actions/attest-build-provenance@v3
	with:
	subject-path: 'artifacts/*'

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

Uh oh!

Merge pull request #9204 from ThomasWaldmann/rel143 #5300

Workflow file

Merge pull request #9204 from ThomasWaldmann/rel143 #5300

Uh oh!

Jobs

Run details

Workflow file for this run