Merge bitcoin/bitcoin#33824: ci: Enable experimental kernel stuff in most CI tasks via dev-mode

fae8361 ci: [refactor] Use --preset=dev-mode in mac_native task (MarcoFalke)
fadb67b ci: [refactor] Base nowallet task on --preset=dev-mode (MarcoFalke)
6666980 ci: Enable bitcoin-chainstate and test_bitcoin-qt in win64 task (MarcoFalke)
faff7b2 ci: Enable experimental kernel stuff in i686 task (MarcoFalke)
fa1632e ci: Enable experimental kernel stuff in mac-cross tasks (MarcoFalke)
fad10ff ci: Enable experimental kernel stuff in armhf task (MarcoFalke)
fa9d67c ci: Enable experimental kernel stuff in Alpine task (MarcoFalke)
fab3fb8 ci: Enable experimental kernel stuff in s390x task (MarcoFalke)
fa7da8a ci: Enable experimental kernel stuff in valgrind task (MarcoFalke)
fa9c297 ci: Enable experimental kernel stuff in TSan task (MarcoFalke)
fad30d4 ci: Enable experimental kernel stuff in MSan task (MarcoFalke)

Pull request description:

  Most of the CI tasks have a long list of stuff that they enable. This makes it hard to see what each CI task is actually running.

  Also, most of the CI tasks should probably mimic the `dev-mode` CMake preset and run on as much stuff as possible. Usually, changing the `dev-mode` comes with changing those CI tasks as well in the same commit, which is verbose.

  Fix both issues, by basing most CI tasks on the `dev-mode`. In the future, this makes it easier to change the `dev-mode` in a single place. If CI tasks explicitly disable something, it will be listed explicitly in them.

  As a side-effect this will enable the kernel stuff for some CI task that did not have it enabled, which seems desirable.

ACKs for top commit:
  TheCharlatan:
    Nice, ACK fae8361
  janb84:
    ACK fae8361
  hebasto:
    ACK fae8361, I have reviewed the code and it looks OK.

Tree-SHA512: 58d9d553437b57362e9ec0766bd202482435f263d3f4c6ee7020c5e1e5ba69f8c064630423424f9d754254a66981e670b964a5aee58ef87f30b7d775642255be

Author: hebasto

.github/workflows/ci.yml

@@ -134,7 +134,7 @@ jobs:
         include:
           - job-type: standard
             file-env: './ci/test/00_setup_env_mac_native.sh'
-            job-name: 'macOS native, no depends, sqlite only, gui'
+            job-name: 'macOS native'
           - job-type: fuzz
             file-env: './ci/test/00_setup_env_mac_native_fuzz.sh'
             job-name: 'macOS native, fuzz'
@@ -344,7 +344,7 @@ jobs:
           py -3 test/fuzz/test_runner.py --par $NUMBER_OF_PROCESSORS --loglevel DEBUG "${RUNNER_TEMP}/qa-assets/fuzz_corpora"
 
   windows-cross:
-    name: 'Linux->Windows cross, no tests'
+    name: 'Windows-cross to x86_64'
     needs: runners
     runs-on: ${{ needs.runners.outputs.provider == 'cirrus' && 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-sm' || 'ubuntu-24.04' }}
     if: ${{ vars.SKIP_BRANCH_PUSH != 'true' || github.event_name == 'pull_request' }}
@@ -381,6 +381,7 @@ jobs:
         with:
           name: x86_64-w64-mingw32-executables-${{ github.run_id }}
           path: |
+            ${{ env.BASE_BUILD_DIR }}/bin/*.dll
             ${{ env.BASE_BUILD_DIR }}/bin/*.exe
             ${{ env.BASE_BUILD_DIR }}/src/secp256k1/bin/*.exe
             ${{ env.BASE_BUILD_DIR }}/src/univalue/*.exe
@@ -478,7 +479,7 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - name: '32 bit ARM, unit tests, no functional tests'
+          - name: '32 bit ARM'
             cirrus-runner: 'ubuntu-24.04-arm' # Cirrus' Arm runners are Apple (with virtual Linux aarch64), which doesn't support 32-bit mode
             fallback-runner: 'ubuntu-24.04-arm'
             timeout-minutes: 120
@@ -503,13 +504,13 @@ jobs:
             timeout-minutes: 120
             file-env: './ci/test/00_setup_env_mac_cross_intel.sh'
 
-          - name: 'No wallet, libbitcoinkernel'
+          - name: 'No wallet'
             cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-sm'
             fallback-runner: 'ubuntu-24.04'
             timeout-minutes: 120
-            file-env: './ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh'
+            file-env: './ci/test/00_setup_env_native_nowallet.sh'
 
-          - name: 'no IPC, i686, DEBUG'
+          - name: 'i686, no IPC'
             cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md'
             fallback-runner: 'ubuntu-24.04'
             timeout-minutes: 120
@@ -533,7 +534,7 @@ jobs:
             timeout-minutes: 120
             file-env: './ci/test/00_setup_env_native_previous_releases.sh'
 
-          - name: 'Alpine (musl), depends, gui'
+          - name: 'Alpine (musl)'
             cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md'
             fallback-runner: 'ubuntu-24.04'
             timeout-minutes: 120
@@ -545,7 +546,7 @@ jobs:
             timeout-minutes: 120
             file-env: './ci/test/00_setup_env_native_tidy.sh'
 
-          - name: 'TSan, depends, no gui'
+          - name: 'TSan'
             cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md'
             fallback-runner: 'ubuntu-24.04'
             timeout-minutes: 120
@@ -557,7 +558,7 @@ jobs:
             timeout-minutes: 150
             file-env: './ci/test/00_setup_env_native_fuzz_with_msan.sh'
 
-          - name: 'MSan, depends'
+          - name: 'MSan'
             cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-lg'
             fallback-runner: 'ubuntu-24.04'
             timeout-minutes: 120

ci/test/00_setup_env_arm.sh

@@ -20,6 +20,7 @@ export CI_LIMIT_STACK_SIZE=1
 # -Wno-psabi is to disable ABI warnings: "note: parameter passing for argument of type ... changed in GCC 7.1"
 # This could be removed once the ABI change warning does not show up by default
 export BITCOIN_CONFIG=" \
+  --preset=dev-mode \
   -DREDUCE_EXPORTS=ON \
   -DCMAKE_CXX_FLAGS='-Wno-psabi -Wno-error=maybe-uninitialized' \
 "

ci/test/00_setup_env_i686_no_ipc.sh

@@ -16,6 +16,8 @@ export GOAL="install"
 export CI_LIMIT_STACK_SIZE=1
 export TEST_RUNNER_EXTRA="--v2transport --usecli"
 export BITCOIN_CONFIG="\
+ --preset=dev-mode \
+ -DENABLE_IPC=OFF \
  -DCMAKE_BUILD_TYPE=Debug \
  -DCMAKE_C_COMPILER='clang;-m32' \
  -DCMAKE_CXX_COMPILER='clang++;-m32' \

ci/test/00_setup_env_mac_cross.sh

@@ -17,4 +17,8 @@ export XCODE_BUILD_ID=15A240d
 export RUN_UNIT_TESTS=false
 export RUN_FUNCTIONAL_TESTS=false
 export GOAL="deploy"
-export BITCOIN_CONFIG="-DBUILD_GUI=ON -DBUILD_KERNEL_LIB=ON -DREDUCE_EXPORTS=ON"
+export BITCOIN_CONFIG="\
+ --preset=dev-mode \
+ -DWITH_USDT=OFF \
+ -DREDUCE_EXPORTS=ON \
+"

ci/test/00_setup_env_mac_cross_intel.sh

@@ -17,4 +17,8 @@ export XCODE_BUILD_ID=15A240d
 export RUN_UNIT_TESTS=false
 export RUN_FUNCTIONAL_TESTS=false
 export GOAL="deploy"
-export BITCOIN_CONFIG="-DBUILD_GUI=ON -DREDUCE_EXPORTS=ON"
+export BITCOIN_CONFIG="\
+ --preset=dev-mode \
+ -DWITH_USDT=OFF \
+ -DREDUCE_EXPORTS=ON \
+"

ci/test/00_setup_env_mac_native.sh

@@ -10,8 +10,13 @@ export CONTAINER_NAME="ci_mac_native"  # macos does not use a container, but the
 export PIP_PACKAGES="--break-system-packages pycapnp zmq"
 export GOAL="install deploy"
 export CMAKE_GENERATOR="Ninja"
-export BITCOIN_CONFIG="-DBUILD_GUI=ON -DWITH_ZMQ=ON -DBUILD_KERNEL_LIB=ON -DBUILD_UTIL_CHAINSTATE=ON -DBUILD_KERNEL_TEST=ON -DREDUCE_EXPORTS=ON -DCMAKE_EXE_LINKER_FLAGS='-Wl,-stack_size -Wl,0x80000'"
 export CI_OS_NAME="macos"
 export NO_DEPENDS=1
 export OSX_SDK=""
+export BITCOIN_CONFIG="\
+  --preset=dev-mode \
+  -DWITH_USDT=OFF \
+  -DREDUCE_EXPORTS=ON \
+  -DCMAKE_EXE_LINKER_FLAGS='-Wl,-stack_size -Wl,0x80000' \
+"
 export BITCOIN_CMD="bitcoin -m" # Used in functional tests

ci/test/00_setup_env_native_alpine_musl.sh

@@ -13,8 +13,7 @@ export PIP_PACKAGES="--break-system-packages pyzmq pycapnp"
 export DEP_OPTS="DEBUG=1"
 export GOAL="install"
 export BITCOIN_CONFIG="\
- -DWITH_ZMQ=ON \
- -DBUILD_GUI=ON \
+ --preset=dev-mode \
  -DREDUCE_EXPORTS=ON \
  -DCMAKE_BUILD_TYPE=Debug \
 "

ci/test/00_setup_env_native_msan.sh

@@ -22,6 +22,8 @@ export CI_LIMIT_STACK_SIZE=1
 # Setting CMAKE_{C,CXX}_FLAGS_DEBUG flags to an empty string ensures that the flags set in MSAN_FLAGS remain unaltered.
 # _FORTIFY_SOURCE is not compatible with MSAN.
 export BITCOIN_CONFIG="\
+ --preset=dev-mode \
+ -DBUILD_GUI=OFF \
  -DCMAKE_BUILD_TYPE=Debug \
  -DCMAKE_C_FLAGS_DEBUG='' \
  -DCMAKE_CXX_FLAGS_DEBUG='' \

ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh renamed to ci/test/00_setup_env_native_nowallet.sh

@@ -6,11 +6,15 @@
 
 export LC_ALL=C.UTF-8
 
-export CONTAINER_NAME=ci_native_nowallet_libbitcoinkernel
+export CONTAINER_NAME=ci_native_nowallet
 export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04"
 # Use minimum supported python3.10 (or best-effort 3.12) and clang-17, see doc/dependencies.md
 export PACKAGES="python3-zmq python3-pip clang-17 llvm-17 libc++abi-17-dev libc++-17-dev"
 export PIP_PACKAGES="--break-system-packages pycapnp"
 export DEP_OPTS="NO_WALLET=1 CC=clang-17 CXX='clang++-17 -stdlib=libc++'"
 export GOAL="install"
-export BITCOIN_CONFIG="-DREDUCE_EXPORTS=ON -DBUILD_UTIL_CHAINSTATE=ON -DBUILD_KERNEL_LIB=ON -DBUILD_KERNEL_TEST=ON -DBUILD_SHARED_LIBS=ON"
+export BITCOIN_CONFIG="\
+  --preset=dev-mode \
+  -DREDUCE_EXPORTS=ON \
+  -DENABLE_WALLET=OFF \
+"

ci/test/00_setup_env_native_tsan.sh

@@ -16,6 +16,10 @@ export PIP_PACKAGES="--break-system-packages pycapnp"
 export DEP_OPTS="CC=clang CXX=clang++ CXXFLAGS='${LIBCXX_FLAGS}' NO_QT=1"
 export GOAL="install"
 export CI_LIMIT_STACK_SIZE=1
-export BITCOIN_CONFIG="-DWITH_ZMQ=ON -DSANITIZERS=thread \
--DAPPEND_CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKCONTENTION -D_LIBCPP_REMOVE_TRANSITIVE_INCLUDES'"
+export BITCOIN_CONFIG="\
+  --preset=dev-mode \
+  -DBUILD_GUI=OFF \
+  -DSANITIZERS=thread \
+  -DAPPEND_CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKCONTENTION -D_LIBCPP_REMOVE_TRANSITIVE_INCLUDES' \
+"
 export USE_INSTRUMENTED_LIBCPP="Thread"