diff --git a/.github/workflows/build-packages-daily-master.yml b/.github/workflows/build-packages-daily-master.yml index 367d307bd486..880be238e6b5 100644 --- a/.github/workflows/build-packages-daily-master.yml +++ b/.github/workflows/build-packages-daily-master.yml @@ -20,33 +20,42 @@ jobs: product: 'authoritative' ref: 'auth-4.9.3' secrets: - PULP_REPO_NAME: ${{ secrets.PULP_REPO_NAME }} + PULP_FILE_REPO_NAME: ${{ secrets.PULP_FILE_REPO_NAME }} PULP_URL: ${{ secrets.PULP_URL }} PULP_CI_USERNAME: ${{ secrets.PULP_CI_USERNAME }} PULP_CI_PASSWORD: ${{ secrets.PULP_CI_PASSWORD }} + DOWNLOADS_AUTOBUILT_SECRET: ${{ secrets.DOWNLOADS_AUTOBUILT_SECRET }} + DOWNLOADS_AUTOBUILT_RSYNCTARGET: ${{ secrets.DOWNLOADS_AUTOBUILT_RSYNCTARGET }} + DOWNLOADS_AUTOBUILT_HOSTKEY: ${{ secrets.DOWNLOADS_AUTOBUILT_HOSTKEY }} - # call-build-packages-dnsdist: - # # if: ${{ vars.SCHEDULED_PACKAGES_DAILY }} - # uses: romeroalx/pdns/.github/workflows/build-packages.yml@test-pulp-file-repository - # with: - # is_release: 'YES' - # product: 'dnsdist' - # ref: 'dnsdist-1.9.8' - # secrets: - # PULP_REPO_NAME: ${{ secrets.PULP_REPO_NAME }} - # PULP_URL: ${{ secrets.PULP_URL }} - # PULP_CI_USERNAME: ${{ secrets.PULP_CI_USERNAME }} - # PULP_CI_PASSWORD: ${{ secrets.PULP_CI_PASSWORD }} -# - # call-build-packages-rec: - # # if: ${{ vars.SCHEDULED_PACKAGES_DAILY }} - # uses: romeroalx/pdns/.github/workflows/build-packages.yml@test-pulp-file-repository - # with: - # is_release: 'YES' - # product: 'recursor' - # ref: 'rec-5.2.0' - # secrets: - # PULP_REPO_NAME: ${{ secrets.PULP_REPO_NAME }} - # PULP_URL: ${{ secrets.PULP_URL }} - # PULP_CI_USERNAME: ${{ secrets.PULP_CI_USERNAME }} - # PULP_CI_PASSWORD: ${{ secrets.PULP_CI_PASSWORD }} + call-build-packages-dnsdist: + # if: ${{ vars.SCHEDULED_PACKAGES_DAILY }} + uses: romeroalx/pdns/.github/workflows/build-packages.yml@test-pulp-file-repository + with: + product: 'dnsdist' + is_release: 'NO' + ref: 'master' + secrets: + PULP_FILE_REPO_NAME: ${{ secrets.PULP_FILE_REPO_NAME }} + PULP_URL: ${{ secrets.PULP_URL }} + PULP_CI_USERNAME: ${{ secrets.PULP_CI_USERNAME }} + PULP_CI_PASSWORD: ${{ secrets.PULP_CI_PASSWORD }} + DOWNLOADS_AUTOBUILT_SECRET: ${{ secrets.DOWNLOADS_AUTOBUILT_SECRET }} + DOWNLOADS_AUTOBUILT_RSYNCTARGET: ${{ secrets.DOWNLOADS_AUTOBUILT_RSYNCTARGET }} + DOWNLOADS_AUTOBUILT_HOSTKEY: ${{ secrets.DOWNLOADS_AUTOBUILT_HOSTKEY }} + + call-build-packages-rec: + # if: ${{ vars.SCHEDULED_PACKAGES_DAILY }} + uses: romeroalx/pdns/.github/workflows/build-packages.yml@test-pulp-file-repository + with: + is_release: 'YES' + product: 'recursor' + ref: 'rec-5.2.0' + secrets: + PULP_FILE_REPO_NAME: ${{ secrets.PULP_FILE_REPO_NAME }} + PULP_URL: ${{ secrets.PULP_URL }} + PULP_CI_USERNAME: ${{ secrets.PULP_CI_USERNAME }} + PULP_CI_PASSWORD: ${{ secrets.PULP_CI_PASSWORD }} + DOWNLOADS_AUTOBUILT_SECRET: ${{ secrets.DOWNLOADS_AUTOBUILT_SECRET }} + DOWNLOADS_AUTOBUILT_RSYNCTARGET: ${{ secrets.DOWNLOADS_AUTOBUILT_RSYNCTARGET }} + DOWNLOADS_AUTOBUILT_HOSTKEY: ${{ secrets.DOWNLOADS_AUTOBUILT_HOSTKEY }} diff --git a/.github/workflows/build-packages.yml b/.github/workflows/build-packages.yml index c59fcb350e08..6483796dec03 100644 --- a/.github/workflows/build-packages.yml +++ b/.github/workflows/build-packages.yml @@ -16,7 +16,12 @@ on: # update this list, as well as the one in builder-dispatch.yml default: >- el-8 + el-9 + debian-bullseye debian-bookworm + ubuntu-focal + ubuntu-jammy + ubuntu-noble ref: description: git ref to checkout type: string @@ -28,7 +33,13 @@ on: required: false default: 'NO' secrets: - PULP_REPO_NAME: + DOWNLOADS_AUTOBUILT_SECRET: + required: true + DOWNLOADS_AUTOBUILT_RSYNCTARGET: + required: true + DOWNLOADS_AUTOBUILT_HOSTKEY: + required: true + PULP_FILE_REPO_NAME: required: true PULP_URL: required: true @@ -36,8 +47,6 @@ on: required: true PULP_CI_PASSWORD: required: true -env: - DESTINATION_PATH: built_pkgs_pdns permissions: # least privileges, see https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions contents: read @@ -58,10 +67,8 @@ jobs: - id: get-oslist run: echo "oslist=$(jo -a ${{ inputs.os }})" >> "$GITHUB_OUTPUT" - id: get-runnerlist - # run: echo "runnerlist=$(jo -a ubuntu-22.04)" >> "$GITHUB_OUTPUT" run: echo "runnerlist=$(jo -a ubuntu-22.04 ${{ vars.ARM64_USE_UBICLOUD == '1' && 'ubicloud-standard-2-arm' || '' }})" >> "$GITHUB_OUTPUT" - id: get-archlist - # run: echo "archlist=$(jo -a x86_64)" >> "$GITHUB_OUTPUT" run: echo "archlist=$(jo -a x86_64 ${{ vars.ARM64_USE_UBICLOUD == '1' && 'aarch64' || '' }})" >> "$GITHUB_OUTPUT" build: @@ -82,20 +89,20 @@ jobs: # pkghashes-el-7-x86_64: ${{ steps.pkghashes.outputs.pkghashes-el-7-x86_64 }} pkghashes-el-8-x86_64: ${{ steps.pkghashes.outputs.pkghashes-el-8-x86_64 }} pkghashes-el-8-aarch64: ${{ steps.pkghashes.outputs.pkghashes-el-8-aarch64 }} - # pkghashes-el-9-x86_64: ${{ steps.pkghashes.outputs.pkghashes-el-9-x86_64 }} - # pkghashes-el-9-aarch64: ${{ steps.pkghashes.outputs.pkghashes-el-9-aarch64 }} + pkghashes-el-9-x86_64: ${{ steps.pkghashes.outputs.pkghashes-el-9-x86_64 }} + pkghashes-el-9-aarch64: ${{ steps.pkghashes.outputs.pkghashes-el-9-aarch64 }} # pkghashes-debian-buster-x86_64: ${{ steps.pkghashes.outputs.pkghashes-debian-buster-x86_64 }} # pkghashes-debian-buster-aarch64: ${{ steps.pkghashes.outputs.pkghashes-debian-buster-aarch64 }} - # pkghashes-debian-bullseye-x86_64: ${{ steps.pkghashes.outputs.pkghashes-debian-bullseye-x86_64 }} - # pkghashes-debian-bullseye-aarch64: ${{ steps.pkghashes.outputs.pkghashes-debian-bullseye-aarch64 }} + pkghashes-debian-bullseye-x86_64: ${{ steps.pkghashes.outputs.pkghashes-debian-bullseye-x86_64 }} + pkghashes-debian-bullseye-aarch64: ${{ steps.pkghashes.outputs.pkghashes-debian-bullseye-aarch64 }} pkghashes-debian-bookworm-x86_64: ${{ steps.pkghashes.outputs.pkghashes-debian-bookworm-x86_64 }} pkghashes-debian-bookworm-aarch64: ${{ steps.pkghashes.outputs.pkghashes-debian-bookworm-aarch64 }} - # pkghashes-ubuntu-focal-x86_64: ${{ steps.pkghashes.outputs.pkghashes-ubuntu-focal-x86_64 }} - # pkghashes-ubuntu-focal-aarch64: ${{ steps.pkghashes.outputs.pkghashes-ubuntu-focal-aarch64 }} - # pkghashes-ubuntu-jammy-x86_64: ${{ steps.pkghashes.outputs.pkghashes-ubuntu-jammy-x86_64 }} - # pkghashes-ubuntu-jammy-aarch64: ${{ steps.pkghashes.outputs.pkghashes-ubuntu-jammy-aarch64 }} - # pkghashes-ubuntu-noble-x86_64: ${{ steps.pkghashes.outputs.pkghashes-ubuntu-noble-x86_64 }} - # pkghashes-ubuntu-noble-aarch64: ${{ steps.pkghashes.outputs.pkghashes-ubuntu-noble-aarch64 }} + pkghashes-ubuntu-focal-x86_64: ${{ steps.pkghashes.outputs.pkghashes-ubuntu-focal-x86_64 }} + pkghashes-ubuntu-focal-aarch64: ${{ steps.pkghashes.outputs.pkghashes-ubuntu-focal-aarch64 }} + pkghashes-ubuntu-jammy-x86_64: ${{ steps.pkghashes.outputs.pkghashes-ubuntu-jammy-x86_64 }} + pkghashes-ubuntu-jammy-aarch64: ${{ steps.pkghashes.outputs.pkghashes-ubuntu-jammy-aarch64 }} + pkghashes-ubuntu-noble-x86_64: ${{ steps.pkghashes.outputs.pkghashes-ubuntu-noble-x86_64 }} + pkghashes-ubuntu-noble-aarch64: ${{ steps.pkghashes.outputs.pkghashes-ubuntu-noble-aarch64 }} srchashes: ${{ steps.srchashes.outputs.srchashes }} steps: - uses: actions/checkout@v4 @@ -136,7 +143,6 @@ jobs: id: extract run: | mkdir -m 700 -p ./packages/ - ls ./built_pkgs/* tar xvf ./built_pkgs/*/*/${{ steps.normalize-name.outputs.normalized-package-name }}-${{ steps.getversion.outputs.version }}-${{ matrix.os }}-${{ steps.getarch.outputs.target-arch }}.tar.bz2 -C ./packages/ --transform='s/.*\///' - name: Generate package hashes for provenance shell: bash @@ -148,32 +154,19 @@ jobs: id: srchashes run: | echo "srchashes=$(sha256sum ./built_pkgs/*/*/${{ steps.normalize-name.outputs.normalized-package-name }}-${{ steps.getversion.outputs.version }}.tar.bz2 | base64 -w0)" >> $GITHUB_OUTPUT - # - name: Install python3 invoke - # run: | - # sudo apt-get update && sudo apt-get -y install python3-invoke - # git checkout test-pulp-file-repository - # - run: inv install-pulpcli - # - name: Upload DEB packages - # run: inv pulp-upload-packages-by-folder ${{ env.DESTINATION_PATH }} built_pkgs - # env: - # PULP_REPO_NAME: ${{ secrets.PULP_REPO_NAME }} - # PULP_URL: ${{ secrets.PULP_URL }} - # PULP_CI_USERNAME: ${{ secrets.PULP_CI_USERNAME }} - # PULP_CI_PASSWORD: ${{ secrets.PULP_CI_PASSWORD }} - # - name: Upload RPM packages - # run: inv pulp-upload-packages-by-folder ${{ env.DESTINATION_PATH }} built_pkgs - # env: - # PULP_REPO_NAME: ${{ secrets.PULP_REPO_NAME }} - # PULP_URL: ${{ secrets.PULP_URL }} - # PULP_CI_USERNAME: ${{ secrets.PULP_CI_USERNAME }} - # PULP_CI_PASSWORD: ${{ secrets.PULP_CI_PASSWORD }} - # - name: Upload packages to Pulp - # run: inv pulp-upload-packages-by-folder ${{ env.DESTINATION_PATH }} built_pkgs - # env: - # PULP_REPO_NAME: ${{ secrets.PULP_REPO_NAME }} - # PULP_URL: ${{ secrets.PULP_URL }} - # PULP_CI_USERNAME: ${{ secrets.PULP_CI_USERNAME }} - # PULP_CI_PASSWORD: ${{ secrets.PULP_CI_PASSWORD }} + - name: Upload packages to downloads.powerdns.com + env: + SSHKEY: ${{ secrets.DOWNLOADS_AUTOBUILT_SECRET }} + RSYNCTARGET: ${{ secrets.DOWNLOADS_AUTOBUILT_RSYNCTARGET }} + HOSTKEY: ${{ secrets.DOWNLOADS_AUTOBUILT_HOSTKEY }} + if: + "${{ env.SSHKEY != '' }}" + run: | + mkdir -m 700 -p ~/.ssh + echo "$SSHKEY" > ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + echo "$HOSTKEY" > ~/.ssh/known_hosts + rsync -4rlptD built_pkgs/* "$RSYNCTARGET" check-hashes: needs: build @@ -228,89 +221,256 @@ jobs: upload-assets: false provenance-name: "${{ inputs.product }}-${{ needs.build.outputs.version }}-src.intoto.jsonl" private-repository: true -# -# upload-provenance: -# needs: [prepare, build, provenance-src, provenance-pkgs] -# name: Upload the provenance artifacts to downloads.powerdns.com -# runs-on: ubuntu-22.04 -# strategy: -# matrix: -# os: ${{fromJson(needs.prepare.outputs.oslist)}} -# architecture: ${{ fromJson(needs.prepare.outputs.archlist )}} -# exclude: -# - os: el-7 -# architecture: aarch64 -# steps: -# - uses: actions/checkout@v4 -# with: -# fetch-depth: 1 -# submodules: recursive -# ref: test-pulp-file-repository -# - name: Download source tarball provenance for ${{ inputs.product }} (${{ inputs.ref }}) -# id: download-src-provenance -# uses: actions/download-artifact@v4 # be careful, this needs to match what https://github.com/slsa-framework/slsa-github-generator is using -# with: -# name: "${{ inputs.product }}-${{ needs.build.outputs.version }}-src.intoto.jsonl" -# - name: Download provenance for ${{ inputs.product }} (${{ inputs.ref }}) for ${{ matrix.os }} -# id: download-provenance -# uses: actions/download-artifact@v4 # be careful, this needs to match what https://github.com/slsa-framework/slsa-github-generator is using -# with: -# name: "${{ inputs.product }}-${{ needs.build.outputs.version }}-${{ matrix.os }}-${{ matrix.architecture }}.intoto.jsonl" -# # - name: Install python3 invoke -# # run: | -# # sudo apt-get update && sudo apt-get -y install python3-invoke -# # - run: inv install-pulpcli -# # - name: Upload provenance artifacts to Pulp -# # id: upload-provenance -# # env: -# # PULP_REPO_NAME: ${{ secrets.PULP_REPO_NAME }} -# # PULP_URL: ${{ secrets.PULP_URL }} -# # PULP_CI_USERNAME: ${{ secrets.PULP_CI_USERNAME }} -# # PULP_CI_PASSWORD: ${{ secrets.PULP_CI_PASSWORD }} -# # PRODUCT: ${{ needs.build.outputs.product-name }} -# # VERSION: ${{ needs.build.outputs.version }} -# # shell: bash -# # run: | -# # mkdir -m 755 -p "slsa/${PRODUCT}/${VERSION}/" -# # mv ${{steps.download-provenance.outputs.download-path}}/*.jsonl "slsa/${PRODUCT}/${VERSION}" -# # inv pulp-upload-packages-by-folder ${{ env.DESTINATION_PATH }} slsa + + upload-provenance: + needs: [prepare, build, provenance-src, provenance-pkgs] + name: Upload the provenance artifacts to downloads.powerdns.com + runs-on: ubuntu-22.04 + strategy: + matrix: + os: ${{fromJson(needs.prepare.outputs.oslist)}} + architecture: ${{ fromJson(needs.prepare.outputs.archlist )}} + exclude: + - os: el-7 + architecture: aarch64 + steps: + - name: Download source tarball provenance for ${{ inputs.product }} (${{ inputs.ref }}) + id: download-src-provenance + uses: actions/download-artifact@v4 # be careful, this needs to match what https://github.com/slsa-framework/slsa-github-generator is using + with: + name: "${{ inputs.product }}-${{ needs.build.outputs.version }}-src.intoto.jsonl" + - name: Download provenance for ${{ inputs.product }} (${{ inputs.ref }}) for ${{ matrix.os }} + id: download-provenance + uses: actions/download-artifact@v4 # be careful, this needs to match what https://github.com/slsa-framework/slsa-github-generator is using + with: + name: "${{ inputs.product }}-${{ needs.build.outputs.version }}-${{ matrix.os }}-${{ matrix.architecture }}.intoto.jsonl" + - name: Upload provenance artifacts to downloads.powerdns.com + id: upload-provenance + env: + SSHKEY: ${{ secrets.DOWNLOADS_AUTOBUILT_SECRET }} + RSYNCTARGET: ${{ secrets.DOWNLOADS_AUTOBUILT_RSYNCTARGET }} + HOSTKEY: ${{ secrets.DOWNLOADS_AUTOBUILT_HOSTKEY }} + PRODUCT: ${{ needs.build.outputs.product-name }} + VERSION: ${{ needs.build.outputs.version }} + if: + "${{ env.SSHKEY != '' }}" + shell: bash + run: | + mkdir -m 700 -p ~/.ssh + echo "$SSHKEY" > ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + echo "$HOSTKEY" > ~/.ssh/known_hosts + mkdir -m 755 -p "slsa/${PRODUCT}/${VERSION}/" + mv ${{steps.download-provenance.outputs.download-path}}/*.jsonl "slsa/${PRODUCT}/${VERSION}" + rsync -4rlptD slsa/* "$RSYNCTARGET" upload-packages: needs: [prepare, build, provenance-src, provenance-pkgs] + # continue-on-error: true name: Upload packages runs-on: ubuntu-22.04 + strategy: + matrix: + os: ${{fromJson(needs.prepare.outputs.oslist)}} + architecture: ${{ fromJson(needs.prepare.outputs.archlist )}} + exclude: + - os: el-7 + architecture: aarch64 + fail-fast: false steps: - uses: actions/checkout@v4 with: - fetch-depth: 0 # for correct version numbers + fetch-depth: 1 submodules: recursive - ref: ${{ inputs.ref }} - # this builds packages and runs our unit tests (make check) - - name: Get version number + ref: test-pulp-file-repository + - name: Install python3 invoke run: | - echo "version=$(readlink builder/tmp/latest)" >> $GITHUB_OUTPUT - id: getversion + sudo apt-get update && sudo apt-get -y install python3-invoke + - run: inv install-pulpcli - name: Download packages - id: download-packages - uses: actions/download-artifact@v4 # be careful, this needs to match what https://github.com/slsa-framework/slsa-github-generator is using + uses: actions/download-artifact@v4 with: - pattern: "${{ inputs.product }}-*" + pattern: "${{ inputs.product }}-${{ matrix.os }}-*-${{ matrix.architecture }}" - name: Normalize package name id: normalize-name run: | + shortversion=$(echo ${{ needs.build.outputs.version }} | grep -qq master && echo master || echo ${{ needs.build.outputs.version }} | awk -F. '{print $1$2}') if [ "x${{ inputs.product }}" = "xauthoritative" ]; then echo "normalized-package-name=pdns" >> $GITHUB_OUTPUT + echo "normalized-product-name=auth-${shortversion}" >> $GITHUB_OUTPUT elif [ "x${{ inputs.product }}" = "xrecursor" ]; then echo "normalized-package-name=pdns-recursor" >> $GITHUB_OUTPUT + echo "normalized-product-name=rec-${shortversion}" >> $GITHUB_OUTPUT else echo "normalized-package-name=${{ inputs.product }}" >> $GITHUB_OUTPUT + echo "normalized-product-name=${{ inputs.product }}-${shortversion}" >> $GITHUB_OUTPUT fi - - run: sudo apt update && sudo apt install -y tree - - run: tree - - run: ls -al - - run: pwd - - run: mkdir -p files/${{ steps.normalize-name.outputs.normalized-package-name }}/${{ needs.build.outputs.version }} - - run: for i in $(ls ./${{ inputs.product }}-*/*/*/*); do cp $i files/${{ steps.normalize-name.outputs.normalized-package-name }}/${{ needs.build.outputs.version }}/; done - - run: cp *.jsonl/*.jsonl files/${{ steps.normalize-name.outputs.normalized-package-name }}/${{ needs.build.outputs.version }}/ - - run: for i in $(ls ./*/*/*/*-*-*-*.tar.bz2); do mkdir -p ./packages/$(echo $i | cut -d '/' -f 2)/; tar -xvf $i -C ./packages/$(echo $i | cut -d '/' -f 2)/ --transform='s/.*\///'; done - - run: tree + - name: Extract and prepare files and packages + run: | + # Prepare files folder + mkdir -p files/${{ steps.normalize-name.outputs.normalized-package-name }}/${{ needs.build.outputs.version }} + # Copy compressed and provenance files + cp ${{ inputs.product }}-${{ matrix.os }}-*-${{ matrix.architecture }}/*/*/*-${{ matrix.architecture }}.tar.bz2 files/${{ steps.normalize-name.outputs.normalized-package-name }}/${{ needs.build.outputs.version }}/ + # Extract packages + for i in $(ls ./${{ inputs.product }}-${{ matrix.os }}-*-${{ matrix.architecture }}/*/*/*${{ matrix.architecture }}.tar.bz2); do mkdir -p ./packages/$(echo $i | cut -d '/' -f 2)/; tar -xvf $i -C ./packages/$(echo $i | cut -d '/' -f 2)/ --transform='s/.*\///'; done + # Move JSON files from packages + for i in $(ls ./packages/*/*.json); do mv $i files/${{ steps.normalize-name.outputs.normalized-package-name }}/${{ needs.build.outputs.version }}/; done + # Move RPM files + for i in $(ls ./packages/*/*.rpm); do f=$(echo $i | awk -F/ '{print $3}' | awk -F'${{ inputs.product }}-' '{print $2}' | awk -F'-${{ needs.build.outputs.version }}-' '{print $1"."$2}'); mkdir -p rpm_packages/${f}; mv $i rpm_packages/${f}/; done + # Move deb files + for i in $(ls ./packages/*/*.deb); do mkdir -p deb_packages/$(echo $i | awk -F/ '{print $3}' | awk -F- '{print $2"-"$3}'); mv $i deb_packages/$(echo $i | awk -F/ '{print $3}' | awk -F- '{print $2"-"$3}'); done + - name: Upload provenance and other data + run: inv pulp-upload-file-packages-by-folder files + env: + PULP_REPO_NAME: ${{ secrets.PULP_FILE_REPO_NAME }} + PULP_URL: ${{ secrets.PULP_URL }} + PULP_CI_USERNAME: ${{ secrets.PULP_CI_USERNAME }} + PULP_CI_PASSWORD: ${{ secrets.PULP_CI_PASSWORD }} + - name: Upload DEB Packages + if: ${{ contains(matrix.os, 'ubuntu-') || ( contains(matrix.os, 'debian-')) }} + run: inv pulp-upload-deb-packages-by-folder deb_packages ${{ steps.normalize-name.outputs.normalized-product-name }} + env: + PULP_URL: ${{ secrets.PULP_URL }} + PULP_CI_USERNAME: ${{ secrets.PULP_CI_USERNAME }} + PULP_CI_PASSWORD: ${{ secrets.PULP_CI_PASSWORD }} + # - name: Publish DEB Packages + # if: ${{ contains(matrix.os, 'ubuntu-') || ( contains(matrix.os, 'debian-')) }} + # run: inv pulp-create-deb-publication + # env: + # PULP_URL: ${{ secrets.PULP_URL }} + # PULP_CI_USERNAME: ${{ secrets.PULP_CI_USERNAME }} + # PULP_CI_PASSWORD: ${{ secrets.PULP_CI_PASSWORD }} + - name: Upload RPM Packages + if: ${{ contains(matrix.os, 'el-') || ( contains(matrix.os, 'centos-')) }} + run: inv pulp-upload-rpm-packages-by-folder rpm_packages ${{ steps.normalize-name.outputs.normalized-product-name }} + env: + PULP_URL: ${{ secrets.PULP_URL }} + PULP_CI_USERNAME: ${{ secrets.PULP_CI_USERNAME }} + PULP_CI_PASSWORD: ${{ secrets.PULP_CI_PASSWORD }} + # - name: Publish RPM Packages + # if: ${{ contains(matrix.os, 'el-') || ( contains(matrix.os, 'centos-')) }} + # run: inv pulp-create-rpm-publication ${{ steps.normalize-name.outputs.normalized-product-name }} ${{fromJson(needs.prepare.outputs.oslist)}} ${{ fromJson(needs.prepare.outputs.archlist )}} + # env: + # PULP_URL: ${{ secrets.PULP_URL }} + # PULP_CI_USERNAME: ${{ secrets.PULP_CI_USERNAME }} + # PULP_CI_PASSWORD: ${{ secrets.PULP_CI_PASSWORD }} + + upload-src-files: + needs: [prepare, build, provenance-src, provenance-pkgs, upload-packages] + name: Upload provenance files + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + submodules: recursive + ref: test-pulp-file-repository + - name: Install python3 invoke + run: | + sudo apt-get update && sudo apt-get -y install python3-invoke + - run: inv install-pulpcli + - name: Download provenance files + uses: actions/download-artifact@v4 + with: + pattern: "*.intoto.jsonl" + - name: Normalize package name + id: normalize-name + run: | + if [ "x${{ inputs.product }}" = "xauthoritative" ]; then + echo "normalized-package-name=pdns" >> $GITHUB_OUTPUT + elif [ "x${{ inputs.product }}" = "xrecursor" ]; then + echo "normalized-package-name=pdns-recursor" >> $GITHUB_OUTPUT + else + echo "normalized-package-name=${{ inputs.product }}" >> $GITHUB_OUTPUT + fi + - name: Extract and prepare files and packages + run: | + # Prepare files folder + mkdir -p files/${{ steps.normalize-name.outputs.normalized-package-name }}/${{ needs.build.outputs.version }} + # Copy compressed and provenance files + cp *.jsonl/*.jsonl files/${{ steps.normalize-name.outputs.normalized-package-name }}/${{ needs.build.outputs.version }}/ + - name: Upload provenance and other data + run: inv pulp-upload-file-packages-by-folder files + env: + PULP_REPO_NAME: ${{ secrets.PULP_FILE_REPO_NAME }} + PULP_URL: ${{ secrets.PULP_URL }} + PULP_CI_USERNAME: ${{ secrets.PULP_CI_USERNAME }} + PULP_CI_PASSWORD: ${{ secrets.PULP_CI_PASSWORD }} + + publish-packages: + needs: [prepare, build, upload-packages, upload-src-files] + # continue-on-error: true + name: Publish packages + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + submodules: recursive + ref: test-pulp-file-repository + - name: Install python3 invoke + run: | + sudo apt-get update && sudo apt-get -y install python3-invoke + - run: inv install-pulpcli + - name: Normalize product name + id: normalize-name + run: | + shortversion=$(echo ${{ needs.build.outputs.version }} | grep -qq master && echo master || echo ${{ needs.build.outputs.version }} | awk -F. '{print $1$2}') + if [ "x${{ inputs.product }}" = "xauthoritative" ]; then + echo "normalized-product-name=auth-${shortversion}" >> $GITHUB_OUTPUT + elif [ "x${{ inputs.product }}" = "xrecursor" ]; then + echo "normalized-product-name=rec-${shortversion}" >> $GITHUB_OUTPUT + else + echo "normalized-product-name=${{ inputs.product }}-${shortversion}" >> $GITHUB_OUTPUT + fi + - name: Publish DEB Packages + run: inv pulp-create-deb-publication + env: + PULP_URL: ${{ secrets.PULP_URL }} + PULP_CI_USERNAME: ${{ secrets.PULP_CI_USERNAME }} + PULP_CI_PASSWORD: ${{ secrets.PULP_CI_PASSWORD }} + - name: Publish RPM Packages + run: inv pulp-create-rpm-publication ${{ steps.normalize-name.outputs.normalized-product-name }} '${{ needs.prepare.outputs.oslist }}' '${{ needs.prepare.outputs.archlist }}' + env: + PULP_URL: ${{ secrets.PULP_URL }} + PULP_CI_USERNAME: ${{ secrets.PULP_CI_USERNAME }} + PULP_CI_PASSWORD: ${{ secrets.PULP_CI_PASSWORD }} + + test-packages: + needs: [prepare, build, provenance-src, provenance-pkgs, upload-packages] + # continue-on-error: true + name: Test packages + runs-on: ${{ matrix.runner-os }} + strategy: + matrix: + os: ${{fromJson(needs.prepare.outputs.oslist)}} + runner-os: ${{ fromJson(needs.prepare.outputs.runnerlist )}} + exclude: + - os: el-7 + runner-os: ubicloud-standard-2-arm + fail-fast: false + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + submodules: recursive + ref: test-pulp-file-repository + - name: Install python3 invoke + run: | + sudo apt-get update && sudo apt-get -y install python3-invoke + - name: Normalize product name + id: normalize-name + run: | + shortversion=$(echo ${{ needs.build.outputs.version }} | grep -qq master && echo master || echo ${{ needs.build.outputs.version }} | awk -F. '{print $1$2}') + if [ "x${{ inputs.product }}" = "xauthoritative" ]; then + echo "normalized-package-name=pdns" >> $GITHUB_OUTPUT + echo "normalized-product-name=auth-${shortversion}" >> $GITHUB_OUTPUT + elif [ "x${{ inputs.product }}" = "xrecursor" ]; then + echo "normalized-package-name=pdns-recursor" >> $GITHUB_OUTPUT + echo "normalized-product-name=rec-${shortversion}" >> $GITHUB_OUTPUT + else + echo "normalized-package-name=${{ inputs.product }}" >> $GITHUB_OUTPUT + echo "normalized-product-name=${{ inputs.product }}-${shortversion}" >> $GITHUB_OUTPUT + fi + + - run: inv test-install-package ${{ steps.normalize-name.outputs.normalized-product-name }} ${{ matrix.os }} http://pulp-dev.cluster.powerdns.equipment/pulp/content http://pulp-dev.cluster.powerdns.equipment/pulp/content/repo-files/RPM-GPG-PUB ${{ steps.normalize-name.outputs.normalized-package-name }} ${{ needs.build.outputs.version }} diff --git a/tasks.py b/tasks.py index 2be438775003..e1b03eb3bb2e 100644 --- a/tasks.py +++ b/tasks.py @@ -5,6 +5,7 @@ import os import sys import time +import requests auth_backend_ip_addr = os.getenv('AUTH_BACKEND_IP_ADDR', '127.0.0.1') @@ -1229,36 +1230,190 @@ def ci_build_and_install_quiche(c, repo): @task def install_pulpcli(c): c.run(f'python3 -m venv {repo_home}/.venv') - c.run(f'. {repo_home}/.venv/bin/activate && pip install pulp-cli==0.29.2') + c.run(f'. {repo_home}/.venv/bin/activate && pip install pulp-cli==0.29.2 pulp-cli-deb==0.3.1') + +pulp_cmd = " ".join([ + "pulp", + f"--base-url {os.getenv('PULP_URL', '')}", + f"--username {os.getenv('PULP_CI_USERNAME', '')}", + f"--password {os.getenv('PULP_CI_PASSWORD', '')}" +]) + +def pulp_create_publication(c, repo_name, repo_type, additional_params): + max_push_attempts = 3 + attempts = 0 + # It could happen that a new version was created in the middle of an upload so a retry is needed + while attempts < max_push_attempts: + try: + c.run(f'. {repo_home}/.venv/bin/activate && {pulp_cmd} {repo_type} publication create --repository {repo_name} {additional_params}') + break + except UnexpectedExit: + attempts += 1 + time.sleep(5) + print(f'Next attempt: {attempts}') + if attempts == max_push_attempts: + raise Failure(f'Error creating {repo_type} publication') @task -def pulp_upload_packages_by_folder(c, destination_path, source): - pulp_repo_name = os.getenv("PULP_REPO_NAME", '') - pulp_cmd = " ".join([ - "pulp", - "--no-verify-ssl", - f"--base-url {os.getenv('PULP_URL', '')}", - f"--username {os.getenv('PULP_CI_USERNAME', '')}", - f"--password {os.getenv('PULP_CI_PASSWORD', '')}" - ]) +def pulp_upload_file_packages_by_folder(c, source): + repo_name = os.getenv("PULP_REPO_NAME", '') for root, dirs, files in os.walk(source): for path in files: file = os.path.join(root, path).split('/',1)[1] - c.run(f'. {repo_home}/.venv/bin/activate && {pulp_cmd} file content upload --repository {pulp_repo_name} --file {source}/{file} --relative-path {destination_path}/{file}') - time.sleep(5) + # file repositories have been configured with autopublish set to true + c.run(f'. {repo_home}/.venv/bin/activate && {pulp_cmd} file content upload --repository {repo_name} --file {source}/{file} --relative-path {file}') @task -def pulp_get_repos(c): - # pulp_repo_name = os.getenv("PULP_REPO_NAME", '') - pulp_cmd = " ".join([ - "pulp", - "--no-verify-ssl", - f"--base-url {os.getenv('PULP_URL', '')}", - f"--username {os.getenv('PULP_CI_USERNAME', '')}", - f"--password {os.getenv('PULP_CI_PASSWORD', '')}" - ]) +def pulp_create_rpm_publication(c, product, list_os_rel, list_arch): + max_push_attempts = 3 + rpm_distros = ["centos", "el"] + for os_rel in json.loads(list_os_rel): + if not "el-" in os_rel: + break + release = os_rel.split('-')[1] + for arch in json.loads(list_arch): + for distro in rpm_distros: + repo_name = f"repo-{distro}-{release}-{arch}-{product}" + attempts = 0 + while attempts < max_push_attempts: + try: + c.run(f'. {repo_home}/.venv/bin/activate && {pulp_cmd} rpm publication create --repository {repo_name} --checksum-type sha512') + break + except UnexpectedExit: + attempts += 1 + time.sleep(5) + print(f'Next attempt: {attempts}') + if attempts == max_push_attempts: + raise Failure(f'Error creating rpm publication') + +@task +def pulp_create_deb_publication(c): + max_push_attempts = 3 + deb_distros = ["debian", "ubuntu"] + for distro in deb_distros: + repo_name = f"repo-{distro}" + attempts = 0 + while attempts < max_push_attempts: + try: + c.run(f'. {repo_home}/.venv/bin/activate && {pulp_cmd} deb publication create --repository {repo_name}') + break + except UnexpectedExit: + attempts += 1 + time.sleep(5) + print(f'Next attempt: {attempts}') + if attempts == max_push_attempts: + raise Failure(f'Error creating deb publication') +@task +def pulp_upload_rpm_packages_by_folder(c, source, product): + rpm_distros = ["centos", "el"] + builds = os.listdir(source) + + for build_folder in builds: + release = build_folder.split('.')[0].split('-')[1] + arch = build_folder.split('.')[1] + for distro in rpm_distros: + repo_name = f"repo-{distro}-{release}-{arch}-{product}" + for root, dirs, files in os.walk(f"{source}/{build_folder}"): + for path in files: + file = os.path.join(root, path).split('/',1)[1] + # Set chunk size to 500MB to avoid creating an "upload" instead of a file. Required for singing RPMs. + c.run(f'. {repo_home}/.venv/bin/activate && {pulp_cmd} rpm content -t package upload --file {source}/{file} --repository {repo_name} --no-publish --chunk-size 500MB') + + # pulp_create_publication(c, repo_name, 'rpm', '--checksum-type sha512') + +def get_pulp_repository_href(c, repo_name, repo_type): + res = c.run(f". {repo_home}/.venv/bin/activate && {pulp_cmd} {repo_type} repository show --name {repo_name} | jq -r '.pulp_href' | tr -d '\n'") + if res.exited != 0: + raise UnexpectedExit(res) + return res.stdout + +def is_pulp_task_completed(c, task_href): + elapsed_time = 0 + check_interval = 5 + max_wait_time = 60 + + while elapsed_time < max_wait_time: + res = c.run(f". {repo_home}/.venv/bin/activate && {pulp_cmd} task show --href {task_href} | jq -r .state | tr -d '\n'") + if res.exited != 0: + raise UnexpectedExit(res) + elif res.stdout == "completed": + return True + time.sleep(check_interval) + elapsed_time += check_interval + + return False + +@task +def pulp_upload_deb_packages_by_folder(c, source, product): + builds = os.listdir(source) + upload_url = os.getenv('PULP_URL', '') + "/pulp/api/v3/content/deb/packages/" + headers = {"Content-Type": "application/json"} + auth = requests.auth.HTTPBasicAuth(os.getenv("PULP_CI_USERNAME", ""), os.getenv("PULP_CI_PASSWORD", "")) + + for build_folder in builds: + distro = build_folder.split('-')[0] + distribution = f"{build_folder.split('-')[1]}-{product}" + repo_name = f"repo-{distro}" + repository_href = get_pulp_repository_href(c, repo_name, "deb") + + for root, dirs, files in os.walk(source): + for path in files: + file = os.path.join(root, path).split('/',1)[1] + res = c.run(f". {repo_home}/.venv/bin/activate && {pulp_cmd} artifact upload --file {source}/{file} | jq -r '.pulp_href' | tr -d '\n'") + if res.exited != 0: + raise UnexpectedExit(res) + artifact_href = res.stdout + + package_data = { + "repository": repository_href, + "distribution": distribution, + "component": "main", + "artifact": artifact_href + } + res = requests.post(upload_url, auth=auth, headers=headers, json=package_data) + res.raise_for_status() + task_href = res.json().get('task') + if not is_pulp_task_completed(c, task_href): + raise Failure('Error uploading DEB packages into Pulp') + + # pulp_create_publication(c, repo_name, 'deb', '') + +@task +def test_install_package(c, product_name, distro_release, content_url, gpgkey_url, package_name, package_version): + distro, release = distro_release.split('-') + repo_domain = content_url.split('/')[2] + if 'el' in distro_release: + image_name = 'oraclelinux' + else: + image_name = distro + # pdns package is called pdns-server for debian/ubuntu + package_name = 'pdns-server' if package_name == 'pdns' else package_name + + dockerfile_rpm = f''' +FROM {image_name}:{release} +RUN curl -L {content_url}/repo-files/{distro}-{product_name}.repo -o /etc/yum.repos.d/{distro}-{product_name}.repo +RUN yum install -y epel-release +RUN yum install -y libsodium boost-devel luajit-devel net-snmp net-snmp-utils net-snmp-libs fstrm libxdp re2 tinycdb lmdb-libs +RUN yum install -y --disablerepo="*" --enablerepo="$(cat /etc/yum.repos.d/{distro}-{product_name}.repo | head -1 | cut -d '[' -f 2 | cut -d ']' -f 1)" {package_name}-{package_version}* +''' + dockerfile_deb = f''' +FROM {image_name}:{release} +RUN apt update && apt install -y curl libluajit-5.1-dev +RUN install -d /etc/apt/keyrings && curl -L {gpgkey_url} -o /etc/apt/keyrings/{product_name}-pub.asc +RUN echo "deb [signed-by=/etc/apt/keyrings/{product_name}-pub.asc] {content_url}/{distro} {release}-{product_name} main" | tee /etc/apt/sources.list.d/pdns.list +RUN bash -c 'echo -e "Package: auth*\\nPin: origin {repo_domain}\\nPin-Priority: 600" | tee /etc/apt/preferences.d/{product_name}' +RUN apt-get update && apt-get install -y {package_name}={package_version}* +''' + dockerfile = dockerfile_rpm if 'el-' in distro_release else dockerfile_deb + with open('/tmp/Dockerfile', "w") as f: + f.write(dockerfile) + + c.run('docker build . -f /tmp/Dockerfile') + +@task +def pulp_get_repos(c): c.run(f'. {repo_home}/.venv/bin/activate && {pulp_cmd} repository list') # this is run always