diff --git a/.github/workflows/presto-release-publish.yml b/.github/workflows/presto-release-publish.yml index 3ac315f13d229..ca3fc85399c18 100644 --- a/.github/workflows/presto-release-publish.yml +++ b/.github/workflows/presto-release-publish.yml @@ -291,179 +291,24 @@ jobs: ./presto-benchmark-driver/target/presto-benchmark-driver-*-executable.jar ./presto-testing-server-launcher/target/presto-testing-server-launcher-*-executable.jar - publish-docker-image: - needs: publish-maven-artifacts - if: (!failure() && !cancelled()) && github.event.inputs.publish_image == 'true' - runs-on: ubuntu-latest - environment: release - timeout-minutes: 150 - permissions: - packages: write - contents: read - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - ref: ${{ env.RELEASE_TAG}} - persist-credentials: false - - - name: Download artifacts - uses: actions/download-artifact@v4 - with: - name: presto-artifacts-${{ env.RELEASE_TAG }} - path: ./ - - - name: Login to dockerhub - uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Set up qemu - uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0 - - - name: Set up docker buildx - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - - - name: Create and use builder - run: | - docker buildx create --name container --use - docker buildx inspect --bootstrap - - - name: Move artifacts to docker directory - run: | - mv ./presto-server/target/presto-server-*.tar.gz docker/ - mv ./presto-cli/target/presto-cli-*-executable.jar docker/ - - - name: Build docker image and publish - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 - with: - context: docker - platforms: linux/amd64,linux/arm64,linux/ppc64le - file: docker/Dockerfile - push: true - build-args: | - PRESTO_VERSION=${{ env.RELEASE_TAG }} - JMX_PROMETHEUS_JAVAAGENT_VERSION=0.20.0 - tags: | - ${{ env.DOCKER_REPO }}:${{ env.RELEASE_TAG }} - ${{ github.event.inputs.tag_image_as_latest == 'true' && format('{0}:latest', env.DOCKER_REPO) || '' }} - - publish-native-image: - needs: publish-release-tag - if: (!failure() && !cancelled()) && github.event.inputs.publish_native_image == 'true' - runs-on: ubuntu-latest + publish-docker-images: + needs: [publish-maven-artifacts, publish-release-tag] + if: (!failure() && !cancelled()) && (github.event.inputs.publish_image == 'true' || github.event.inputs.publish_native_image == 'true') + uses: ./.github/workflows/publish-docker-images.yml + with: + branch_or_tag: ${{ github.event.inputs.RELEASE_VERSION }} + os: centos + tag_suffix: '' + tag_latest: ${{ github.event.inputs.tag_image_as_latest == 'true' }} + publish_dependency: ${{ github.event.inputs.dependency_image == '' && github.event.inputs.publish_native_image == 'true' }} + publish_presto: ${{ github.event.inputs.publish_image == 'true' }} + publish_prestissimo: ${{ github.event.inputs.publish_native_image == 'true' }} + secrets: inherit permissions: contents: read packages: write attestations: write id-token: write - environment: release - timeout-minutes: 300 - steps: - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - with: - tool-cache: false - large-packages: false - docker-images: false - swap-storage: false - - - name: Checkout - uses: actions/checkout@v4 - with: - ref: ${{ env.RELEASE_TAG }} - submodules: true - persist-credentials: false - - - name: Initialize Prestissimo submodules - run: | - df -h - cd presto-native-execution && make submodules - echo "COMMIT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_ENV - - - name: Login to DockerHub - uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Set dependency image tag - env: - DEPENDENCY_IMAGE: ${{ github.event.inputs.dependency_image }} - RELEASE_TAG: ${{ env.RELEASE_TAG }} - run: | - if [[ -n "$DEPENDENCY_IMAGE" ]]; then - echo "DEPENDENCY_IMAGE=$DEPENDENCY_IMAGE" >> $GITHUB_ENV - else - echo "DEPENDENCY_IMAGE=${{ github.repository_owner }}/presto-native-dependency:$RELEASE_TAG-${{ env.COMMIT_SHA }}" >> $GITHUB_ENV - fi - - - name: Build Dependency Image - working-directory: presto-native-execution - env: - PUBLISH_NATIVE_IMAGE: ${{ github.event.inputs.publish_native_image }} - TAG_IMAGE_AS_LATEST: ${{ github.event.inputs.tag_image_as_latest }} - RELEASE_TAG: ${{ env.RELEASE_TAG }} - run: | - df -h - if docker pull ${{ env.DEPENDENCY_IMAGE }}; then - echo "Using dependency image ${{ env.DEPENDENCY_IMAGE }}" - docker tag ${{ env.DEPENDENCY_IMAGE }} presto/prestissimo-dependency:centos9 - else - echo "Building new depedency image" - docker compose build centos-native-dependency - if [[ "$PUBLISH_NATIVE_IMAGE" == "true" ]]; then - docker tag presto/prestissimo-dependency:centos9 ${{ github.repository_owner }}/presto-native-dependency:$RELEASE_TAG-${{ env.COMMIT_SHA }} - docker push ${{ github.repository_owner }}/presto-native-dependency:$RELEASE_TAG-${{ env.COMMIT_SHA }} - - if [[ "$TAG_IMAGE_AS_LATEST" == "true" ]]; then - docker tag presto/prestissimo-dependency:centos9 ${{ github.repository_owner }}/presto-native-dependency:latest - docker push ${{ github.repository_owner }}/presto-native-dependency:latest - fi - fi - fi - docker images - - - name: Build Runtime Image - working-directory: presto-native-execution - run: | - df -h - docker compose build --build-arg EXTRA_CMAKE_FLAGS=" - -DPRESTO_ENABLE_PARQUET=ON \ - -DPRESTO_ENABLE_REMOTE_FUNCTIONS=ON \ - -DPRESTO_ENABLE_JWT=ON \ - -DPRESTO_STATS_REPORTER_TYPE=PROMETHEUS \ - -DPRESTO_MEMORY_CHECKER_TYPE=LINUX_MEMORY_CHECKER \ - -DPRESTO_ENABLE_SPATIAL=ON \ - -DPRESTO_ENABLE_TESTING=OFF \ - -DPRESTO_ENABLE_S3=ON" \ - --build-arg NUM_THREADS=2 \ - centos-native-runtime - - - name: Add release tag - working-directory: presto-native-execution - env: - TAG_IMAGE_AS_LATEST: ${{ github.event.inputs.tag_image_as_latest }} - ORG_NAME: ${{ env.ORG_NAME }} - RELEASE_TAG: ${{ env.RELEASE_TAG }} - run: | - docker tag presto/prestissimo-runtime:centos9 $ORG_NAME/${{ env.IMAGE_NAME }}:$RELEASE_TAG - if [[ "$TAG_IMAGE_AS_LATEST" == "true" ]]; then - docker tag presto/prestissimo-runtime:centos9 $ORG_NAME/${{ env.IMAGE_NAME }}:latest - fi - - - name: Push to DockerHub - env: - TAG_IMAGE_AS_LATEST: ${{ github.event.inputs.tag_image_as_latest }} - ORG_NAME: ${{ env.ORG_NAME }} - RELEASE_TAG: ${{ env.RELEASE_TAG }} - run: | - docker push $ORG_NAME/${{ env.IMAGE_NAME }}:$RELEASE_TAG - if [[ "$TAG_IMAGE_AS_LATEST" == "true" ]]; then - docker tag $ORG_NAME/${{ env.IMAGE_NAME }}:$RELEASE_TAG $ORG_NAME/${{ env.IMAGE_NAME }}:latest - docker push $ORG_NAME/${{ env.IMAGE_NAME }}:latest - fi publish-docs: needs: publish-maven-artifacts diff --git a/.github/workflows/publish-docker-images.yml b/.github/workflows/publish-docker-images.yml new file mode 100644 index 0000000000000..2239850008db3 --- /dev/null +++ b/.github/workflows/publish-docker-images.yml @@ -0,0 +1,608 @@ +name: Publish Docker Images + +on: + workflow_dispatch: + inputs: + branch_or_tag: + description: Branch or tag to checkout (e.g., master, 0.295) + required: true + default: master + os: + description: Operating system (ubuntu/centos) + required: true + default: centos + type: choice + options: + - centos + - ubuntu + tag_suffix: + description: Tag suffix (can be empty) + required: false + default: '' + tag_latest: + description: Tag the image as latest + type: boolean + default: true + required: false + publish_dependency: + description: Publish dependency image + type: boolean + default: true + required: false + publish_presto: + description: Publish presto images + type: boolean + default: true + required: false + publish_prestissimo: + description: Publish prestissimo images + type: boolean + default: true + required: false + workflow_call: + inputs: + branch_or_tag: + description: Branch or tag to checkout (e.g., master, 0.295) + required: true + type: string + os: + description: Operating system (ubuntu/centos) + required: true + type: string + tag_suffix: + description: Tag suffix (can be empty) + required: false + type: string + default: '' + tag_latest: + description: Tag the image as latest + type: boolean + default: true + required: false + publish_dependency: + description: Publish dependency image + type: boolean + default: true + required: false + publish_presto: + description: Publish presto images + type: boolean + default: true + required: false + publish_prestissimo: + description: Publish prestissimo images + type: boolean + default: true + required: false + +concurrency: + group: publish-docker-images + cancel-in-progress: false + +env: + JAVA_VERSION: ${{ vars.JAVA_VERSION || '17' }} + JAVA_DISTRIBUTION: ${{ vars.JAVA_DISTRIBUTION || 'temurin' }} + DOCKER_REPO: ${{ github.repository }} + ORG_NAME: ${{ github.repository_owner }} + GIT_CI_USER: ${{ vars.GIT_CI_USER || 'prestodb-ci' }} + GIT_CI_EMAIL: ${{ vars.GIT_CI_EMAIL || 'ci@lists.prestodb.io' }} + JMX_PROMETHEUS_JAVAAGENT_VERSION: 0.20.0 + PRESTO_IMAGE_NAME: presto + NATIVE_IMAGE_NAME: presto-native + DEPENDENCY_IMAGE_NAME: presto-native-dependency + TAG_SUFFIX: ${{ inputs.tag_suffix }} + INPUT_OS: ${{ inputs.os }} + INPUT_TAG_LATEST: ${{ inputs.tag_latest }} + INPUT_PUBLISH_DEPENDENCY: ${{ inputs.publish_dependency }} + +jobs: + prepare: + runs-on: ubuntu-24.04 + outputs: + version: ${{ steps.extract-version.outputs.version }} + commit_sha: ${{ steps.extract-commit.outputs.commit_sha }} + presto_version: ${{ steps.extract-version.outputs.presto_version }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch_or_tag }} + fetch-depth: 1 + + - name: Configure git + run: | + git config --global user.email "${{ env.GIT_CI_EMAIL }}" + git config --global user.name "${{ env.GIT_CI_USER }}" + + - name: Extract commit SHA + id: extract-commit + run: | + COMMIT_SHA=$(git rev-parse --short HEAD) + echo "commit_sha=${COMMIT_SHA}" >> $GITHUB_OUTPUT + echo "Commit SHA: ${COMMIT_SHA}" + + - name: Set up JDK ${{ env.JAVA_DISTRIBUTION }}/${{ env.JAVA_VERSION }} + uses: actions/setup-java@v4 + with: + java-version: ${{ env.JAVA_VERSION }} + distribution: ${{ env.JAVA_DISTRIBUTION }} + + - name: Extract version + id: extract-version + run: | + VERSION=$(./mvnw help:evaluate -Dexpression=project.version -q -DforceStdout) + echo "Raw version: $VERSION" + + if [ -z "$VERSION" ]; then + echo "Failed to extract project version with Maven" + exit 1 + fi + + if [[ "$VERSION" == *"-SNAPSHOT" ]]; then + # Remove -SNAPSHOT and append commit SHA + CLEAN_VERSION=${VERSION%-SNAPSHOT} + TAG_VERSION="${CLEAN_VERSION}-${{ steps.extract-commit.outputs.commit_sha }}" + echo "SNAPSHOT version detected, using: $TAG_VERSION" + else + TAG_VERSION="$VERSION" + echo "Release version detected, using: $TAG_VERSION" + fi + + echo "version=${TAG_VERSION}" >> $GITHUB_OUTPUT + echo "presto_version=${VERSION}" >> $GITHUB_OUTPUT + + publish-dependency-image: + needs: prepare + if: ( !failure() && !cancelled() && inputs.publish_dependency ) + strategy: + matrix: + arch: [amd64, arm64-generic, arm64] + fail-fast: false + runs-on: ${{ matrix.arch == 'amd64' && 'ubuntu-24.04' || 'ubuntu-24.04-arm' }} + environment: release + permissions: + packages: write + contents: read + steps: + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 + with: + tool-cache: false + docker-images: false + large-packages: false + + - name: Set up JDK ${{ env.JAVA_DISTRIBUTION }}/${{ env.JAVA_VERSION }} + uses: actions/setup-java@v4 + with: + java-version: ${{ env.JAVA_VERSION }} + distribution: ${{ env.JAVA_DISTRIBUTION }} + + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch_or_tag }} + fetch-depth: 10 + + - name: Reset to specific commit + run: | + git reset --hard ${{ needs.prepare.outputs.commit_sha }} + echo "Using commit SHA: ${{ needs.prepare.outputs.commit_sha }}" + + - name: Configure git + run: | + git config --global user.email "${{ env.GIT_CI_EMAIL }}" + git config --global user.name "${{ env.GIT_CI_USER }}" + + - name: Checkout submodules + working-directory: presto-native-execution + run: | + df -h + make submodules + + - name: Set version + run: | + echo "VERSION=${{ needs.prepare.outputs.version }}" >> $GITHUB_ENV + echo "Using version: ${{ needs.prepare.outputs.version }}" + + - name: Login to DockerHub + uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.0.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set image tag + run: | + TAG_BASE="${{ env.ORG_NAME }}/${{ env.DEPENDENCY_IMAGE_NAME }}:${INPUT_OS}-${{ env.VERSION }}-${{ matrix.arch }}" + echo "IMAGE_TAG=${TAG_BASE}${TAG_SUFFIX}" >> $GITHUB_ENV + + if [[ "${INPUT_OS}" == "ubuntu" ]]; then + echo "DEPENDENCY_TARGET=ubuntu-native-dependency" >> $GITHUB_ENV + echo "LOCAL_IMAGE_TAG=presto/prestissimo-dependency:ubuntu-22.04" >> $GITHUB_ENV + else + echo "DEPENDENCY_TARGET=centos-native-dependency" >> $GITHUB_ENV + echo "LOCAL_IMAGE_TAG=presto/prestissimo-dependency:centos9" >> $GITHUB_ENV + fi + + # Store the dependency image tag for later jobs + echo "DEPENDENCY_IMAGE_TAG=${TAG_BASE}" >> $GITHUB_ENV + + - name: Build image + working-directory: presto-native-execution + run: | + df -h + echo "Using image tag: $IMAGE_TAG" + + if [[ "${{ matrix.arch }}" == "arm64-generic" ]]; then + BUILD_ARGS="--build-arg ARM_BUILD_TARGET=generic" + else + BUILD_ARGS="" + fi + + echo "BUILD_ARGS=${BUILD_ARGS}" + docker compose build ${BUILD_ARGS} ${{ env.DEPENDENCY_TARGET }} + + - name: Publish image + run: | + set -e + docker tag ${{ env.LOCAL_IMAGE_TAG }} ${{ env.IMAGE_TAG }} + docker push ${{ env.IMAGE_TAG }} + + if [[ "${INPUT_TAG_LATEST}" == "true" ]]; then + LATEST_TAG="${{ env.ORG_NAME }}/${{ env.DEPENDENCY_IMAGE_NAME }}:${INPUT_OS}-${{ matrix.arch }}-latest" + docker tag ${{ env.LOCAL_IMAGE_TAG }} ${LATEST_TAG} + docker push ${LATEST_TAG} + echo "Tagged and pushed as latest: ${LATEST_TAG}" + fi + + create-dependency-manifest: + if: ( !failure() && !cancelled() && inputs.publish_dependency ) + needs: [prepare, publish-dependency-image] + runs-on: ubuntu-24.04 + environment: release + permissions: + packages: write + contents: read + steps: + - name: Login to DockerHub + uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.0.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set version + run: | + echo "VERSION=${{ needs.prepare.outputs.version }}" >> $GITHUB_ENV + echo "Using version: ${{ needs.prepare.outputs.version }}" + + - name: Create and push multi-arch manifest + env: + IMAGE_NAME: ${{ env.DEPENDENCY_IMAGE_NAME }} + ORG_NAME: ${{ github.repository_owner }} + OS: ${{ inputs.os }} + VERSION: ${{ env.VERSION }} + run: | + # Create manifest for the versioned tag + MANIFEST_TAG="${ORG_NAME}/${IMAGE_NAME}:${OS}-${VERSION}${TAG_SUFFIX}" + AMD64_TAG="${ORG_NAME}/${IMAGE_NAME}:${OS}-${VERSION}-amd64${TAG_SUFFIX}" + ARM64_GENERIC_TAG="${ORG_NAME}/${IMAGE_NAME}:${OS}-${VERSION}-arm64-generic${TAG_SUFFIX}" + ARM64_TAG="${ORG_NAME}/${IMAGE_NAME}:${OS}-${VERSION}-arm64${TAG_SUFFIX}" + + echo "Creating manifest: ${MANIFEST_TAG}" + docker manifest create ${MANIFEST_TAG} ${AMD64_TAG} ${ARM64_GENERIC_TAG} ${ARM64_TAG} + docker manifest push ${MANIFEST_TAG} + + # Create latest manifest if requested + if [[ "${INPUT_TAG_LATEST}" == "true" ]]; then + # Create OS-specific latest manifest + LATEST_MANIFEST="${ORG_NAME}/${IMAGE_NAME}:${OS}-latest" + LATEST_AMD64="${ORG_NAME}/${IMAGE_NAME}:${OS}-amd64-latest" + LATEST_ARM64_GENERIC="${ORG_NAME}/${IMAGE_NAME}:${OS}-arm64-generic-latest" + LATEST_ARM64="${ORG_NAME}/${IMAGE_NAME}:${OS}-arm64-latest" + + # If OS is centos, also tag as latest (without OS prefix) + if [[ "${OS}" == "centos" ]]; then + GLOBAL_LATEST="${ORG_NAME}/${IMAGE_NAME}:latest" + echo "Creating global latest manifest: ${GLOBAL_LATEST}" + docker manifest create ${GLOBAL_LATEST} ${LATEST_AMD64} ${LATEST_ARM64_GENERIC} ${LATEST_ARM64} + docker manifest push ${GLOBAL_LATEST} + else + echo "Creating OS-specific latest manifest: ${LATEST_MANIFEST}" + docker manifest create ${LATEST_MANIFEST} ${LATEST_AMD64} ${LATEST_ARM64_GENERIC} ${LATEST_ARM64} + docker manifest push ${LATEST_MANIFEST} + fi + fi + + publish-presto-image: + if: (!failure() && !cancelled() && inputs.publish_presto) + needs: prepare + runs-on: ubuntu-24.04 + environment: release + permissions: + packages: write + contents: read + steps: + - name: Check OS compatibility + run: | + if [[ "${INPUT_OS}" == "ubuntu" ]]; then + echo "::error::Presto Docker image is only supported for CentOS. Ubuntu is not supported for the Presto server image." + exit 1 + fi + + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 + with: + tool-cache: false + docker-images: false + large-packages: false + + - name: Set up JDK ${{ env.JAVA_DISTRIBUTION }}/${{ env.JAVA_VERSION }} + uses: actions/setup-java@v4 + with: + java-version: ${{ env.JAVA_VERSION }} + distribution: ${{ env.JAVA_DISTRIBUTION }} + + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch_or_tag }} + fetch-depth: 10 + + - name: Reset to specific commit + run: | + git reset --hard ${{ needs.prepare.outputs.commit_sha }} + echo "Using commit SHA: ${{ needs.prepare.outputs.commit_sha }}" + + - name: Configure git + run: | + git config --global user.email "${{ env.GIT_CI_EMAIL }}" + git config --global user.name "${{ env.GIT_CI_USER }}" + + - name: Set version + run: | + echo "VERSION=${{ needs.prepare.outputs.version }}" >> $GITHUB_ENV + echo "PRESTO_VERSION=${{ needs.prepare.outputs.presto_version }}" >> $GITHUB_ENV + echo "Using version: ${{ needs.prepare.outputs.version }}" + + - name: Build Presto Server + run: | + df -h + ./mvnw clean install -DskipTests -T1C + df -h + + - name: Login to DockerHub + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set up qemu + uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0 + + - name: Set up docker buildx + uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1 + + - name: Create and use builder + run: | + docker buildx create --name container --use + docker buildx inspect --bootstrap + + - name: Set image tag and base image + run: | + TAG_BASE="${{ env.ORG_NAME }}/${{ env.PRESTO_IMAGE_NAME }}:${{ env.VERSION }}" + echo "IMAGE_TAG=${TAG_BASE}${TAG_SUFFIX}" >> $GITHUB_ENV + + - name: Move artifacts to docker directory + run: | + mkdir -p docker_build + cp presto-server/target/presto-server-*.tar.gz docker/ + cp presto-cli/target/presto-cli-*-executable.jar docker/ + + - name: Build docker image and publish + uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75 # v6.9.0 + with: + context: docker + platforms: linux/amd64,linux/arm64,linux/ppc64le + file: docker/Dockerfile + push: true + build-args: | + PRESTO_VERSION=${{ env.PRESTO_VERSION }} + JMX_PROMETHEUS_JAVAAGENT_VERSION=0.20.0 + tags: | + ${{ env.IMAGE_TAG }} + ${{ inputs.tag_latest == 'true' && format('{0}/{1}:latest', env.ORG_NAME, env.PRESTO_IMAGE_NAME) || '' }} + + publish-prestissimo-image: + if: (!failure() && !cancelled() && inputs.publish_prestissimo) + needs: [prepare, publish-dependency-image] + strategy: + matrix: + arch: [amd64, arm64-generic, arm64] + fail-fast: false + runs-on: ${{ matrix.arch == 'amd64' && 'ubuntu-24.04' || 'ubuntu-24.04-arm' }} + environment: release + timeout-minutes: 150 + permissions: + packages: write + contents: read + steps: + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 + with: + tool-cache: false + docker-images: false + large-packages: false + + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch_or_tag }} + fetch-depth: 10 + + - name: Reset to specific commit + run: | + git reset --hard ${{ needs.prepare.outputs.commit_sha }} + echo "Using commit SHA: ${{ needs.prepare.outputs.commit_sha }}" + + - name: Configure git + run: | + git config --global user.email "${{ env.GIT_CI_EMAIL }}" + git config --global user.name "${{ env.GIT_CI_USER }}" + + - name: Checkout submodules + working-directory: presto-native-execution + run: | + df -h + make submodules + + # Use version from prepare job + - name: Set version + run: | + echo "VERSION=${{ needs.prepare.outputs.version }}" >> $GITHUB_ENV + echo "Using version: ${{ needs.prepare.outputs.version }}" + + - name: Login to DockerHub + uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.0.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set image tag and base image + run: | + TAG_BASE="${{ env.ORG_NAME }}/${{ env.NATIVE_IMAGE_NAME }}:${INPUT_OS}-${{ env.VERSION }}-${{ matrix.arch }}" + echo "IMAGE_TAG=${TAG_BASE}${TAG_SUFFIX}" >> $GITHUB_ENV + + if [[ "${INPUT_OS}" == "ubuntu" ]]; then + echo "BASE_IMAGE=ubuntu:22.04" >> $GITHUB_ENV + else + echo "BASE_IMAGE=quay.io/centos/centos:stream9" >> $GITHUB_ENV + fi + + # Set dependency image based on whether we built it or need to use latest from dockerhub + if [[ "${INPUT_PUBLISH_DEPENDENCY}" == "true" ]]; then + # Use the dependency image we just built + DEPENDENCY_IMAGE="${{ env.ORG_NAME }}/presto-native-dependency:${INPUT_OS}-${{ env.VERSION }}-${{ matrix.arch }}${TAG_SUFFIX}" + else + # Use the latest dependency image from dockerhub + if [[ "${INPUT_OS}" == "centos" ]]; then + DEPENDENCY_IMAGE="${{ env.ORG_NAME }}/presto-native-dependency:${{ matrix.arch }}-latest" + else + DEPENDENCY_IMAGE="${{ env.ORG_NAME }}/presto-native-dependency:${INPUT_OS}-${{ matrix.arch }}-latest" + fi + fi + echo "DEPENDENCY_IMAGE=${DEPENDENCY_IMAGE}" >> $GITHUB_ENV + + if [[ "${INPUT_OS}" == "ubuntu" ]]; then + LOCAL_IMAGE_TAG="presto/prestissimo-dependency:ubuntu-22.04" + else + LOCAL_IMAGE_TAG="presto/prestissimo-dependency:centos9" + fi + docker pull ${DEPENDENCY_IMAGE} + docker tag ${DEPENDENCY_IMAGE} ${LOCAL_IMAGE_TAG} + + - name: Build prestissimo image + working-directory: presto-native-execution + run: | + df -h + echo "Using image tag: $IMAGE_TAG" + echo "Using dependency image: ${{ env.DEPENDENCY_IMAGE }}" + echo "Using base image: $BASE_IMAGE" + EXTRA_CMAKE_FLAGS=" \ + -DPRESTO_ENABLE_PARQUET=ON \ + -DPRESTO_ENABLE_REMOTE_FUNCTIONS=ON \ + -DPRESTO_ENABLE_JWT=ON \ + -DPRESTO_STATS_REPORTER_TYPE=PROMETHEUS \ + -DPRESTO_MEMORY_CHECKER_TYPE=LINUX_MEMORY_CHECKER \ + -DPRESTO_ENABLE_SPATIAL=ON \ + -DPRESTO_ENABLE_TESTING=OFF \ + -DPRESTO_ENABLE_S3=ON" + if [[ "${INPUT_OS}" == "ubuntu" ]]; then + EXTRA_CMAKE_FLAGS="$EXTRA_CMAKE_FLAGS -DCMAKE_CXX_FLAGS='-Wno-error=stringop-overflow -Wno-psabi'" + fi + if [[ "${{ matrix.arch }}" == "arm64-generic" ]]; then + BUILD_ARGS="--build-arg ARM_BUILD_TARGET=generic" + else + BUILD_ARGS="" + fi + + # Build the prestissimo image using standard Docker build + docker build \ + --build-arg EXTRA_CMAKE_FLAGS="$EXTRA_CMAKE_FLAGS" \ + --build-arg DEPENDENCY_IMAGE=${{ env.DEPENDENCY_IMAGE }} \ + --build-arg BASE_IMAGE=$BASE_IMAGE \ + --build-arg OSNAME=${INPUT_OS} \ + --build-arg BUILD_TYPE=Release \ + --build-arg NUM_THREADS=2 \ + ${BUILD_ARGS} \ + -f scripts/dockerfiles/prestissimo-runtime.dockerfile \ + -t ${{ env.IMAGE_TAG }} \ + . + + - name: Publish image + run: | + set -e + docker push ${{ env.IMAGE_TAG }} + + if [[ "${INPUT_TAG_LATEST}" == "true" ]]; then + LATEST_TAG="${{ env.ORG_NAME }}/${{ env.NATIVE_IMAGE_NAME }}:${INPUT_OS}-${{ matrix.arch }}-latest" + docker tag ${{ env.IMAGE_TAG }} ${LATEST_TAG} + docker push ${LATEST_TAG} + echo "Tagged and pushed as latest: ${LATEST_TAG}" + fi + + create-prestissimo-manifest: + if: (!failure() && !cancelled() && inputs.publish_prestissimo) + needs: [prepare, publish-prestissimo-image] + runs-on: ubuntu-24.04 + environment: release + permissions: + packages: write + contents: read + steps: + - name: Login to DockerHub + uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.0.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set version + run: | + echo "VERSION=${{ needs.prepare.outputs.version }}" >> $GITHUB_ENV + echo "Using version: ${{ needs.prepare.outputs.version }}" + + - name: Create and push multi-arch manifest + env: + ORG_NAME: ${{ github.repository_owner }} + IMAGE_NAME: ${{ env.NATIVE_IMAGE_NAME }} + OS: ${{ inputs.os }} + VERSION: ${{ env.VERSION }} + run: | + # Create manifest for the versioned tag + MANIFEST_TAG="${ORG_NAME}/${IMAGE_NAME}:${OS}-${VERSION}${TAG_SUFFIX}" + AMD64_TAG="${ORG_NAME}/${IMAGE_NAME}:${OS}-${VERSION}-amd64${TAG_SUFFIX}" + ARM64_GENERIC_TAG="${ORG_NAME}/${IMAGE_NAME}:${OS}-${VERSION}-arm64-generic${TAG_SUFFIX}" + ARM64_TAG="${ORG_NAME}/${IMAGE_NAME}:${OS}-${VERSION}-arm64${TAG_SUFFIX}" + + echo "Creating manifest: ${MANIFEST_TAG}" + docker manifest create ${MANIFEST_TAG} ${AMD64_TAG} ${ARM64_GENERIC_TAG} ${ARM64_TAG} + docker manifest push ${MANIFEST_TAG} + + # Create latest manifest if requested + if [[ "${INPUT_TAG_LATEST}" == "true" ]]; then + # Create OS-specific latest manifest + LATEST_MANIFEST="${ORG_NAME}/${IMAGE_NAME}:${OS}-latest" + LATEST_AMD64="${ORG_NAME}/${IMAGE_NAME}:${OS}-amd64-latest" + LATEST_ARM64_GENERIC="${ORG_NAME}/${IMAGE_NAME}:${OS}-arm64-generic-latest" + LATEST_ARM64="${ORG_NAME}/${IMAGE_NAME}:${OS}-arm64-latest" + + # If OS is centos, also tag as latest (without OS prefix) + if [[ "${OS}" == "centos" ]]; then + GLOBAL_LATEST="${ORG_NAME}/${IMAGE_NAME}:latest" + echo "Creating global latest manifest: ${GLOBAL_LATEST}" + docker manifest create ${GLOBAL_LATEST} ${LATEST_AMD64} ${LATEST_ARM64_GENERIC} ${LATEST_ARM64} + docker manifest push ${GLOBAL_LATEST} + else + echo "Creating OS-specific latest manifest: ${LATEST_MANIFEST}" + docker manifest create ${LATEST_MANIFEST} ${LATEST_AMD64} ${LATEST_ARM64_GENERIC} ${LATEST_ARM64} + docker manifest push ${LATEST_MANIFEST} + fi + fi diff --git a/presto-native-execution/scripts/dockerfiles/ubuntu-22.04-dependency.dockerfile b/presto-native-execution/scripts/dockerfiles/ubuntu-22.04-dependency.dockerfile index 8c548c2679adb..174509694dc6e 100644 --- a/presto-native-execution/scripts/dockerfiles/ubuntu-22.04-dependency.dockerfile +++ b/presto-native-execution/scripts/dockerfiles/ubuntu-22.04-dependency.dockerfile @@ -36,6 +36,15 @@ COPY velox/scripts /velox/scripts # from https://github.com/facebookincubator/velox/pull/14016 COPY velox/CMake/resolve_dependency_modules/arrow/cmake-compatibility.patch /velox ENV VELOX_ARROW_CMAKE_PATCH=/velox/cmake-compatibility.patch + +RUN if [ "$(dpkg --print-architecture)" = "arm64" ]; then \ + apt update && \ + apt install -y software-properties-common && \ + add-apt-repository -y ppa:ubuntu-toolchain-r/test && \ + apt update && \ + apt install -y gcc-12 g++-12; \ + fi + # install rpm needed for minio install. RUN mkdir build && \ (cd build && ../scripts/setup-ubuntu.sh && \