Merge from master into ADH-5740

Author: vitaliy-popov

buildall.sh

@@ -64,8 +64,8 @@ export -n BZIP2_VERSION
 # Build LLVM 3.7+ with and without assertions. For LLVM 3.7+, the default is a
 # release build with no assertions.
 (
-  LLVM_VERSION=5.0.1-p7 $SOURCE_DIR/source/llvm/build.sh
-  LLVM_VERSION=5.0.1-asserts-p7 $SOURCE_DIR/source/llvm/build.sh
+  LLVM_VERSION=5.0.1-p8 $SOURCE_DIR/source/llvm/build.sh
+  LLVM_VERSION=5.0.1-asserts-p8 $SOURCE_DIR/source/llvm/build.sh
 )
 
 ################################################################################
@@ -141,7 +141,7 @@ GFLAGS_VERSION=2.2.0-p2 GLOG_VERSION=0.3.5-p3 $SOURCE_DIR/source/glog/build.sh
 GTEST_VERSION=1.6.0 $SOURCE_DIR/source/gtest/build.sh
 
 # New versions of gtest are named googletest
-GOOGLETEST_VERSION=1.8.0 $SOURCE_DIR/source/googletest/build.sh
+GOOGLETEST_VERSION=1.14.0 $SOURCE_DIR/source/googletest/build.sh
 
 ################################################################################
 # Build Snappy
@@ -194,7 +194,7 @@ AVRO_VERSION=1.7.4-p5 $SOURCE_DIR/source/avro/build.sh
 ################################################################################
 # Build Rapidjson
 ################################################################################
-RAPIDJSON_VERSION=1.1.0 $SOURCE_DIR/source/rapidjson/build.sh
+RAPIDJSON_VERSION=1.1.0-p1 $SOURCE_DIR/source/rapidjson/build.sh
 
 ################################################################################
 # Build Libunwind
@@ -239,7 +239,6 @@ TPC_DS_VERSION=2.1.0-p1 $SOURCE_DIR/source/tpc-ds/build.sh
   export SNAPPY_VERSION=1.1.8
   export ZLIB_VERSION=1.2.13
   export ZSTD_VERSION=1.5.2
-  export GOOGLETEST_VERSION=1.8.0
   ORC_VERSION=1.7.9-p10 $SOURCE_DIR/source/orc/build.sh
 )
 
@@ -264,7 +263,7 @@ ARROW_VERSION=13.0.0 $SOURCE_DIR/source/arrow/build.sh
 
 # CURL
 ################################################################################
-CURL_VERSION=7.78.0 $SOURCE_DIR/source/curl/build.sh
+CURL_VERSION=8.10.1 $SOURCE_DIR/source/curl/build.sh
 
 # CALLONCEHACK
 ################################################################################
@@ -296,3 +295,11 @@ LIBPFM_VERSION=4.13.0 $SOURCE_DIR/source/libpfm/build.sh
 export LIBPFM_VERSION=4.13.0
 GOOGLEBENCHMARK_VERSION=1.8.3 $SOURCE_DIR/source/googlebenchmark/build.sh
 unset LIBPFM_VERSION
+
+# AWS-SDK-CPP
+################################################################################
+export CURL_VERSION=8.10.1
+export ZLIB_VERSION=1.3.1
+AWS_SDK_CPP_VERSION=1.11.485 $SOURCE_DIR/source/aws-sdk-cpp/build.sh
+unset CURL_VERSION
+unset ZLIB_VERSION

docker/all/assert-dependencies-present.py

@@ -124,7 +124,7 @@ def check_mvn_works():
 def check_ccache_works():
   # Older versions of ccache can cause build failures and weirdness (e.g. on Redhat 6)
   # Verify that the version we installed is present.
-  want = '3.3.3'
+  want = '3.7.12'
   LOG.info('Checking that ccache is correctly installed.')
   out = check_output(['ccache', '--version'])[0]
   if want not in out:
@@ -160,6 +160,7 @@ def main():
   check_python_headers_present()
   check_aws_works()
   check_mvn_works()
+  check_ccache_works()
   check_java_version()
 
 

docker/all/postinstall.sh

@@ -89,10 +89,10 @@ EOF
 }
 
 install_ccache() {
-  dl_verify https://github.com/ccache/ccache/releases/download/v3.3.3/ccache-3.3.3.tar.gz 87a399a2267cfac3f36411fbc12ff8959f408cffd050ad15fe423df88e977e8f
-  tar xvzf ccache-3.3.3.tar.gz
+  dl_verify https://github.com/ccache/ccache/releases/download/v3.7.12/ccache-3.7.12.tar.gz d2abe88d4c283ce960e233583061127b156ffb027c6da3cf10770fc0c7244194
+  tar xvzf ccache-3.7.12.tar.gz
   (
-  cd ccache-3.3.3
+  cd ccache-3.7.12
   ./configure
   make -j
   make install

functions.sh

@@ -374,6 +374,24 @@ function build_fake_package() {
   fi
 }
 
+function upload_to_s3() {
+  local FILENAME=$1
+  local S3_LOCATION=$2
+
+  # Upload to s3 can be flaky on certain platforms, so this retries a few times
+  # to compensate.
+  local NUM_RETRIES=10
+  for ((i=1; i<=${NUM_RETRIES}; i++)); do
+    echo "Uploading ${FILENAME} to ${S3_LOCATION} (attempt #${i})"
+    if aws s3 cp --only-show-errors ${FILENAME} ${S3_LOCATION} ; then
+      echo "Successfully uploaded ${FILENAME} to ${S3_LOCATION}"
+      return 0;
+    fi
+    echo "Failed to upload ${FILENAME} to ${S3_LOCATION}"
+  done
+  return 1;
+}
+
 # Build the RPM or DEB package depending on the operating system.
 # Depends on the LOCAL_INSTALL variable containing the target directory and the
 # TOOLCHAIN_BUILD_ID variable containing a unique string.
@@ -430,16 +448,12 @@ function build_dist_package() {
     local ARCH=$(uname -m)
     local PACKAGE_RELATIVE_URL="build/${TOOLCHAIN_BUILD_ID}/${PACKAGE}/${PACKAGE_VERSION}${PATCH_VERSION}-${COMPILER}-${COMPILER_VERSION}/${FULL_TAR_NAME}-${BUILD_LABEL}-${ARCH}.tar.gz"
     local PACKAGE_S3_DESTINATION="s3://${S3_BUCKET}/${PACKAGE_RELATIVE_URL}"
-    echo "Uploading ${PACKAGE_FINAL_TGZ} to ${PACKAGE_S3_DESTINATION}"
-    aws s3 cp --only-show-errors \
-        "${PACKAGE_FINAL_TGZ}" "${PACKAGE_S3_DESTINATION}"  || $RET_VAL
+    upload_to_s3 ${PACKAGE_FINAL_TGZ} ${PACKAGE_S3_DESTINATION} || $RET_VAL
     # S3_MIRROR_BUCKET may be empty for experimental builds
     if [[ -n ${S3_MIRROR_BUCKET:-} ]]; then
       local PACKAGE_MIRROR_DESTINATION="s3://${S3_MIRROR_BUCKET}/${PACKAGE_RELATIVE_URL}"
       echo "Uploading to mirror:"
-      echo "Uploading ${PACKAGE_FINAL_TGZ} to ${PACKAGE_MIRROR_DESTINATION}"
-      aws s3 cp --only-show-errors \
-          "${PACKAGE_FINAL_TGZ}" "${PACKAGE_MIRROR_DESTINATION}" || $RET_VAL
+      upload_to_s3 ${PACKAGE_FINAL_TGZ} ${PACKAGE_MIRROR_DESTINATION} || $RET_VAL
     fi
   fi
 }

init-compiler.sh

@@ -49,18 +49,20 @@ fi
 GDB_VERSION=12.1 "$SOURCE_DIR"/source/gdb/build.sh
 
 if [[ "$OSTYPE" =~ ^linux ]]; then
-  # ARCH_FLAGS are used to convey architectur dependent flags that should
+  # ARCH_FLAGS are used to convey architecture dependent flags that should
   # be obeyed by libraries explicitly needing this information.
   if [[ "$ARCH_NAME" == "ppc64le" ]]; then
-     ARCH_FLAGS="-mvsx -maltivec"
+    ARCH_FLAGS="-mvsx -maltivec"
   elif [[ "$ARCH_NAME" == "aarch64" ]]; then
     ARCH_FLAGS="-march=armv8-a"
   else
-     ARCH_FLAGS="-mno-avx2"
+    # x86_64
+    ARCH_FLAGS="-m64 -mno-avx2"
   fi
+  ARCH_CXXFLAGS=""
 elif [[ "$OSTYPE" == "darwin"* ]]; then
   # Setting the C++ stlib to libstdc++ on Mac instead of the default libc++
-  ARCH_FLAGS="-stdlib=libstdc++"
+  ARCH_CXXFLAGS="-stdlib=libstdc++"
 fi
 
 export SYSTEM_GCC_VERSION=$(gcc -dumpversion)
@@ -92,24 +94,15 @@ if [[ $SYSTEM_GCC -eq 0 ]]; then
 
   FULL_LPATH="-L$BUILD_DIR/gcc-$GCC_VERSION/lib64"
   LDFLAGS="$ARCH_FLAGS $FULL_RPATH $FULL_LPATH"
-
-  if [[ "$ARCH_NAME" == "aarch64" ]]; then
-    CXXFLAGS="$ARCH_FLAGS -fPIC -O3"
-  else
-    CXXFLAGS="$ARCH_FLAGS -fPIC -O3 -m64"
-  fi
 else
-  if [[ "$OSTYPE" == "darwin"* ]]; then
-    CXX="g++ -stdlib=libstdc++"
-  fi
   LDFLAGS=""
-  CXXFLAGS="-fPIC -O3 -m64"
-fi
-if [[ "$ARCH_NAME" == "aarch64" ]]; then
-  CFLAGS="-fPIC -O3"
-else
-  CFLAGS="-fPIC -O3 -m64"
 fi
+# -g1 enables basic debug information for backtraces
+# -gz enables compression of debug information
+# -gdwarf-4 sets the DWARF version to 4. It is better to be explicit about the DWARF
+#     version because newer compilers default to DWARF 5.
+CFLAGS="$ARCH_FLAGS -fPIC -O3 -g1 -gz -gdwarf-4"
+CXXFLAGS="$CFLAGS $ARCH_CXXFLAGS"
 if [[ $USE_CCACHE -ne 0 ]]; then
   CC=$(setup_ccache $CC)
   CXX=$(setup_ccache $CXX)

source/abseil-cpp/build.sh

@@ -45,7 +45,7 @@ mkdir build_static
 pushd build_static
 wrap cmake -DCMAKE_BUILD_TYPE=RELEASE -DCMAKE_INSTALL_PREFIX=$LOCAL_INSTALL \
      -DABSL_ENABLE_INSTALL=ON -DABSL_BUILD_TESTING=OFF ..
-wrap make -j${BUILD_THREADS:-4}
+wrap make VERBOSE=1 -j${BUILD_THREADS:-4}
 wrap make install
 popd
 
@@ -55,7 +55,7 @@ pushd build_shared
 wrap cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=RELEASE \
      -DCMAKE_INSTALL_PREFIX=$LOCAL_INSTALL \
      -DABSL_ENABLE_INSTALL=ON -DABSL_BUILD_TESTING=OFF ..
-wrap make -j${BUILD_THREADS:-4}
+wrap make VERBOSE=1 -j${BUILD_THREADS:-4}
 wrap make install
 popd
 

source/arrow/build.sh

@@ -30,6 +30,6 @@ if needs_build_package ; then
   cd cpp
   wrap cmake -DARROW_JSON=ON -DARROW_OPTIONAL_INSTALL=ON -DARROW_JEMALLOC=OFF \
       -DCMAKE_INSTALL_PREFIX=$LOCAL_INSTALL -DCMAKE_BUILD_TYPE=RELEASE
-  CFLAGS="-fPIC -DPIC" wrap make VERBOSE=1 -j${BUILD_THREADS:-4} all install
+  wrap make VERBOSE=1 -j${BUILD_THREADS:-4} all install
   finalize_package_build $PACKAGE $PACKAGE_VERSION
 fi

source/avro/build-cpp.sh

@@ -37,7 +37,7 @@ if needs_build_package ; then
     -DCMAKE_INSTALL_PREFIX=$LOCAL_INSTALL \
     -DBOOST_ROOT=${BOOST_ROOT} \
     ..
-  wrap make -C . -j${BUILD_THREADS:-4}
+  wrap make VERBOSE=1 -C . -j${BUILD_THREADS:-4}
 
   # Different versions of CMake produce different locations for the avro-c.pc file
   if [[ -e avro-c.pc ]]; then

source/avro/build.sh

@@ -33,7 +33,7 @@ if needs_build_package ; then
   mkdir -p build
   cd build
   wrap cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$LOCAL_INSTALL ..
-  wrap make -C . -j${BUILD_THREADS:-4}
+  wrap make VERBOSE=1 -C . -j${BUILD_THREADS:-4}
 
   # Different versions of CMake produce different locations for the avro-c.pc file
   if [[ -e avro-c.pc ]]; then

source/aws-sdk-cpp/build.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+# Copyright 2025 Cloudera Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Exit on non-true return value
+set -e
+# Exit on reference to uninitialized variable
+set -u
+
+set -o pipefail
+
+source $SOURCE_DIR/functions.sh
+THIS_DIR="$( cd "$( dirname "$0" )" && pwd )"
+prepare $THIS_DIR
+
+if needs_build_package ; then
+  download_dependency $PACKAGE "${PACKAGE_STRING}.tar.gz" $THIS_DIR
+  setup_package_build $PACKAGE $PACKAGE_VERSION
+  # This is required for downloading some dependencies.
+  ORIGINAL_PATH=$PATH
+  CURL_PATH=${BUILD_DIR}/curl-${CURL_VERSION}/bin
+  export PATH="$CURL_PATH:$PATH"
+  wrap ./prefetch_crt_dependency.sh
+  export PATH=$ORIGINAL_PATH
+  # Build only the required bedrock components.
+  wrap cmake -DCMAKE_INSTALL_PREFIX=$LOCAL_INSTALL -DCMAKE_BUILD_TYPE=RELEASE \
+    -DBUILD_ONLY="bedrock-runtime" -DENABLE_TESTING="OFF" -DUSE_OPENSSL="ON" \
+    -DBUILD_SHARED_LIBS="OFF" -DCPP_STANDARD="17" \
+    -DCURL_LIBRARY=${BUILD_DIR}/curl-${CURL_VERSION}/lib/libcurl.so \
+    -DCURL_INCLUDE_DIR=${BUILD_DIR}/curl-${CURL_VERSION}/include \
+    -DZLIB_ROOT=${BUILD_DIR}/zlib-${ZLIB_VERSION}
+  wrap make VERBOSE=1 -j${BUILD_THREADS:-4} install
+  finalize_package_build $PACKAGE $PACKAGE_VERSION
+fi