Skip to content

Commit 5757d48

Browse files
yokoflyyokofly
authored
CI switch to clang-19 and misc build enhance (#921)
1 parent dca34fa commit 5757d48

File tree

11 files changed

+112
-94
lines changed

11 files changed

+112
-94
lines changed

‎.github/workflows/manual_trigger_build.yml

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ on: # yamllint disable-line rule:truthy
1515
- memory
1616
- undefined
1717
arch:
18-
descrition: 'Hardware architecture'
18+
description: 'Hardware architecture'
1919
required: true
2020
default: x64
2121
type: choice
@@ -55,7 +55,7 @@ jobs:
5555
rm cache${SANITIZER:+_}$SANITIZER${ARCH:+_}$ARCH.tar.gz
5656
5757
# compiling
58-
./docker/packager/packager --package-type binary --docker-image-version clang-16 --sanitizer $SANITIZER --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output
58+
./docker/packager/packager --package-type binary --docker-image-version clang-19 --sanitizer $SANITIZER --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output
5959
6060
# clear compiling footprint
6161
rm -rf $GITHUB_WORKSPACE/ccache
@@ -103,7 +103,7 @@ jobs:
103103
rm cache${ARCH:+_}$ARCH.tar.gz
104104
105105
# compiling
106-
./docker/packager/packager --package-type binary --docker-image-version clang-16 --build-type release --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output
106+
./docker/packager/packager --package-type binary --docker-image-version clang-19 --build-type release --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output
107107
# release doesn't build unit_tests_dbms
108108
# clear compiling footprint
109109
rm -rf $GITHUB_WORKSPACE/ccache
@@ -232,9 +232,9 @@ jobs:
232232
- name: Create and Configure Build
233233
run: |
234234
mkdir -p ${{ env.build_directory }}
235-
export CC=$(brew --prefix llvm@17)/bin/clang
236-
export CXX=$(brew --prefix llvm@17)/bin/clang++
237-
export PATH=$(brew --prefix llvm@17)/bin:$PATH
235+
export CC=$(brew --prefix llvm@19)/bin/clang
236+
export CXX=$(brew --prefix llvm@19)/bin/clang++
237+
export PATH=$(brew --prefix llvm@19)/bin:$PATH
238238
cmake -B ${{ env.build_directory }} -G "Ninja" -DCMAKE_BUILD_TYPE=${{ env.build_type }} -DENABLE_TESTS=OFF -DENABLE_UTILS=OFF -DENABLE_EXAMPLES=OFF -DENABLE_PULSAR=OFF
239239
240240
- name: Build with Ninja
@@ -247,7 +247,7 @@ jobs:
247247
ORIGINAL_BINARY=${{ env.build_directory }}/programs/proton
248248
STRIPPED_BINARY=${{ env.build_directory }}/programs/proton_${COMMIT_HASH}_${ARCH}
249249
ls -lh $ORIGINAL_BINARY
250-
/opt/homebrew/opt/llvm@17/bin/llvm-strip $ORIGINAL_BINARY
250+
/opt/homebrew/opt/llvm@19/bin/llvm-strip $ORIGINAL_BINARY
251251
mv $ORIGINAL_BINARY $STRIPPED_BINARY
252252
ls -lh $STRIPPED_BINARY
253253

‎.github/workflows/nightly_test.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ jobs:
3030
mkdir $GITHUB_WORKSPACE/ccache
3131
echo "max_size = 100.0G" > $GITHUB_WORKSPACE/ccache/ccache.conf
3232
33-
./docker/packager/packager --package-type binary --docker-image-version clang-16 ${SANITIZER:+--sanitizer} $SANITIZER --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output
33+
./docker/packager/packager --package-type binary --docker-image-version clang-19 ${SANITIZER:+--sanitizer} $SANITIZER --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output
3434
3535
if [ ! -f "$GITHUB_WORKSPACE/output/proton" ]; then
3636
echo "Compiling proton Failed"
@@ -179,7 +179,7 @@ jobs:
179179
mkdir $GITHUB_WORKSPACE/ccache
180180
echo "max_size = 100.0G" > $GITHUB_WORKSPACE/ccache/ccache.conf
181181
182-
./docker/packager/packager --package-type binary --docker-image-version clang-16 --build-type release --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output
182+
./docker/packager/packager --package-type binary --docker-image-version clang-19 --build-type release --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output
183183
184184
if [ ! -f "$GITHUB_WORKSPACE/output/proton" ]; then
185185
echo "Compiling proton Failed"
@@ -300,7 +300,7 @@ jobs:
300300
mkdir $GITHUB_WORKSPACE/ccache
301301
echo "max_size = 100.0G" > $GITHUB_WORKSPACE/ccache/ccache.conf
302302
303-
./docker/packager/packager --package-type binary --docker-image-version clang-16 ${SANITIZER:+--sanitizer} $SANITIZER --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output
303+
./docker/packager/packager --package-type binary --docker-image-version clang-19 ${SANITIZER:+--sanitizer} $SANITIZER --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output
304304
305305
if [ ! -f "$GITHUB_WORKSPACE/output/proton" ]; then
306306
echo "Compiling proton Failed"

‎.github/workflows/proton_ci.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ jobs:
6666
rm cache${SANITIZER:+_}$SANITIZER${ARCH:+_}$ARCH.tar.gz
6767
6868
# compiling
69-
./docker/packager/packager --package-type binary --docker-image-version clang-16 --sanitizer $SANITIZER --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output
69+
./docker/packager/packager --package-type binary --docker-image-version clang-19 --sanitizer $SANITIZER --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output
7070
7171
tar -zcf ./cache${SANITIZER:+_}$SANITIZER${ARCH:+_}$ARCH.tar.gz -C $GITHUB_WORKSPACE/ccache .
7272
aws s3 cp --no-progress ./cache${SANITIZER:+_}$SANITIZER${ARCH:+_}$ARCH.tar.gz s3://tp-internal/proton/ci_cache/

‎.github/workflows/release_build.yml

Lines changed: 46 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ jobs:
88
Build_Linux_X86_64:
99
uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
1010
with:
11-
ec2-instance-type: c5.4xlarge
11+
ec2-instance-type: c7i.8xlarge
1212
ec2-image-id: ami-042a37e33a285c22b
1313
submodules: 'recursive'
1414
run_mode: 'start' # start ec2 on demand instance
@@ -30,7 +30,7 @@ jobs:
3030
rm cache.tar.gz
3131
3232
# compiling
33-
./docker/packager/packager --package-type binary --docker-image-version clang-16 --build-type release --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output
33+
./docker/packager/packager --package-type binary --docker-image-version clang-19 --build-type release --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output
3434
3535
if [ ! -f "$GITHUB_WORKSPACE/output/proton" ]; then
3636
echo "Compiling proton Failed"
@@ -75,7 +75,7 @@ jobs:
7575
Build_Linux_Arm64:
7676
uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
7777
with:
78-
ec2-instance-type: c6g.4xlarge
78+
ec2-instance-type: c6g.8xlarge
7979
ec2-image-id: ami-0f3dbc4cc9994fdee
8080
submodules: 'recursive'
8181
run_mode: 'start' # start ec2 on demand instance
@@ -97,7 +97,7 @@ jobs:
9797
rm cache-arm.tar.gz
9898
9999
# compiling
100-
./docker/packager/packager --package-type binary --docker-image-version clang-16 --build-type release --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output
100+
./docker/packager/packager --package-type binary --docker-image-version clang-19 --build-type release --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output
101101
102102
# get proton tag
103103
PROTON_TAG=`grep "SET(VERSION_DESCRIBE" $GITHUB_WORKSPACE/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_DESCRIBE \(.*\)$/\1/' | sed 's/[) ].*//'`
@@ -190,7 +190,7 @@ jobs:
190190
Build_Darwin_X86_64:
191191
uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
192192
with:
193-
ec2-instance-type: c5.4xlarge
193+
ec2-instance-type: c7i.8xlarge
194194
ec2-image-id: ami-042a37e33a285c22b
195195
submodules: 'recursive'
196196
run_mode: 'start' # start ec2 on demand instance
@@ -218,7 +218,7 @@ jobs:
218218
echo "max_size = 100.0G" > $GITHUB_WORKSPACE/ccache/ccache.conf
219219
220220
# compiling
221-
./docker/packager/packager --package-type binary --docker-image-version clang-16 --build-type release --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output --compiler clang-16-darwin
221+
./docker/packager/packager --package-type binary --docker-image-version clang-19 --build-type release --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output --compiler clang-19-darwin
222222
223223
if [ ! -f "$GITHUB_WORKSPACE/output/proton" ]; then
224224
echo "Compiling proton Failed"
@@ -276,7 +276,7 @@ jobs:
276276
mkdir $GITHUB_WORKSPACE/ccache
277277
echo "max_size = 100.0G" > $GITHUB_WORKSPACE/ccache/ccache.conf
278278
# compiling
279-
./docker/packager/packager --package-type binary --docker-image-version clang-16 --build-type release --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output --compiler clang-16-darwin-aarch64
279+
./docker/packager/packager --package-type binary --docker-image-version clang-19 --build-type release --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output --compiler clang-19-darwin-aarch64
280280
if [ ! -f "$GITHUB_WORKSPACE/output/proton" ]; then
281281
echo "Compiling proton Failed"
282282
exit 127
@@ -329,9 +329,9 @@ jobs:
329329
- name: Create and Configure Build
330330
run: |
331331
mkdir -p ${{ env.build_directory }}
332-
export CC=$(brew --prefix llvm@17)/bin/clang
333-
export CXX=$(brew --prefix llvm@17)/bin/clang++
334-
export PATH=$(brew --prefix llvm@17)/bin:$PATH
332+
export CC=$(brew --prefix llvm@19)/bin/clang
333+
export CXX=$(brew --prefix llvm@19)/bin/clang++
334+
export PATH=$(brew --prefix llvm@19)/bin:$PATH
335335
cmake -B ${{ env.build_directory }} -G "Ninja" -DCMAKE_BUILD_TYPE=${{ env.build_type }} -DENABLE_TESTS=OFF -DENABLE_UTILS=OFF -DENABLE_EXAMPLES=OFF -DENABLE_PULSAR=OFF
336336
337337
# Compile the project using Ninja
@@ -343,7 +343,7 @@ jobs:
343343
run: |
344344
ORIGINAL_BINARY=${{ env.build_directory }}/programs/proton
345345
STRIPPED_BINARY=${{ env.build_directory }}/programs/proton-${{ env.LATEST_TAG }}-Darwin-arm64
346-
/opt/homebrew/opt/llvm@17/bin/llvm-strip $ORIGINAL_BINARY -o $STRIPPED_BINARY
346+
/opt/homebrew/opt/llvm@19/bin/llvm-strip $ORIGINAL_BINARY -o $STRIPPED_BINARY
347347
348348
# Set up AWS credentials for S3 upload
349349
- name: Configure AWS credentials
@@ -353,11 +353,44 @@ jobs:
353353
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
354354
aws-region: ${{ secrets.AWS_REGION }}
355355

356-
# Upload the stripped binary to an AWS S3 bucket
356+
# Upload the stripped binary to an AWS S3 bucket with improved retry
357357
- name: Upload Artifact To S3
358358
run: |
359359
STRIPPED_BINARY=${{ env.build_directory }}/programs/proton-${{ env.LATEST_TAG }}-Darwin-arm64
360-
aws s3 cp --no-progress $STRIPPED_BINARY s3://tp-internal/proton/native_build_macOS/
360+
361+
# Advanced retry logic with exponential backoff
362+
max_attempts=5
363+
attempt=1
364+
wait_time=15
365+
366+
while [ $attempt -le $max_attempts ]; do
367+
echo "S3 upload attempt $attempt of $max_attempts"
368+
369+
# Using specific AWS CLI options to handle large file uploads more reliably
370+
if aws s3 cp \
371+
--no-progress \
372+
--only-show-errors \
373+
--cli-connect-timeout 30 \
374+
--cli-read-timeout 600 \
375+
"$STRIPPED_BINARY" \
376+
"s3://tp-internal/proton/native_build_macOS/"; then
377+
echo "Upload successful"
378+
break
379+
else
380+
upload_status=$?
381+
echo "Upload failed on attempt $attempt with exit code $upload_status"
382+
383+
if [ $attempt -eq $max_attempts ]; then
384+
echo "All upload attempts failed"
385+
exit 1
386+
fi
387+
388+
echo "Retrying in $wait_time seconds..."
389+
sleep $wait_time
390+
wait_time=$((wait_time * 2)) # Exponential backoff
391+
attempt=$((attempt + 1))
392+
fi
393+
done
361394
362395
# Print SHA256 Checksum
363396
- name: Print SHA256 Checksum

‎BUILD.md

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -7,27 +7,27 @@ Proton requires the Clang compiler and related LLVM toolchain. GCC is not suppor
77
### Build with docker container
88

99
```sh
10-
./docker/packager/packager --package-type binary --docker-image-version clang-17 --proton-build --enable-proton-local --output-dir `pwd`/build_output
10+
./docker/packager/packager --package-type binary --docker-image-version clang-19 --proton-build --enable-proton-local --output-dir `pwd`/build_output
1111
```
1212

1313
### Bare metal build
1414

1515
#### Install toolchain
1616

17-
- clang-17 /clang++-17 or above (minimum supported version: Clang 16, GCC not supported)
17+
- clang-19 /clang++-19 or above (minimum supported version: Clang 16, GCC not supported)
1818
- cmake 3.20 or above
1919
- ninja
2020

2121
```sh
2222
apt install git cmake ccache python3 ninja-build wget apt-transport-https apt-utils ca-certificates dnsutils gnupg iputils-ping lsb-release gpg curl software-properties-common
2323
```
2424

25-
install llvm-17 compiler
25+
install llvm-19 compiler
2626

2727
```sh
2828
wget https://apt.llvm.org/llvm.sh
2929
chmod +x llvm.sh
30-
sudo ./llvm.sh 17
30+
sudo ./llvm.sh 19
3131
```
3232

3333
#### Build
@@ -60,7 +60,7 @@ $ ninja
6060

6161
#### Install toolchain
6262

63-
- clang-17 /clang++-17 or above
63+
- clang-19 /clang++-19 or above
6464
- cmake 3.20 or above
6565
- ninja
6666

@@ -81,7 +81,7 @@ $ ninja
8181

8282
#### Install toolchain
8383

84-
- clang-17 /clang++-17 or above
84+
- clang-19 /clang++-19 or above
8585
- cmake 3.20 or above
8686
- ninja
8787

@@ -102,7 +102,7 @@ $ ninja
102102

103103
#### Install toolchain
104104

105-
- clang-17 /clang++-17 or above
105+
- clang-19 /clang++-19 or above
106106
- cmake 3.20 or above
107107
- ninja
108108

@@ -126,10 +126,10 @@ Not supported.
126126
#### Install toolchain
127127

128128
We don't support build Proton by using Apple Clang. Please use `brew install llvm` to install
129-
clang-17 / clang++-17.
129+
clang-19 / clang++-19.
130130

131131

132-
- clang-17 /clang++-17 or above
132+
- clang-19 /clang++-19 or above
133133
- cmake 3.20 or above
134134
- ninja
135135

@@ -177,26 +177,26 @@ Build version 15C5042i
177177

178178
#### Step 2: Install Dependencies with Homebrew
179179

180-
Proton build is not supported with Apple Clang. Use Homebrew to install LLVM version 17 instead:
180+
Proton build is not supported with Apple Clang. Use Homebrew to install LLVM version 19 instead:
181181
```shell
182-
brew install llvm@17
182+
brew install llvm@19
183183
```
184184

185185
First, if you haven't installed Homebrew yet, follow the instructions at [https://brew.sh/](https://brew.sh/).
186186

187187
Next, install the required dependencies using the following commands:
188188
```shell
189189
brew update
190-
brew install ccache cmake ninja libtool gettext llvm@17 gcc binutils grep findutils libiconv
190+
brew install ccache cmake ninja libtool gettext llvm@19 gcc binutils grep findutils libiconv
191191
```
192192

193193
#### Step 3: Build Proton Manually
194194

195195
Set up the environment variables required for building Proton:
196196
```shell
197-
export PATH=$(brew --prefix llvm@17)/bin:$PATH
198-
export CC=$(brew --prefix llvm@17)/bin/clang
199-
export CXX=$(brew --prefix llvm@17)/bin/clang++
197+
export PATH=$(brew --prefix llvm@19)/bin:$PATH
198+
export CC=$(brew --prefix llvm@19)/bin/clang
199+
export CXX=$(brew --prefix llvm@19)/bin/clang++
200200
```
201201

202202
Clone the Proton repository and initiate the build process:

‎contrib/google-protobuf-cmake/CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -358,7 +358,7 @@ else ()
358358

359359
# This is quite ugly but I cannot make dependencies work propery.
360360

361-
set(abseil_source_dir "${ClickHouse_SOURCE_DIR}/contrib/abseil-cpp")
361+
set(abseil_source_dir "${proton_SOURCE_DIR}/contrib/abseil-cpp")
362362

363363
execute_process(
364364
COMMAND mkdir -p ${PROTOC_BUILD_DIR}

0 commit comments

Comments
 (0)