Skip to content

Commit 2922b87

Browse files
czentgrczentgr
committed
[native] Test standard runner prestocpp build
1 parent ed176b9 commit 2922b87

File tree

6 files changed

+391
-0
lines changed

6 files changed

+391
-0
lines changed

‎.github/workflows/format-check.yml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
name: format-check
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
paths:
7+
- 'presto-native-execution/**'
8+
9+
jobs:
10+
format-check:
11+
runs-on: ubuntu-latest
12+
container:
13+
image: public.ecr.aws/oss-presto/velox-dev:check
14+
steps:
15+
- uses: actions/[email protected]
16+
- name: Fix git permissions
17+
# Usually actions/checkout does this but as we run in a container
18+
# it doesn't work
19+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
20+
- name: Check formatting
21+
run: |
22+
git fetch origin master
23+
cd presto-native-execution
24+
make format-check

‎.github/workflows/header-check.yml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
name: header-check
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
paths:
7+
- 'presto-native-execution/**'
8+
9+
jobs:
10+
header-check:
11+
runs-on: ubuntu-latest
12+
container:
13+
image: public.ecr.aws/oss-presto/velox-dev:check
14+
steps:
15+
- uses: actions/[email protected]
16+
- name: Fix git permissions
17+
# Usually actions/checkout does this but as we run in a container
18+
# it doesn't work
19+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
20+
- name: Check license headers
21+
run: |
22+
git fetch origin master
23+
cd presto-native-execution
24+
make header-check

‎.github/workflows/linux-build-and-unit-test.yml‎

Lines changed: 241 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,241 @@
1+
2+
name: linux-build-and-unit-test
3+
4+
on:
5+
workflow_dispatch:
6+
pull_request:
7+
8+
jobs:
9+
prestocpp-linux-build-and-unit-test:
10+
runs-on: ubuntu-22.04
11+
container:
12+
image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090
13+
env:
14+
CCACHE_DIR: "${{ github.workspace }}/ccache"
15+
MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError"
16+
MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
17+
MAVEN_FAST_INSTALL: "-B -V --quiet -T C1 -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
18+
MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end"
19+
steps:
20+
- uses: actions/checkout@v4
21+
- name: Fix git permissions
22+
# Usually actions/checkout does this but as we run in a container
23+
# it doesn't work
24+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
25+
26+
- name: Update velox
27+
run: |
28+
cd presto-native-execution
29+
make velox-submodule
30+
31+
- name: Install prestocpp adapter dependencies
32+
run: |
33+
mkdir -p ${HOME}/adapter-deps/install
34+
source /opt/rh/gcc-toolset-12/enable
35+
set -xu
36+
cd presto-native-execution
37+
DEPENDENCY_DIR=${HOME}/adapter-deps PROMPT_ALWAYS_RESPOND=n ./scripts/setup-adapters.sh
38+
39+
- name: Install github CLI for the ccache stash
40+
run: ./presto-native-execution/scripts/setup-centos.sh install_gh
41+
42+
- name: Calculate merge-base date for ccache
43+
run: |
44+
git fetch origin master
45+
git show -s --format=%cd --date="format:%Y%m%d" $(git merge-base origin/master HEAD) | tee merge-base-date
46+
47+
- uses: apache/infrastructure-actions/stash/restore@4ab8682fbd4623d2b4fc1c98db38aba5091924c3
48+
with:
49+
path: '${{ env.CCACHE_DIR }}'
50+
key: ccache-linux-prestocpp-${{ runner.os }}-${{ hashFiles('merge-base-date') }}
51+
52+
- name: Zero ccache statistics
53+
run: ccache -sz
54+
55+
- name: Build engine
56+
run: |
57+
source /opt/rh/gcc-toolset-12/enable
58+
cd presto-native-execution
59+
cmake \
60+
-B _build/release \
61+
-GNinja \
62+
-DTREAT_WARNINGS_AS_ERRORS=1 \
63+
-DENABLE_ALL_WARNINGS=1 \
64+
-DCMAKE_BUILD_TYPE=Release \
65+
-DPRESTO_ENABLE_PARQUET=ON \
66+
-DPRESTO_ENABLE_REMOTE_FUNCTIONS=ON \
67+
-DPRESTO_ENABLE_JWT=ON \
68+
-DPRESTO_STATS_REPORTER_TYPE=PROMETHEUS \
69+
-DPRESTO_MEMORY_CHECKER_TYPE=LINUX_MEMORY_CHECKER \
70+
-DCMAKE_PREFIX_PATH=/usr/local \
71+
-DThrift_ROOT=/usr/local \
72+
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
73+
-DMAX_LINK_JOBS=2
74+
ninja -C _build/release -j 4
75+
76+
- name: Ccache after
77+
run: ccache -s
78+
79+
- uses: apache/infrastructure-actions/stash/restore@4ab8682fbd4623d2b4fc1c98db38aba5091924c3
80+
with:
81+
path: '${{ env.CCACHE_DIR }}'
82+
key: ccache-linux-prestocpp-${{ runner.os }}-${{ hashFiles('merge-base-date') }}
83+
84+
- name: Run Unit Tests
85+
run: |
86+
cd presto-native-execution/_build/release
87+
ctest -j 4 -VV --output-on-failure --exclude-regex velox.*
88+
89+
- name: Upload artifacts
90+
uses: actions/upload-artifact@v4
91+
with:
92+
name: presto-native-build
93+
path: |
94+
presto-native-execution/_build/release/presto_cpp/main/presto_server
95+
presto-native-execution/_build/release/velox/velox/functions/remote/server/velox_functions_remote_server_main
96+
97+
linux-presto-e2e-tests:
98+
needs: prestocpp-linux-build-and-unit-test
99+
runs-on: ubuntu-22.04
100+
container:
101+
image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090
102+
env:
103+
MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError"
104+
MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
105+
MAVEN_FAST_INSTALL: "-B -V --quiet -T C1 -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
106+
MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end"
107+
steps:
108+
- uses: actions/checkout@v4
109+
- name: Fix git permissions
110+
# Usually actions/checkout does this but as we run in a container
111+
# it doesn't work
112+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
113+
114+
- name: Download artifacts
115+
uses: actions/download-artifact@v4
116+
with:
117+
name: presto-native-build
118+
path: presto-native-execution/_build/release
119+
120+
- name: Install OpenJDK8
121+
uses: actions/setup-java@v4
122+
with:
123+
distribution: 'temurin'
124+
java-version: '8'
125+
126+
- name: Cache local Maven repository
127+
id: cache-maven
128+
uses: actions/cache@v4
129+
with:
130+
path: ~/.m2/repository
131+
key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }}
132+
restore-keys: |
133+
${{ runner.os }}-maven-2-
134+
135+
- name: Populate maven cache
136+
if: steps.cache-maven.outputs.cache-hit != 'true'
137+
run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies
138+
139+
- name: Maven install
140+
env:
141+
MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
142+
MAVEN_FAST_INSTALL: "-B -V --quiet -T C1 -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
143+
run: |
144+
for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-execution' -am && s=0 && break || s=$? && sleep 10; done; (exit $s)
145+
146+
- name: Run presto-native e2e tests
147+
run: |
148+
export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server"
149+
export TESTFILES=`find ./presto-native-execution/src/test -type f -name 'TestPrestoNative*.java'`
150+
# Convert file paths to comma separated class names
151+
export TESTCLASSES=
152+
for test_file in $TESTFILES
153+
do
154+
tmp=${test_file##*/}
155+
test_class=${tmp%%\.*}
156+
export TESTCLASSES="${TESTCLASSES},$test_class"
157+
done
158+
export TESTCLASSES=${TESTCLASSES#,}
159+
echo "TESTCLASSES = $TESTCLASSES"
160+
# TODO: neeed to enable remote function tests with
161+
# "-Ppresto-native-execution-remote-functions" once
162+
# > https://github.com/facebookincubator/velox/discussions/6163
163+
# is fixed.
164+
165+
mvn test \
166+
${MAVEN_TEST} \
167+
-pl 'presto-native-execution' \
168+
-Dtest="${TESTCLASSES}" \
169+
-DPRESTO_SERVER=${PRESTO_SERVER_PATH} \
170+
-DDATA_DIR=${RUNNER_TEMP} \
171+
-Duser.timezone=America/Bahia_Banderas \
172+
-T1C
173+
174+
linux-spark-e2e-tests:
175+
needs: prestocpp-linux-build-and-unit-test
176+
runs-on: ubuntu-22.04
177+
container:
178+
image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090
179+
env:
180+
MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError"
181+
MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
182+
MAVEN_FAST_INSTALL: "-B -V --quiet -T C1 -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
183+
MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end"
184+
steps:
185+
- uses: actions/checkout@v4
186+
- name: Fix git permissions
187+
# Usually actions/checkout does this but as we run in a container
188+
# it doesn't work
189+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
190+
191+
- name: Download artifacts
192+
uses: actions/download-artifact@v4
193+
with:
194+
name: presto-native-build
195+
path: presto-native-execution/_build/release
196+
197+
- name: Check Java
198+
run: java --version
199+
200+
- name: Cache local Maven repository
201+
id: cache-maven
202+
uses: actions/cache@v4
203+
with:
204+
path: ~/.m2/repository
205+
key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }}
206+
restore-keys: |
207+
${{ runner.os }}-maven-2-
208+
209+
- name: Populate maven cache
210+
if: steps.cache-maven.outputs.cache-hit != 'true'
211+
run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies
212+
213+
- name: Maven install
214+
env:
215+
MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
216+
MAVEN_FAST_INSTALL: "-B -V --quiet -T C1 -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
217+
run: |
218+
for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-execution' -am && s=0 && break || s=$? && sleep 10; done; (exit $s)
219+
220+
- name: Run spark e2e tests
221+
run: |
222+
export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server"
223+
export TESTFILES=`find ./presto-native-execution/src/test -type f -name 'TestPrestoSpark*.java'`
224+
# Convert file paths to comma separated class names
225+
export TESTCLASSES=
226+
for test_file in $TESTFILES
227+
do
228+
tmp=${test_file##*/}
229+
test_class=${tmp%%\.*}
230+
export TESTCLASSES="${TESTCLASSES},$test_class"
231+
done
232+
export TESTCLASSES=${TESTCLASSES#,}
233+
echo "TESTCLASSES = $TESTCLASSES"
234+
mvn test \
235+
${MAVEN_TEST} \
236+
-pl 'presto-native-execution' \
237+
-Dtest="${TESTCLASSES}" \
238+
-DPRESTO_SERVER=${PRESTO_SERVER_PATH} \
239+
-DDATA_DIR=${RUNNER_TEMP} \
240+
-Duser.timezone=America/Bahia_Banderas \
241+
-T1C

‎.github/workflows/macos-build.yml‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
name: macos-build
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
paths:
7+
- 'presto-native-execution/**'
8+
9+
jobs:
10+
macos-build:
11+
runs-on: macos-13
12+
steps:
13+
- uses: actions/[email protected]
14+
- name: Fix git permissions
15+
# Usually actions/checkout does this but as we run in a container
16+
# it doesn't work
17+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
18+
- name: Update submodules
19+
run: |
20+
git config --global --add safe.directory $(pwd)
21+
cd presto-native-execution
22+
make submodules
23+
- uses: maxim-lobanov/[email protected]
24+
with:
25+
xcode-version: 15
26+
- name: "Setup MacOS"
27+
run: |
28+
set -xu
29+
mkdir ~/deps ~/deps-src
30+
git clone --depth 1 https://github.com/Homebrew/brew ~/deps
31+
PATH=~/deps/bin:${PATH} DEPENDENCY_DIR=~/deps-src INSTALL_PREFIX=~/deps PROMPT_ALWAYS_RESPOND=n ./presto-native-execution/scripts/setup-macos.sh
32+
# Calculate the prefix path before we delete brew's repos and taps.
33+
rm -rf ~/deps/.git ~/deps/Library/Taps/ # Reduce cache size by 70%.
34+
rm -rf ~/deps-src
35+
- name: "Build presto_cpp on MacOS"
36+
run: |
37+
clang --version
38+
export INSTALL_PREFIX=~/deps
39+
export PATH=~/deps/bin:$(brew --prefix m4)/bin:$(brew --prefix bison)/bin:${PATH}
40+
cd presto-native-execution
41+
cmake -B _build/debug -GNinja -DTREAT_WARNINGS_AS_ERRORS=1 -DENABLE_ALL_WARNINGS=1 -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
42+
ninja -C _build/debug

‎.github/workflows/prestocpp-linux-build.yml‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
name: prestocpp-linux-build
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
paths:
7+
- 'presto-native-execution/**'
8+
9+
jobs:
10+
linux-build-all:
11+
runs-on: ubuntu-22.04
12+
container:
13+
image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090
14+
steps:
15+
- uses: actions/[email protected]
16+
- name: Fix git permissions
17+
# Usually actions/checkout does this but as we run in a container
18+
# it doesn't work
19+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
20+
- name: Update velox
21+
run: |
22+
cd presto-native-execution
23+
make velox-submodule
24+
- name: Install all adapter dependencies
25+
run: |
26+
mkdir -p ${GITHUB_WORKSPACE}/adapter-deps/install
27+
source /opt/rh/gcc-toolset-12/enable
28+
set -xu
29+
cd presto-native-execution
30+
DEPENDENCY_DIR=${GITHUB_WORKSPACE}/adapter-deps PROMPT_ALWAYS_RESPOND=n ./velox/scripts/setup-adapters.sh
31+
DEPENDENCY_DIR=${GITHUB_WORKSPACE}/adapter-deps PROMPT_ALWAYS_RESPOND=n ./scripts/setup-adapters.sh
32+
- name: Build engine
33+
run: |
34+
source /opt/rh/gcc-toolset-12/enable
35+
cd presto-native-execution
36+
cmake \
37+
-B _build/debug \
38+
-GNinja \
39+
-DAWSSDK_ROOT_DIR=${GITHUB_WORKSPACE}/adapter-deps/install \
40+
-DTREAT_WARNINGS_AS_ERRORS=1 \
41+
-DENABLE_ALL_WARNINGS=1 \
42+
-DCMAKE_BUILD_TYPE=Debug \
43+
-DPRESTO_ENABLE_PARQUET=ON \
44+
-DPRESTO_ENABLE_S3=ON \
45+
-DPRESTO_ENABLE_REMOTE_FUNCTIONS=ON \
46+
-DPRESTO_ENABLE_JWT=ON \
47+
-DPRESTO_STATS_REPORTER_TYPE=PROMETHEUS \
48+
-DPRESTO_MEMORY_CHECKER_TYPE=LINUX_MEMORY_CHECKER \
49+
-DPRESTO_ENABLE_TESTING=OFF \
50+
-DCMAKE_PREFIX_PATH=/usr/local \
51+
-DThrift_ROOT=/usr/local \
52+
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
53+
-DMAX_LINK_JOBS=2
54+
ninja -C _build/debug -j 4

0 commit comments

Comments
 (0)